From 447e23392451f1c91c78a7465fb1680e8f21384b Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Mon, 27 Jun 2016 12:48:57 +0530 Subject: [PATCH 001/295] Raise version number after cloning 5.5.51 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index db9d497c141a9..acabf9b42d0ca 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=51 +MYSQL_VERSION_PATCH=52 MYSQL_VERSION_EXTRA= From 6986645c7924ef3292e233d9365520b29eb8fa0d Mon Sep 17 00:00:00 2001 From: Christopher Powers Date: Thu, 30 Jun 2016 20:42:29 +0200 Subject: [PATCH 002/295] Bug#14111584 PB2: PERFSCHEMA.AGGREGATE FAILS ON PB2 SPORADICALLY Permanently removed test case perfschema.aggregate. The Performance Schema is generally lock-free, allowing for race conditions that might arise from multi-threaded operation which occasionally results in temporary and/or minor variances when aggregating statistics. This test needs to be redesigned to accommodate such variances. --- .../suite/perfschema/r/aggregate.result | 102 ---------- mysql-test/suite/perfschema/t/aggregate.test | 174 ------------------ 2 files changed, 276 deletions(-) delete mode 100644 mysql-test/suite/perfschema/r/aggregate.result delete mode 100644 mysql-test/suite/perfschema/t/aggregate.test diff --git a/mysql-test/suite/perfschema/r/aggregate.result b/mysql-test/suite/perfschema/r/aggregate.result deleted file mode 100644 index edc7ce0bcca19..0000000000000 --- a/mysql-test/suite/perfschema/r/aggregate.result +++ /dev/null @@ -1,102 +0,0 @@ -"General cleanup" -drop table if exists t1; -update performance_schema.setup_instruments set enabled = 'NO'; -update performance_schema.setup_consumers set enabled = 'NO'; -truncate table performance_schema.file_summary_by_event_name; -truncate table performance_schema.file_summary_by_instance; -truncate table performance_schema.events_waits_summary_global_by_event_name; -truncate table performance_schema.events_waits_summary_by_instance; -truncate table performance_schema.events_waits_summary_by_thread_by_event_name; -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments -set enabled = 'YES', timed = 'YES'; -create table t1 ( -id INT PRIMARY KEY, -b CHAR(100) DEFAULT 'initial value') -ENGINE=MyISAM; -insert into t1 (id) values (1), (2), (3), (4), (5), (6), (7), (8); -update performance_schema.setup_instruments SET enabled = 'NO'; -update performance_schema.setup_consumers set enabled = 'NO'; -set @dump_all=FALSE; -"Verifying file aggregate consistency" -SELECT EVENT_NAME, e.COUNT_READ, SUM(i.COUNT_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_READ <> SUM(i.COUNT_READ)) -OR @dump_all; -EVENT_NAME COUNT_READ SUM(i.COUNT_READ) -SELECT EVENT_NAME, e.COUNT_WRITE, SUM(i.COUNT_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_WRITE <> SUM(i.COUNT_WRITE)) -OR @dump_all; -EVENT_NAME COUNT_WRITE SUM(i.COUNT_WRITE) -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_READ, SUM(i.SUM_NUMBER_OF_BYTES_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_READ <> SUM(i.SUM_NUMBER_OF_BYTES_READ)) -OR @dump_all; -EVENT_NAME SUM_NUMBER_OF_BYTES_READ SUM(i.SUM_NUMBER_OF_BYTES_READ) -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_WRITE, SUM(i.SUM_NUMBER_OF_BYTES_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_WRITE <> SUM(i.SUM_NUMBER_OF_BYTES_WRITE)) -OR @dump_all; -EVENT_NAME SUM_NUMBER_OF_BYTES_WRITE SUM(i.SUM_NUMBER_OF_BYTES_WRITE) -"Verifying waits aggregate consistency (instance)" -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(i.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(i.SUM_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME SUM_TIMER_WAIT SUM(i.SUM_TIMER_WAIT) -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(i.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(i.MIN_TIMER_WAIT)) -AND (MIN(i.MIN_TIMER_WAIT) != 0) -OR @dump_all; -EVENT_NAME MIN_TIMER_WAIT MIN(i.MIN_TIMER_WAIT) -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(i.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(i.MAX_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME MAX_TIMER_WAIT MAX(i.MAX_TIMER_WAIT) -"Verifying waits aggregate consistency (thread)" -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(t.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(t.SUM_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME SUM_TIMER_WAIT SUM(t.SUM_TIMER_WAIT) -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(t.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(t.MIN_TIMER_WAIT)) -AND (MIN(t.MIN_TIMER_WAIT) != 0) -OR @dump_all; -EVENT_NAME MIN_TIMER_WAIT MIN(t.MIN_TIMER_WAIT) -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(t.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(t.MAX_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME MAX_TIMER_WAIT MAX(t.MAX_TIMER_WAIT) -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments -set enabled = 'YES', timed = 'YES'; -drop table test.t1; diff --git a/mysql-test/suite/perfschema/t/aggregate.test b/mysql-test/suite/perfschema/t/aggregate.test deleted file mode 100644 index 91794f0434607..0000000000000 --- a/mysql-test/suite/perfschema/t/aggregate.test +++ /dev/null @@ -1,174 +0,0 @@ -# Tests for PERFORMANCE_SCHEMA -# Verify that statistics aggregated by different criteria are consistent. - ---source include/not_embedded.inc ---source include/have_perfschema.inc - ---echo "General cleanup" - ---disable_warnings -drop table if exists t1; ---enable_warnings - -update performance_schema.setup_instruments set enabled = 'NO'; -update performance_schema.setup_consumers set enabled = 'NO'; - -# Cleanup statistics -truncate table performance_schema.file_summary_by_event_name; -truncate table performance_schema.file_summary_by_instance; -truncate table performance_schema.events_waits_summary_global_by_event_name; -truncate table performance_schema.events_waits_summary_by_instance; -truncate table performance_schema.events_waits_summary_by_thread_by_event_name; - -# Start recording data -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments - set enabled = 'YES', timed = 'YES'; - - -create table t1 ( - id INT PRIMARY KEY, - b CHAR(100) DEFAULT 'initial value') - ENGINE=MyISAM; - -insert into t1 (id) values (1), (2), (3), (4), (5), (6), (7), (8); - -# Stop recording data, so the select below don't add noise. -update performance_schema.setup_instruments SET enabled = 'NO'; -# Disable all consumers, for long standing waits -update performance_schema.setup_consumers set enabled = 'NO'; - -# Helper to debug -set @dump_all=FALSE; - -# Note that in general: -# - COUNT/SUM/MAX(file_summary_by_event_name) >= -# COUNT/SUM/MAX(file_summary_by_instance). -# - MIN(file_summary_by_event_name) <= -# MIN(file_summary_by_instance). -# There will be equality only when file instances are not removed, -# aka when a file is not deleted from the file system, -# because doing so removes a row in file_summary_by_instance. - -# Likewise: -# - COUNT/SUM/MAX(events_waits_summary_global_by_event_name) >= -# COUNT/SUM/MAX(events_waits_summary_by_instance) -# - MIN(events_waits_summary_global_by_event_name) <= -# MIN(events_waits_summary_by_instance) -# There will be equality only when an instrument instance -# is not removed, which is next to impossible to predictably guarantee -# in the server. -# For example, a MyISAM table removed from the table cache -# will cause a mysql_mutex_destroy on myisam/MYISAM_SHARE::intern_lock. -# Another example, a thread terminating will cause a mysql_mutex_destroy -# on sql/LOCK_delete -# Both cause a row to be deleted from events_waits_summary_by_instance. - -# Likewise: -# - COUNT/SUM/MAX(events_waits_summary_global_by_event_name) >= -# COUNT/SUM/MAX(events_waits_summary_by_thread_by_event_name) -# - MIN(events_waits_summary_global_by_event_name) <= -# MIN(events_waits_summary_by_thread_by_event_name) -# There will be equality only when no thread is removed, -# that is if no thread disconnects, or no sub thread (for example insert -# delayed) ever completes. -# A thread completing will cause rows in -# events_waits_summary_by_thread_by_event_name to be removed. - ---echo "Verifying file aggregate consistency" - -# Since the code generating the load in this test does: -# - create table -# - insert -# - does not cause temporary tables to be used -# we can test for equality here for file aggregates. - -# If any of these queries returns data, the test failed. - -SELECT EVENT_NAME, e.COUNT_READ, SUM(i.COUNT_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_READ <> SUM(i.COUNT_READ)) -OR @dump_all; - -SELECT EVENT_NAME, e.COUNT_WRITE, SUM(i.COUNT_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_WRITE <> SUM(i.COUNT_WRITE)) -OR @dump_all; - -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_READ, SUM(i.SUM_NUMBER_OF_BYTES_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_READ <> SUM(i.SUM_NUMBER_OF_BYTES_READ)) -OR @dump_all; - -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_WRITE, SUM(i.SUM_NUMBER_OF_BYTES_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_WRITE <> SUM(i.SUM_NUMBER_OF_BYTES_WRITE)) -OR @dump_all; - ---echo "Verifying waits aggregate consistency (instance)" - -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(i.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(i.SUM_TIMER_WAIT)) -OR @dump_all; - -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(i.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(i.MIN_TIMER_WAIT)) -AND (MIN(i.MIN_TIMER_WAIT) != 0) -OR @dump_all; - -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(i.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(i.MAX_TIMER_WAIT)) -OR @dump_all; - ---echo "Verifying waits aggregate consistency (thread)" - -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(t.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(t.SUM_TIMER_WAIT)) -OR @dump_all; - -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(t.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(t.MIN_TIMER_WAIT)) -AND (MIN(t.MIN_TIMER_WAIT) != 0) -OR @dump_all; - -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(t.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(t.MAX_TIMER_WAIT)) -OR @dump_all; - - -# Cleanup - -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments - set enabled = 'YES', timed = 'YES'; - -drop table test.t1; From 07a33cdcef16b21c5d9bb5f0c538066e24eb8dc0 Mon Sep 17 00:00:00 2001 From: Kailasnath Nagarkar Date: Fri, 1 Jul 2016 12:01:27 +0530 Subject: [PATCH 003/295] Bug #23296299 : HANDLE_FATAL_SIGNAL (SIG=11) IN MY_TOSORT_UTF32 This patch is specific for mysql-5.5 ISSUE: When a charater that is larger than possible to handle is passed to function my_tosort_utf32(), it results in segmentation fault. In the scenario mentioned in the bug AES_ENCRYPT function is used which returns large value. This value is further passed to my_tosort_utf32 function. This causes to cross array bound for array uni_plane, resulting in segment violation. SOLUTION: This issue has got addressed in 5.6 onward releases through worklog 2673. The fix is similar backport of that. Check for maximum character before accessing the array uni_plane. In addition to function my_tosort_utf32, the same potential problem is also present in functions my_tolower_utf16, my_toupper_utf16, my_tosort_utf16, my_tolower_utf32, my_toupper_utf32, my_tosort_unicode, my_tolower_utf8mb4 and my_toupper_utf8mb4. Fixed these functions as well. --- include/m_ctype.h | 4 ++-- strings/ctype-ucs2.c | 14 +++++++------- strings/ctype-utf8.c | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/m_ctype.h b/include/m_ctype.h index 81096f60c78a8..87b1e529f657b 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -33,11 +33,11 @@ extern "C" { #define MY_CS_TO_UPPER_TABLE_SIZE 256 #define MY_CS_SORT_ORDER_TABLE_SIZE 256 #define MY_CS_TO_UNI_TABLE_SIZE 256 - #define CHARSET_DIR "charsets/" #define my_wc_t ulong +#define MY_CS_MAX_CHAR 0xFFFF #define MY_CS_REPLACEMENT_CHARACTER 0xFFFD /* diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c index 37fd1b5349ff8..f1beff82a409a 100644 --- a/strings/ctype-ucs2.c +++ b/strings/ctype-ucs2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -1099,7 +1099,7 @@ static inline void my_tolower_utf16(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256 && uni_plane[page]) + if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].tolower; } @@ -1108,7 +1108,7 @@ static inline void my_toupper_utf16(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256 && uni_plane[page]) + if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].toupper; } @@ -1117,7 +1117,7 @@ static inline void my_tosort_utf16(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256) + if (page < 256 && *wc <= MY_CS_MAX_CHAR) { if (uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].sort; @@ -1728,7 +1728,7 @@ static inline void my_tolower_utf32(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256 && uni_plane[page]) + if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].tolower; } @@ -1737,7 +1737,7 @@ static inline void my_toupper_utf32(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256 && uni_plane[page]) + if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].toupper; } @@ -1746,7 +1746,7 @@ static inline void my_tosort_utf32(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256) + if (page < 256 && *wc <= MY_CS_MAX_CHAR) { if (uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].sort; diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c index 52e05f17d61db..33e5703ffd9d4 100644 --- a/strings/ctype-utf8.c +++ b/strings/ctype-utf8.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -1941,7 +1941,7 @@ static inline void my_tosort_unicode(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256) + if (page < 256 && *wc <= MY_CS_MAX_CHAR) { if (uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].sort; @@ -5023,7 +5023,7 @@ static inline void my_tolower_utf8mb4(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256 && uni_plane[page]) + if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].tolower; } @@ -5032,7 +5032,7 @@ static inline void my_toupper_utf8mb4(MY_UNICASE_INFO **uni_plane, my_wc_t *wc) { int page= *wc >> 8; - if (page < 256 && uni_plane[page]) + if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].toupper; } From 09784e244bfbb16d3a1b54d1018995984ac8c121 Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Tue, 5 Jul 2016 17:08:37 +0530 Subject: [PATCH 004/295] Bug#23736787 - YUM UPDATE FAIL FROM 5.5.51(COMUNITY/COMMERCIAL) TO 5.6.32(COMUNITY/COMMERCIAL) Remove mysql_config from client sub-package --- packaging/rpm-oel/mysql.spec.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/rpm-oel/mysql.spec.in b/packaging/rpm-oel/mysql.spec.in index 29957d98ed0ed..409c325b6759f 100644 --- a/packaging/rpm-oel/mysql.spec.in +++ b/packaging/rpm-oel/mysql.spec.in @@ -835,8 +835,6 @@ fi %attr(755, root, root) %{_bindir}/mysqlimport %attr(755, root, root) %{_bindir}/mysqlshow %attr(755, root, root) %{_bindir}/mysqlslap -%attr(755, root, root) %{_bindir}/mysql_config -%attr(755, root, root) %{_bindir}/mysql_config-%{__isa_bits} %attr(644, root, root) %{_mandir}/man1/msql2mysql.1* %attr(644, root, root) %{_mandir}/man1/mysql.1* @@ -918,6 +916,9 @@ fi %endif %changelog +* Tue Jul 05 2016 Balasubramanian Kandasamy - 5.5.51-1 +- Remove mysql_config from client subpackage + * Tue Sep 29 2015 Balasubramanian Kandasamy - 5.5.47-1 - Added conflicts to mysql-connector-c-shared dependencies From 54e887b2fe24056b633957e8f9aff9d7f50b5089 Mon Sep 17 00:00:00 2001 From: Chaithra Gopalareddy Date: Tue, 19 Jul 2016 08:03:09 +0530 Subject: [PATCH 005/295] Bug#23280059: ITEM_ROW::ILLEGAL_METHOD_CALL(CONST CHAR*): ASSERTION `0' FAILED ON SELECT AREA Problem: Optimizer tries to get the points to calculate area without checking the return value of uint4korr for 0 "points". As a result server exits. Solution: Check the return value from uint4korr(). --- sql/spatial.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/spatial.cc b/sql/spatial.cc index 7aab91b5a5bc1..7643b3a649dd0 100644 --- a/sql/spatial.cc +++ b/sql/spatial.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -849,6 +849,8 @@ int Gis_polygon::area(double *ar, const char **end_of_data) const if (no_data(data, 4)) return 1; n_points= uint4korr(data); + if (n_points == 0) + return 1; if (not_enough_points(data, n_points)) return 1; get_point(&prev_x, &prev_y, data+4); From ac460e584d9706e02ce3dcb86bd994276672b267 Mon Sep 17 00:00:00 2001 From: Sreeharsha Ramanavarapu Date: Fri, 22 Jul 2016 07:33:43 +0530 Subject: [PATCH 006/295] Bug #23280699: MYSQLD GOT SIGNAL 11 IN IS_NULL ON SELECT FROM I_S Issue: ------ There is a difference in the field type created when the following DDLs are used: 1) CREATE TABLE t0 AS SELECT NULL; 2) CREATE TABLE t0 AS SELECT GREATEST(NULL,NULL); The first statement creates field of type Field_string and the second one creates a field of type Field_null. This creates a problem when the query mentioned in this bug is used. Since the null_ptr is calculated differently for Field_null. Solution: --------- When there is a function returning null in the select list as mentioned above, the field should be of type Field_string. This was fixed in 5.6+ as part of Bug#14021323. This is a backport to mysql-5.5. An incorrect comment in innodb_bug54044.test has been corrected in all versions. --- mysql-test/suite/innodb/r/innodb_bug54044.result | 14 ++++++++++---- mysql-test/suite/innodb/t/innodb_bug54044.test | 11 +++++------ sql/item.cc | 5 +---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_bug54044.result b/mysql-test/suite/innodb/r/innodb_bug54044.result index 350c500cb9b74..29b0127f20b92 100644 --- a/mysql-test/suite/innodb/r/innodb_bug54044.result +++ b/mysql-test/suite/innodb/r/innodb_bug54044.result @@ -6,7 +6,13 @@ table_54044 CREATE TEMPORARY TABLE `table_54044` ( `IF(NULL IS NOT NULL, NULL, NULL)` binary(0) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 DROP TABLE table_54044; -CREATE TABLE tmp ENGINE = INNODB AS SELECT COALESCE(NULL, NULL, NULL); -ERROR HY000: Can't create table 'test.tmp' (errno: -1) -CREATE TABLE tmp ENGINE = INNODB AS SELECT GREATEST(NULL, NULL); -ERROR HY000: Can't create table 'test.tmp' (errno: -1) +CREATE TABLE tmp ENGINE = INNODB +AS SELECT COALESCE(NULL, NULL, NULL), GREATEST(NULL, NULL), NULL; +SHOW CREATE TABLE tmp; +Table Create Table +tmp CREATE TABLE `tmp` ( + `COALESCE(NULL, NULL, NULL)` binary(0) DEFAULT NULL, + `GREATEST(NULL, NULL)` binary(0) DEFAULT NULL, + `NULL` binary(0) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +DROP TABLE tmp; diff --git a/mysql-test/suite/innodb/t/innodb_bug54044.test b/mysql-test/suite/innodb/t/innodb_bug54044.test index 0bbd7da00658b..cfc6f3c3f0a9b 100644 --- a/mysql-test/suite/innodb/t/innodb_bug54044.test +++ b/mysql-test/suite/innodb/t/innodb_bug54044.test @@ -10,10 +10,9 @@ CREATE TEMPORARY TABLE table_54044 ENGINE = INNODB SHOW CREATE TABLE table_54044; DROP TABLE table_54044; -# These 'create table' operations should fail because of -# using NULL datatype +# This 'create table' should pass since it uses a Field_string of size 0. ---error ER_CANT_CREATE_TABLE -CREATE TABLE tmp ENGINE = INNODB AS SELECT COALESCE(NULL, NULL, NULL); ---error ER_CANT_CREATE_TABLE -CREATE TABLE tmp ENGINE = INNODB AS SELECT GREATEST(NULL, NULL); +CREATE TABLE tmp ENGINE = INNODB + AS SELECT COALESCE(NULL, NULL, NULL), GREATEST(NULL, NULL), NULL; +SHOW CREATE TABLE tmp; +DROP TABLE tmp; diff --git a/sql/item.cc b/sql/item.cc index 1541314ec97ce..34157c33cf425 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5360,10 +5360,6 @@ Field *Item::tmp_table_field_from_field_type(TABLE *table, bool fixed_length) field= new Field_double((uchar*) 0, max_length, null_ptr, 0, Field::NONE, name, decimals, 0, unsigned_flag); break; - case MYSQL_TYPE_NULL: - field= new Field_null((uchar*) 0, max_length, Field::NONE, - name, &my_charset_bin); - break; case MYSQL_TYPE_INT24: field= new Field_medium((uchar*) 0, max_length, null_ptr, 0, Field::NONE, name, 0, unsigned_flag); @@ -5394,6 +5390,7 @@ Field *Item::tmp_table_field_from_field_type(TABLE *table, bool fixed_length) DBUG_ASSERT(0); /* If something goes awfully wrong, it's better to get a string than die */ case MYSQL_TYPE_STRING: + case MYSQL_TYPE_NULL: if (fixed_length && max_length < CONVERT_IF_BIGGER_TO_BLOB) { field= new Field_string(max_length, maybe_null, name, From fd31eea949e5cbfccb97715bec62f6c63ece6010 Mon Sep 17 00:00:00 2001 From: Arun Kuruvila Date: Fri, 22 Jul 2016 13:15:32 +0530 Subject: [PATCH 007/295] Bug #23295288: HANDLE_FATAL_SIGNAL (SIG=11) IN GET_SERVER_FROM_TABLE_TO_CACHE Description:- Server received SIG11 in the function, "get_server_from_table_to_cache()". Analysis:- Defining a server with a blank name is not handled properly. Fix:- Modified "get_server_from_table_to_cache()" to take care of blank server name. --- sql/sql_yacc.yy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 16ed61334c65d..b8ddc8bd49fc5 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2148,6 +2148,11 @@ server_def: ident_or_text OPTIONS_SYM '(' server_options_list ')' { + if ($2.length == 0) + { + my_error(ER_WRONG_VALUE, MYF(0), "server name", ""); + MYSQL_YYABORT; + } Lex->server_options.server_name= $2.str; Lex->server_options.server_name_length= $2.length; Lex->server_options.scheme= $6.str; From 2674cf91c5dad871f59704685e77e76ba8f462cd Mon Sep 17 00:00:00 2001 From: Thayumanavar S Date: Mon, 25 Jul 2016 06:43:16 +0100 Subject: [PATCH 008/295] BUG#23703568 - IS CLIENT LIBRARY SUPPOSED TO RETRY EINTR INDEFINITELY OR NOT Commit#ebd24626ca38e7fa1e3da2acdcf88540be70fabe obsoleted the THREAD and THREAD_SAFE_CLIENT preprocessor symbols. This is not removed in the sql/net_serv.cc thereby the code that retries on EINTR became dead code. Remove the THREAD_SAFE_CLIENT preprocessor directive form sql/net_serv.cc. Also check errno for EINTR only if there is an error in preceding read call. --- sql/net_serv.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/net_serv.cc b/sql/net_serv.cc index 8b44c7d443f1a..9c0c84bb2924b 100644 --- a/sql/net_serv.cc +++ b/sql/net_serv.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -676,13 +676,13 @@ net_real_write(NET *net,const uchar *packet, size_t len) my_progname); #endif /* EXTRA_DEBUG */ } -#if defined(THREAD_SAFE_CLIENT) && !defined(MYSQL_SERVER) +#ifndef MYSQL_SERVER if (vio_errno(net->vio) == SOCKET_EINTR) { DBUG_PRINT("warning",("Interrupted write. Retrying...")); continue; } -#endif /* defined(THREAD_SAFE_CLIENT) && !defined(MYSQL_SERVER) */ +#endif /* !defined(MYSQL_SERVER) */ net->error= 2; /* Close socket */ net->last_errno= (interrupted ? ER_NET_WRITE_INTERRUPTED : ER_NET_ERROR_ON_WRITE); @@ -887,8 +887,9 @@ my_real_read(NET *net, size_t *complen) my_progname,vio_errno(net->vio)); #endif /* EXTRA_DEBUG */ } -#if defined(THREAD_SAFE_CLIENT) && !defined(MYSQL_SERVER) - if (vio_errno(net->vio) == SOCKET_EINTR) +#ifndef MYSQL_SERVER + if (static_cast(length) < 0 && + vio_errno(net->vio) == SOCKET_EINTR) { DBUG_PRINT("warning",("Interrupted read. Retrying...")); continue; From a63a250d40be0254c6a5633329a6b0577eaee74c Mon Sep 17 00:00:00 2001 From: Neha Kumari Date: Mon, 25 Jul 2016 20:34:20 +0530 Subject: [PATCH 009/295] BUG#23509275 :DBUG_PRINT in THD::decide_logging_format prints incorrectly, access out-of-bound Problem: In debug builds, there is a chance that an out-of-bounds read is performed when tables are locked in LTM_PRELOCKED_UNDER_LOCK_TABLES mode. It can happen because the debug code uses enum values as index for an array of mode descriptions, but it only takes into consideration 3 out of 4 of the enum values. Fix: This patch fixes it by implementing a getter for the enum which returns a string representation of the enum, effectively removing the out-of-bounds read. Moreover, it also fixes the lock mode descriptions that would be print out in debug builds. --- sql/sql_class.cc | 32 ++++++++++++++++++++++---------- sql/sql_class.h | 11 +++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 4711009d7cd83..0696021cfc09b 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4246,6 +4246,25 @@ has_write_table_auto_increment_not_first_in_pk(TABLE_LIST *tables) return 0; } +#ifndef DBUG_OFF +const char * get_locked_tables_mode_name(enum_locked_tables_mode locked_tables_mode) +{ + switch (locked_tables_mode) + { + case LTM_NONE: + return "LTM_NONE"; + case LTM_LOCK_TABLES: + return "LTM_LOCK_TABLES"; + case LTM_PRELOCKED: + return "LTM_PRELOCKED"; + case LTM_PRELOCKED_UNDER_LOCK_TABLES: + return "LTM_PRELOCKED_UNDER_LOCK_TABLES"; + default: + return "Unknown table lock mode"; + } +} +#endif + /** Decide on logging format to use for the statement and issue errors or warnings as needed. The decision depends on the following @@ -4397,15 +4416,8 @@ int THD::decide_logging_format(TABLE_LIST *tables) TABLE* prev_access_table= NULL; #ifndef DBUG_OFF - { - static const char *prelocked_mode_name[] = { - "NON_PRELOCKED", - "PRELOCKED", - "PRELOCKED_UNDER_LOCK_TABLES", - }; - DBUG_PRINT("debug", ("prelocked_mode: %s", - prelocked_mode_name[locked_tables_mode])); - } + DBUG_PRINT("debug", ("prelocked_mode: %s", + get_locked_tables_mode_name(locked_tables_mode))); #endif if (variables.binlog_format != BINLOG_FORMAT_ROW && tables) diff --git a/sql/sql_class.h b/sql/sql_class.h index 0df8c70e18443..dcc7458ee5043 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -963,6 +963,8 @@ typedef I_List Item_change_list; /** Type of locked tables mode. See comment for THD::locked_tables_mode for complete description. + While adding new enum values add them to the getter method for this enum + declared below and defined in sql_class.cc as well. */ enum enum_locked_tables_mode @@ -973,6 +975,15 @@ enum enum_locked_tables_mode LTM_PRELOCKED_UNDER_LOCK_TABLES }; +#ifndef DBUG_OFF +/** + Getter for the enum enum_locked_tables_mode + @param locked_tables_mode enum for types of locked tables mode + + @return The string represantation of that enum value +*/ +const char * get_locked_tables_mode_name(enum_locked_tables_mode locked_tables_mode); +#endif /** Class that holds information about tables which were opened and locked From 8bb95e9a974b5eee44764698fafc56279f8b7691 Mon Sep 17 00:00:00 2001 From: Sreeharsha Ramanavarapu Date: Wed, 3 Aug 2016 09:58:36 +0530 Subject: [PATCH 010/295] Bug #24380263: INCORRECT BEHAVIOR WITH PARAMETER AND DERIVED TABLE IN JOIN ISSUE: ------ This problem occurs under the following conditions: 1) A parameter is used in the select-list of a derived table. 2) The derived table is part of a JOIN. SOLUTION: --------- When a derived table is materialized, a temporary table is created. This temporary table creates a field each for the items in the select-list of the derived table. This set of fields is later used to setup the join. Currently no field is created in the temporary table if a parameter is used in the select-list. Create a field for the parameter. By default Item_param's result type in a prepared statement is set to STRING_RESULT. This can change during the execute phase depending on the user variable. But since the execute phase creates its own temporary table, it will be handled separately. This is a backport of the fix for BUG#22392374. --- sql/sql_select.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b5ecebdadc861..fb705e9ba6ad2 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -10240,6 +10240,7 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type, case Item::REF_ITEM: case Item::NULL_ITEM: case Item::VARBIN_ITEM: + case Item::PARAM_ITEM: if (make_copy_field) { DBUG_ASSERT(((Item_result_field*)item)->result_field); From 322afb2c0a6bc7c8ffb365c61b90392bfe4b4f31 Mon Sep 17 00:00:00 2001 From: Kailasnath Nagarkar Date: Wed, 3 Aug 2016 12:54:58 +0530 Subject: [PATCH 011/295] Bug #19984392 : MEDIUMINT: STACK BUFFER OVERFLOW IN PROTOCOL_TEXT::STORE_LONG ISSUE: Queries with mediumint as column when operated with long long type of data results in buffer overflow in store_long function. The merging rule specified for (MYSQL_TYPE_LONGLONG MYSQL_TYPE_INT24) is MYSQL_TYPE_LONG. Due to this store_long function was getting called which resulted in buffer overflow. SOLUTION: The correct merging rule for (MYSQL_TYPE_LONGLONG, MYSQL_TYPE_INT24) should be MYSQL_TYPE_LONGLONG. So, instead of function store_long, function store_longlong is called which correctly handles the type MYSQL_TYPE_LONGLONG. External Bug #23645238 is a duplicate of this issue. --- sql/field.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/field.cc b/sql/field.cc index abe856a8292c6..d9889f0fb489b 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -347,7 +347,7 @@ static enum_field_types field_types_merge_rules [FIELDTYPE_NUM][FIELDTYPE_NUM]= //MYSQL_TYPE_NULL MYSQL_TYPE_TIMESTAMP MYSQL_TYPE_LONGLONG, MYSQL_TYPE_VARCHAR, //MYSQL_TYPE_LONGLONG MYSQL_TYPE_INT24 - MYSQL_TYPE_LONGLONG, MYSQL_TYPE_LONG, + MYSQL_TYPE_LONGLONG, MYSQL_TYPE_LONGLONG, //MYSQL_TYPE_DATE MYSQL_TYPE_TIME MYSQL_TYPE_VARCHAR, MYSQL_TYPE_VARCHAR, //MYSQL_TYPE_DATETIME MYSQL_TYPE_YEAR From 194776ce00f6fea37551ea25584798f78b0ad24b Mon Sep 17 00:00:00 2001 From: Kailasnath Nagarkar Date: Thu, 4 Aug 2016 12:49:50 +0530 Subject: [PATCH 012/295] Bug #19984392 : MEDIUMINT: STACK BUFFER OVERFLOW IN PROTOCOL_TEXT::STORE_LONG Reverting the patch due to some issues. --- sql/field.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/field.cc b/sql/field.cc index d9889f0fb489b..3ca072e777164 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -347,7 +347,7 @@ static enum_field_types field_types_merge_rules [FIELDTYPE_NUM][FIELDTYPE_NUM]= //MYSQL_TYPE_NULL MYSQL_TYPE_TIMESTAMP MYSQL_TYPE_LONGLONG, MYSQL_TYPE_VARCHAR, //MYSQL_TYPE_LONGLONG MYSQL_TYPE_INT24 - MYSQL_TYPE_LONGLONG, MYSQL_TYPE_LONGLONG, + MYSQL_TYPE_LONGLONG, MYSQL_TYPE_LONG, //MYSQL_TYPE_DATE MYSQL_TYPE_TIME MYSQL_TYPE_VARCHAR, MYSQL_TYPE_VARCHAR, //MYSQL_TYPE_DATETIME MYSQL_TYPE_YEAR From 22eec68941f3acbd9033e7fb33d10c63e6b388da Mon Sep 17 00:00:00 2001 From: Neha Kumari Date: Fri, 5 Aug 2016 12:17:11 +0530 Subject: [PATCH 013/295] Bug#23540182:MYSQLBINLOG DOES NOT FREE THE EXISTING CONNECTION BEFORE OPENING NEW REMOTE ONE It happens when you are trying to read two or more log files from a remote server using mysqlbinlog utility. The reason for this is no matching mysql_close() that concludes the life time of 'mysql' struct describing connection to the server. This happens when mysqlbinlog is invoked with connecting to the server and requesting more than one binlog file. In such case dump_remote_log_entries() keeps calling safe_connect() per eachfile, never caring to invoke mysql_close(). Only the final safe_connect()'s allocation effect are cleaned by the base code. That is with 2 files there's one 'mysql' connection descriptor struct uncleaned/deallocated. We are backporting the bug 21255763 (pushed in mysql-trunk) in the earlier version of MySQL starting from 5.5 to 5.7. which was pushed in mysql-trunk. Fix: Invoke mysql_close() just before mysql_init() in safe_connect() defined in mysqlbinlog.cc. That makes possibly previously used 'mysql' be reclaimed prior a new one is allocated. --- client/mysqlbinlog.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 73a801c4b21e0..955d9e3fb3c1e 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1444,6 +1444,12 @@ static int parse_args(int *argc, char*** argv) */ static Exit_status safe_connect() { + /* + A possible old connection's resources are reclaimed now + at new connect attempt. The final safe_connect resources + are mysql_closed at the end of program, explicitly. + */ + mysql_close(mysql); mysql= mysql_init(NULL); if (!mysql) From 0c6eac64c7d63d1fdf6fa78724b817f03e5d7454 Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Mon, 8 Aug 2016 15:15:17 +0530 Subject: [PATCH 014/295] Raise version number after cloning 5.5.52 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index acabf9b42d0ca..d44c8b2800612 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=52 +MYSQL_VERSION_PATCH=53 MYSQL_VERSION_EXTRA= From 737964dcd12e61ae7fb4b47505158e2fecce9f2b Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 10 Aug 2016 11:24:18 -0400 Subject: [PATCH 015/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index db9d497c141a9..acabf9b42d0ca 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=51 +MYSQL_VERSION_PATCH=52 MYSQL_VERSION_EXTRA= From 09cb64682bda8648b0cdad606b169b4d592f839d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 11 Aug 2016 19:35:53 +0000 Subject: [PATCH 016/295] Windows : fix search for WiX root directory when using 64bit cmake "C:\Program Files (x86)" directory needs to be checked as well in this case. --- win/packaging/CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/win/packaging/CMakeLists.txt b/win/packaging/CMakeLists.txt index 0535a486d57ba..1682bae6986cb 100644 --- a/win/packaging/CMakeLists.txt +++ b/win/packaging/CMakeLists.txt @@ -24,10 +24,13 @@ ENDIF() SET(MANUFACTURER "MariaDB Corporation Ab") -FIND_PATH(WIX_DIR heat.exe - "$ENV{ProgramFiles}/WiX Toolset v3.9/bin" - "$ENV{ProgramFiles}/WiX Toolset v3.10/bin" -) +SET(WIX_BIN_PATHS) +FOREACH(WIX_VER 3.9 3.10 3.11) + LIST(APPEND WIX_BIN_PATHS "$ENV{ProgramFiles}/WiX Toolset v${WIX_VER}/bin") + LIST(APPEND WIX_BIN_PATHS "$ENV{ProgramFiles} (x86)/WiX Toolset v${WIX_VER}/bin") +ENDFOREACH() + +FIND_PATH(WIX_DIR heat.exe ${WIX_BIN_PATHS}) SET(CPACK_WIX_PACKAGE_BASE_NAME "MariaDB") IF(CMAKE_SIZEOF_VOID_P EQUAL 4) SET(CPACK_WIX_UPGRADE_CODE "49EB7A6A-1CEF-4A1E-9E89-B9A4993963E3") From 723488bba162109f241bc764b6e33c6f3d8b39d6 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Thu, 4 Aug 2016 15:43:52 +0400 Subject: [PATCH 017/295] MDEV-10424 - Assertion `ticket == __null' failed in MDL_request::set_type Reexecution of prepared "ANALYZE TABLE merge_table, table" may miss to reinitialize "table" for subsequent execution and trigger assertion failure. This happens because MERGE engine may adjust table->next_global chain, which gets cleared by close_thread_tables()/ha_myisammrg::detach_children() later. Since reinitilization iterates next_global chain, it won't see tables following merge table. Fixed by appending saved next_global chain after merge children. --- mysql-test/r/merge.result | 17 +++++++++++++++++ mysql-test/t/merge.test | 13 +++++++++++++ sql/sql_admin.cc | 14 +++++++++++++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/merge.result b/mysql-test/r/merge.result index 41ee148cee326..66ba6cea70e3c 100644 --- a/mysql-test/r/merge.result +++ b/mysql-test/r/merge.result @@ -3832,4 +3832,21 @@ test.m1 repair error Corrupt # Clean-up. drop tables m1, t1, t4; drop view t3; +# +# MDEV-10424 - Assertion `ticket == __null' failed in +# MDL_request::set_type +# +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +CREATE TABLE tmerge (f1 INT) ENGINE=MERGE UNION=(t1); +PREPARE stmt FROM "ANALYZE TABLE tmerge, t1"; +EXECUTE stmt; +Table Op Msg_type Msg_text +test.tmerge analyze note The storage engine for the table doesn't support analyze +test.t1 analyze status Table is already up to date +EXECUTE stmt; +Table Op Msg_type Msg_text +test.tmerge analyze note The storage engine for the table doesn't support analyze +test.t1 analyze status Table is already up to date +DEALLOCATE PREPARE stmt; +DROP TABLE t1, tmerge; End of 5.5 tests diff --git a/mysql-test/t/merge.test b/mysql-test/t/merge.test index 6573c2b09c009..9d0ddd01752ce 100644 --- a/mysql-test/t/merge.test +++ b/mysql-test/t/merge.test @@ -2880,6 +2880,19 @@ drop tables m1, t1, t4; drop view t3; +--echo # +--echo # MDEV-10424 - Assertion `ticket == __null' failed in +--echo # MDL_request::set_type +--echo # +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +CREATE TABLE tmerge (f1 INT) ENGINE=MERGE UNION=(t1); +PREPARE stmt FROM "ANALYZE TABLE tmerge, t1"; +EXECUTE stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +DROP TABLE t1, tmerge; + + --echo End of 5.5 tests --disable_result_log diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 12a59fa6ee80c..55effcd70027d 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -441,7 +441,19 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, } thd->prepare_derived_at_open= FALSE; - table->next_global= save_next_global; + /* + MERGE engine may adjust table->next_global chain, thus we have to + append save_next_global after merge children. + */ + if (save_next_global) + { + TABLE_LIST *table_list_iterator= table; + while (table_list_iterator->next_global) + table_list_iterator= table_list_iterator->next_global; + table_list_iterator->next_global= save_next_global; + save_next_global->prev_global= &table_list_iterator->next_global; + } + table->next_local= save_next_local; thd->open_options&= ~extra_open_options; From a92a8cc817649df80fd84b6a466da345772660fb Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 19 Aug 2016 17:11:20 +0000 Subject: [PATCH 018/295] Windows packaging : use /d switch to sign MSI, to prevent installer showing randomly generated name in UAC prompt --- win/packaging/create_msi.cmake.in | 1 + 1 file changed, 1 insertion(+) diff --git a/win/packaging/create_msi.cmake.in b/win/packaging/create_msi.cmake.in index a8ca35906e8da..fceea04167646 100644 --- a/win/packaging/create_msi.cmake.in +++ b/win/packaging/create_msi.cmake.in @@ -430,6 +430,7 @@ EXECUTE_PROCESS( IF(SIGNCODE) EXECUTE_PROCESS( COMMAND ${SIGNTOOL_EXECUTABLE} sign ${SIGNTOOL_PARAMETERS} + /d ${CPACK_PACKAGE_FILE_NAME}.msi ${CPACK_PACKAGE_FILE_NAME}.msi ) ENDIF() From ee97274ca7d9ea8d8f00e40476a039c35399ee15 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 25 Aug 2016 09:50:04 +0300 Subject: [PATCH 019/295] DEV-10595 MariaDB daemon leaks memory with specific query The issue was that in some extreme cases when doing GROUP BY, buffers for temporary blobs where not properly cleared. --- mysql-test/r/group_min_max_innodb.result | 16 +++++++++++++ mysql-test/t/group_min_max_innodb.test | 13 +++++++++++ sql/sql_class.h | 5 ++++ sql/sql_select.cc | 29 +++++++++++++++++++----- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/group_min_max_innodb.result b/mysql-test/r/group_min_max_innodb.result index c4d2fb88784dc..1e3ee793a7f97 100644 --- a/mysql-test/r/group_min_max_innodb.result +++ b/mysql-test/r/group_min_max_innodb.result @@ -286,3 +286,19 @@ F 28 28 F 29 29 F 30 30 DROP TABLE t0,t1,t2; +# +# MDEV-MariaDB daemon leaks memory with specific query +# +CREATE TABLE t1 (`voter_id` int(11) unsigned NOT NULL, +`language_id` int(11) unsigned NOT NULL DEFAULT '1' +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE t2 (`voter_id` int(10) unsigned NOT NULL DEFAULT '0', +`serialized_c` mediumblob) ENGINE=InnoDB DEFAULT CHARSET=utf8; +insert into t2 values (1,repeat("a",1000)),(2,repeat("a",1000)),(3,repeat("b",1000)),(4,repeat("c",1000)),(4,repeat("b",1000)); +SELECT GROUP_CONCAT(t1.language_id SEPARATOR ',') AS `translation_resources`, `d`.`serialized_c` FROM t2 AS `d` LEFT JOIN t1 ON `d`.`voter_id` = t1.`voter_id` GROUP BY `d`.`voter_id` ORDER BY 10-d.voter_id+RAND()*0; +translation_resources serialized_c +NULL cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +NULL bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +NULL aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +NULL aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +drop table t1,t2; diff --git a/mysql-test/t/group_min_max_innodb.test b/mysql-test/t/group_min_max_innodb.test index 6967f84714729..91e0bd3279fb4 100644 --- a/mysql-test/t/group_min_max_innodb.test +++ b/mysql-test/t/group_min_max_innodb.test @@ -230,3 +230,16 @@ eval EXPLAIN $query; eval $query; DROP TABLE t0,t1,t2; + +--echo # +--echo # MDEV-MariaDB daemon leaks memory with specific query +--echo # + +CREATE TABLE t1 (`voter_id` int(11) unsigned NOT NULL, + `language_id` int(11) unsigned NOT NULL DEFAULT '1' +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE t2 (`voter_id` int(10) unsigned NOT NULL DEFAULT '0', + `serialized_c` mediumblob) ENGINE=InnoDB DEFAULT CHARSET=utf8; +insert into t2 values (1,repeat("a",1000)),(2,repeat("a",1000)),(3,repeat("b",1000)),(4,repeat("c",1000)),(4,repeat("b",1000)); +SELECT GROUP_CONCAT(t1.language_id SEPARATOR ',') AS `translation_resources`, `d`.`serialized_c` FROM t2 AS `d` LEFT JOIN t1 ON `d`.`voter_id` = t1.`voter_id` GROUP BY `d`.`voter_id` ORDER BY 10-d.voter_id+RAND()*0; +drop table t1,t2; diff --git a/sql/sql_class.h b/sql/sql_class.h index d24dad7c2ca7b..da83382d5e93b 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3607,6 +3607,11 @@ class TMP_TABLE_PARAM :public Sql_alloc save_copy_field_end= copy_field_end= NULL; } } + void free_copy_field_data() + { + for (Copy_field *ptr= copy_field ; ptr != copy_field_end ; ptr++) + ptr->tmp.free(); + } }; class select_union :public select_result_interceptor diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 613cbb2e08699..121805dd0e2ea 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -8275,9 +8275,26 @@ JOIN::make_simple_join(JOIN *parent, TABLE *temp_table) We need to destruct the copy_field (allocated in create_tmp_table()) before setting it to 0 if the join is not "reusable". */ - if (!tmp_join || tmp_join != this) - tmp_table_param.cleanup(); - tmp_table_param.copy_field= tmp_table_param.copy_field_end=0; + if (!tmp_join || tmp_join != this) + tmp_table_param.cleanup(); + else + { + /* + Free data buffered in copy_fields, but keep data pointed by copy_field + around for next iteration (possibly stored in save_copy_fields). + + It would be logically simpler to not clear copy_field + below, but as we have loops that runs over copy_field to + copy_field_end that should not be done anymore, it's simpler to + just clear the pointers. + + Another option would be to just clear copy_field_end and not run + the loops if this is not set or to have tmp_table_param.cleanup() + to run cleanup on save_copy_field if copy_field is not set. + */ + tmp_table_param.free_copy_field_data(); + tmp_table_param.copy_field= tmp_table_param.copy_field_end=0; + } first_record= sort_and_group=0; send_records= (ha_rows) 0; @@ -10866,7 +10883,7 @@ void JOIN::join_free() /** Free resources of given join. - @param fill true if we should free all resources, call with full==1 + @param full true if we should free all resources, call with full==1 should be last, before it this function can be called with full==0 @@ -10982,7 +10999,7 @@ void JOIN::cleanup(bool full) /* If we have tmp_join and 'this' JOIN is not tmp_join and tmp_table_param.copy_field's of them are equal then we have to remove - pointer to tmp_table_param.copy_field from tmp_join, because it qill + pointer to tmp_table_param.copy_field from tmp_join, because it will be removed in tmp_table_param.cleanup(). */ if (tmp_join && @@ -21397,7 +21414,7 @@ setup_copy_fields(THD *thd, TMP_TABLE_PARAM *param, err: if (copy) delete [] param->copy_field; // This is never 0 - param->copy_field=0; + param->copy_field= 0; err2: DBUG_RETURN(TRUE); } From 684a165f28b3718160a3e4c5ebd18a465d85e97c Mon Sep 17 00:00:00 2001 From: Terje Rosten Date: Fri, 12 Aug 2016 12:38:20 +0200 Subject: [PATCH 020/295] Bug#24464380 PRIVILEGE ESCALATION USING MYSQLD_SAFE Argument to malloc-lib must be included in restricted list of directories, symlink guards added, and mysqld and mysqld-version options restricted to command line only. Don't redirect errors to stderr. --- packaging/rpm-oel/mysql.init | 2 +- packaging/rpm-sles/mysql.init | 2 +- scripts/mysqld_safe.sh | 79 +++++++++++++++++++++-------------- support-files/mysql.server.sh | 2 +- 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/packaging/rpm-oel/mysql.init b/packaging/rpm-oel/mysql.init index 262d0582f68b4..aaea498d15339 100644 --- a/packaging/rpm-oel/mysql.init +++ b/packaging/rpm-oel/mysql.init @@ -102,7 +102,7 @@ start(){ # alarms, per bug #547485 $exec --datadir="$datadir" --socket="$socketfile" \ --pid-file="$mypidfile" \ - --basedir=/usr --user=mysql >/dev/null 2>&1 & + --basedir=/usr --user=mysql >/dev/null & safe_pid=$! # Spin for a maximum of N seconds waiting for the server to come up; # exit the loop immediately if mysqld_safe process disappears. diff --git a/packaging/rpm-sles/mysql.init b/packaging/rpm-sles/mysql.init index 50ca4c9033c75..dda0bebba5658 100644 --- a/packaging/rpm-sles/mysql.init +++ b/packaging/rpm-sles/mysql.init @@ -137,7 +137,7 @@ start () { rc_failed 6 ; rc_status -v ; rc_exit fi - $PROG --basedir=/usr --datadir="$datadir" --pid-file="$pidfile" >/dev/null 2>&1 & + $PROG --basedir=/usr --datadir="$datadir" --pid-file="$pidfile" >/dev/null & if pinger $! ; then echo -n "Starting service MySQL:" touch $lockfile diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index f705953059e0b..11b692ec92874 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -208,8 +208,17 @@ parse_arguments() { --core-file-size=*) core_file_size="$val" ;; --ledir=*) ledir="$val" ;; --malloc-lib=*) set_malloc_lib "$val" ;; - --mysqld=*) MYSQLD="$val" ;; + --mysqld=*) + if [ -z "$pick_args" ]; then + log_error "--mysqld option can only be used as command line option, found in config file" + exit 1 + fi + MYSQLD="$val" ;; --mysqld-version=*) + if [ -z "$pick_args" ]; then + log_error "--mysqld-version option can only be used as command line option, found in config file" + exit 1 + fi if test -n "$val" then MYSQLD="mysqld-$val" @@ -297,38 +306,22 @@ mysqld_ld_preload_text() { echo "$text" } - -mysql_config= -get_mysql_config() { - if [ -z "$mysql_config" ]; then - mysql_config=`echo "$0" | sed 's,/[^/][^/]*$,/mysql_config,'` - if [ ! -x "$mysql_config" ]; then - log_error "Can not run mysql_config $@ from '$mysql_config'" - exit 1 - fi - fi - - "$mysql_config" "$@" -} - - # set_malloc_lib LIB # - If LIB is empty, do nothing and return -# - If LIB is 'tcmalloc', look for tcmalloc shared library in /usr/lib -# then pkglibdir. tcmalloc is part of the Google perftools project. +# - If LIB is 'tcmalloc', look for tcmalloc shared library in $malloc_dirs. +# tcmalloc is part of the Google perftools project. # - If LIB is an absolute path, assume it is a malloc shared library # # Put LIB in mysqld_ld_preload, which will be added to LD_PRELOAD when # running mysqld. See ld.so for details. set_malloc_lib() { + # This list is kept intentionally simple. + malloc_dirs="/usr/lib /usr/lib64 /usr/lib/i386-linux-gnu /usr/lib/x86_64-linux-gnu" malloc_lib="$1" if [ "$malloc_lib" = tcmalloc ]; then - pkglibdir=`get_mysql_config --variable=pkglibdir` malloc_lib= - # This list is kept intentionally simple. Simply set --malloc-lib - # to a full path if another location is desired. - for libdir in /usr/lib "$pkglibdir" "$pkglibdir/mysql"; do + for libdir in $(echo $malloc_dirs); do for flavor in _minimal '' _and_profiler _debug; do tmp="$libdir/libtcmalloc$flavor.so" #log_notice "DEBUG: Checking for malloc lib '$tmp'" @@ -339,7 +332,7 @@ set_malloc_lib() { done if [ -z "$malloc_lib" ]; then - log_error "no shared library for --malloc-lib=tcmalloc found in /usr/lib or $pkglibdir" + log_error "no shared library for --malloc-lib=tcmalloc found in $malloc_dirs" exit 1 fi fi @@ -350,9 +343,21 @@ set_malloc_lib() { case "$malloc_lib" in /*) if [ ! -r "$malloc_lib" ]; then - log_error "--malloc-lib '$malloc_lib' can not be read and will not be used" + log_error "--malloc-lib can not be read and will not be used" exit 1 fi + + # Restrict to a the list in $malloc_dirs above + case "$(dirname "$malloc_lib")" in + /usr/lib) ;; + /usr/lib64) ;; + /usr/lib/i386-linux-gnu) ;; + /usr/lib/x86_64-linux-gnu) ;; + *) + log_error "--malloc-lib must be located in one of the directories: $malloc_dirs" + exit 1 + ;; + esac ;; *) log_error "--malloc-lib must be an absolute path or 'tcmalloc'; " \ @@ -569,7 +574,7 @@ then log_notice "Logging to '$err_log'." logging=file - if [ ! -f "$err_log" ]; then # if error log already exists, + if [ ! -f "$err_log" -a ! -h "$err_log" ]; then # if error log already exists, touch "$err_log" # we just append. otherwise, chmod "$fmode" "$err_log" # fix the permissions here! fi @@ -594,7 +599,7 @@ then USER_OPTION="--user=$user" fi # Change the err log to the right user, if it is in use - if [ $want_syslog -eq 0 ]; then + if [ $want_syslog -eq 0 -a ! -h "$err_log" ]; then touch "$err_log" chown $user "$err_log" fi @@ -614,9 +619,11 @@ safe_mysql_unix_port=${mysql_unix_port:-${MYSQL_UNIX_PORT:-@MYSQL_UNIX_ADDR@}} mysql_unix_port_dir=`dirname $safe_mysql_unix_port` if [ ! -d $mysql_unix_port_dir ] then - mkdir $mysql_unix_port_dir - chown $user $mysql_unix_port_dir - chmod 755 $mysql_unix_port_dir + if [ ! -h $mysql_unix_port_dir ]; then + mkdir $mysql_unix_port_dir + chown $user $mysql_unix_port_dir + chmod 755 $mysql_unix_port_dir + fi fi # If the user doesn't specify a binary, we assume name "mysqld" @@ -728,7 +735,9 @@ then exit 1 fi fi - rm -f "$pid_file" + if [ ! -h "$pid_file" ]; then + rm -f "$pid_file" + fi if test -f "$pid_file" then log_error "Fatal error: Can't remove the pid file: @@ -779,13 +788,19 @@ have_sleep=1 while true do - rm -f $safe_mysql_unix_port "$pid_file" # Some extra safety + # Some extra safety + if [ ! -h "$safe_mysql_unix_port" ]; then + rm -f "$safe_mysql_unix_port" + fi + if [ ! -h "$pid_file" ]; then + rm -f "$pid_file" + fi start_time=`date +%M%S` eval_log_error "$cmd" - if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then + if [ $want_syslog -eq 0 -a ! -f "$err_log" -a ! -h "$err_log" ]; then touch "$err_log" # hypothetical: log was renamed but not chown $user "$err_log" # flushed yet. we'd recreate it with chmod "$fmode" "$err_log" # wrong owner next time we log, so set diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index 7487d5acc0f9d..909d33f877085 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -280,7 +280,7 @@ case "$mode" in then # Give extra arguments to mysqld with the my.cnf file. This script # may be overwritten at next upgrade. - $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null 2>&1 & + $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null & wait_for_pid created "$!" "$mysqld_pid_file_path"; return_value=$? # Make lock for RedHat / SuSE From 4e5473862e6852b0f3802b0cd0c6fa10b5253291 Mon Sep 17 00:00:00 2001 From: Jon Olav Hauglid Date: Tue, 16 Aug 2016 15:35:19 +0200 Subject: [PATCH 021/295] Bug#24388746: PRIVILEGE ESCALATION AND RACE CONDITION USING CREATE TABLE During REPAIR TABLE of a MyISAM table, a temporary data file (.TMD) is created. When repair finishes, this file is renamed to the original .MYD file. The problem was that during this rename, we copied the stats from the old file to the new file with chmod/chown. If a user managed to replace the temporary file before chmod/chown was executed, it was possible to get an arbitrary file with the privileges of the mysql user. This patch fixes the problem by not copying stats from the old file to the new file. This is not needed as the new file was created with the correct stats. This fix only changes server behavior - external utilities such as myisamchk still does chmod/chown. No test case provided since the problem involves synchronization with file system operations. --- include/my_sys.h | 3 ++- include/myisam.h | 11 +++++----- mysys/my_redel.c | 12 ++++++++--- storage/myisam/ha_myisam.cc | 26 ++++++++++++++++++----- storage/myisam/mi_check.c | 41 ++++++++++++++++++++++++++----------- storage/myisam/myisamchk.c | 16 +++++++++------ 6 files changed, 77 insertions(+), 32 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index b1b8bf15be328..472c2ba5ca069 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -83,6 +83,7 @@ typedef struct my_aio_result { #define MY_RESOLVE_LINK 128 /* my_realpath(); Only resolve links */ #define MY_HOLD_ORIGINAL_MODES 128 /* my_copy() holds to file modes */ #define MY_REDEL_MAKE_BACKUP 256 +#define MY_REDEL_NO_COPY_STAT 512 /* my_redel() doesn't call my_copystat() */ #define MY_SEEK_NOT_DONE 32 /* my_lock may have to do a seek */ #define MY_DONT_WAIT 64 /* my_lock() don't wait if can't lock */ #define MY_ZEROFILL 32 /* my_malloc(), fill array with zero */ diff --git a/include/myisam.h b/include/myisam.h index 85d37a81bc699..a9fcd7e436975 100644 --- a/include/myisam.h +++ b/include/myisam.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -426,12 +426,13 @@ int chk_size(MI_CHECK *param, MI_INFO *info); int chk_key(MI_CHECK *param, MI_INFO *info); int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend); int mi_repair(MI_CHECK *param, register MI_INFO *info, - char * name, int rep_quick); -int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name); + char * name, int rep_quick, my_bool no_copy_stat); +int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name, + my_bool no_copy_stat); int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, - const char * name, int rep_quick); + const char * name, int rep_quick, my_bool no_copy_stat); int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, - const char * name, int rep_quick); + const char * name, int rep_quick, my_bool no_copy_stat); int change_to_newfile(const char * filename, const char * old_ext, const char * new_ext, myf myflags); int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type, diff --git a/mysys/my_redel.c b/mysys/my_redel.c index a47df8265c808..25391cd4e8fa0 100644 --- a/mysys/my_redel.c +++ b/mysys/my_redel.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -35,6 +35,9 @@ struct utimbuf { if MY_REDEL_MAKE_COPY is given, then the orginal file is renamed to org_name-'current_time'.BAK + + if MY_REDEL_NO_COPY_STAT is given, stats are not copied + from org_name to tmp_name. */ #define REDEL_EXT ".BAK" @@ -46,8 +49,11 @@ int my_redel(const char *org_name, const char *tmp_name, myf MyFlags) DBUG_PRINT("my",("org_name: '%s' tmp_name: '%s' MyFlags: %d", org_name,tmp_name,MyFlags)); - if (my_copystat(org_name,tmp_name,MyFlags) < 0) - goto end; + if (!(MyFlags & MY_REDEL_NO_COPY_STAT)) + { + if (my_copystat(org_name,tmp_name,MyFlags) < 0) + goto end; + } if (MyFlags & MY_REDEL_MAKE_BACKUP) { char name_buff[FN_REFLEN+20]; diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 602a0ae6cc11f..21cbef32188ef 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1091,24 +1091,36 @@ int ha_myisam::repair(THD *thd, MI_CHECK ¶m, bool do_optimize) /* TODO: respect myisam_repair_threads variable */ my_snprintf(buf, 40, "Repair with %d threads", my_count_bits(key_map)); thd_proc_info(thd, buf); + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ error = mi_repair_parallel(¶m, file, fixed_name, - param.testflag & T_QUICK); + param.testflag & T_QUICK, TRUE); thd_proc_info(thd, "Repair done"); // to reset proc_info, as // it was pointing to local buffer } else { thd_proc_info(thd, "Repair by sorting"); + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ error = mi_repair_by_sort(¶m, file, fixed_name, - param.testflag & T_QUICK); + param.testflag & T_QUICK, TRUE); } } else { thd_proc_info(thd, "Repair with keycache"); param.testflag &= ~T_REP_BY_SORT; + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ error= mi_repair(¶m, file, fixed_name, - param.testflag & T_QUICK); + param.testflag & T_QUICK, TRUE); } #ifdef HAVE_MMAP if (remap) @@ -1124,7 +1136,11 @@ int ha_myisam::repair(THD *thd, MI_CHECK ¶m, bool do_optimize) { optimize_done=1; thd_proc_info(thd, "Sorting index"); - error=mi_sort_index(¶m,file,fixed_name); + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ + error=mi_sort_index(¶m,file,fixed_name, TRUE); } if (!statistics_done && (local_testflag & T_STATISTICS)) { diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index ba1f975549a1c..fe0d4c9c30b47 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1512,7 +1512,7 @@ static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force) /* Save new datafile-name in temp_filename */ int mi_repair(MI_CHECK *param, register MI_INFO *info, - char * name, int rep_quick) + char * name, int rep_quick, my_bool no_copy_stat) { int error,got_error; ha_rows start_records,new_header_length; @@ -1726,6 +1726,11 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, /* Replace the actual file with the temporary file */ if (new_file >= 0) { + myf flags= 0; + if (param->testflag & T_BACKUP_DATA) + flags |= MY_REDEL_MAKE_BACKUP; + if (no_copy_stat) + flags |= MY_REDEL_NO_COPY_STAT; mysql_file_close(new_file, MYF(0)); info->dfile=new_file= -1; /* @@ -1744,8 +1749,7 @@ int mi_repair(MI_CHECK *param, register MI_INFO *info, info->s->file_map= NULL; } if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT, - (param->testflag & T_BACKUP_DATA ? - MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + flags) || mi_open_datafile(info,share,name,-1)) got_error=1; @@ -1933,7 +1937,8 @@ int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file) /* Sort index for more efficent reads */ -int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name) +int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name, + my_bool no_copy_stat) { reg2 uint key; reg1 MI_KEYDEF *keyinfo; @@ -2004,7 +2009,7 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name) share->kfile = -1; (void) mysql_file_close(new_file, MYF(MY_WME)); if (change_to_newfile(share->index_file_name, MI_NAME_IEXT, INDEX_TMP_EXT, - MYF(0)) || + no_copy_stat ? MYF(MY_REDEL_NO_COPY_STAT) : MYF(0)) || mi_open_keyfile(share)) goto err2; info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */ @@ -2209,6 +2214,8 @@ int filecopy(MI_CHECK *param, File to,File from,my_off_t start, info MyISAM handler to repair name Name of table (for warnings) rep_quick set to <> 0 if we should not change data file + no_copy_stat Don't copy file stats from old to new file, + assume that new file was created with correct stats RESULT 0 ok @@ -2216,7 +2223,7 @@ int filecopy(MI_CHECK *param, File to,File from,my_off_t start, */ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, - const char * name, int rep_quick) + const char * name, int rep_quick, my_bool no_copy_stat) { int got_error; uint i; @@ -2543,11 +2550,15 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, /* Replace the actual file with the temporary file */ if (new_file >= 0) { + myf flags= 0; + if (param->testflag & T_BACKUP_DATA) + flags |= MY_REDEL_MAKE_BACKUP; + if (no_copy_stat) + flags |= MY_REDEL_NO_COPY_STAT; mysql_file_close(new_file, MYF(0)); info->dfile=new_file= -1; if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, DATA_TMP_EXT, - (param->testflag & T_BACKUP_DATA ? - MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + flags) || mi_open_datafile(info,share,name,-1)) got_error=1; } @@ -2595,6 +2606,8 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, info MyISAM handler to repair name Name of table (for warnings) rep_quick set to <> 0 if we should not change data file + no_copy_stat Don't copy file stats from old to new file, + assume that new file was created with correct stats DESCRIPTION Same as mi_repair_by_sort but do it multithreaded @@ -2629,7 +2642,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, */ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, - const char * name, int rep_quick) + const char * name, int rep_quick, my_bool no_copy_stat) { int got_error; uint i,key, total_key_length, istep; @@ -3076,11 +3089,15 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, /* Replace the actual file with the temporary file */ if (new_file >= 0) { + myf flags= 0; + if (param->testflag & T_BACKUP_DATA) + flags |= MY_REDEL_MAKE_BACKUP; + if (no_copy_stat) + flags |= MY_REDEL_NO_COPY_STAT; mysql_file_close(new_file, MYF(0)); info->dfile=new_file= -1; if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT, - (param->testflag & T_BACKUP_DATA ? - MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + flags) || mi_open_datafile(info,share,name,-1)) got_error=1; } diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c index 8606bd7c74868..9360a0548726a 100644 --- a/storage/myisam/myisamchk.c +++ b/storage/myisam/myisamchk.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -993,14 +993,18 @@ static int myisamchk(MI_CHECK *param, char * filename) info->s->state.key_map, param->force_sort)) { + /* + The new file might not be created with the right stats depending + on how myisamchk is run, so we must copy file stats from old to new. + */ if (param->testflag & T_REP_BY_SORT) - error=mi_repair_by_sort(param,info,filename,rep_quick); + error= mi_repair_by_sort(param, info, filename, rep_quick, FALSE); else - error=mi_repair_parallel(param,info,filename,rep_quick); + error= mi_repair_parallel(param, info, filename, rep_quick, FALSE); state_updated=1; } else if (param->testflag & T_REP_ANY) - error=mi_repair(param, info,filename,rep_quick); + error= mi_repair(param, info, filename, rep_quick, FALSE); } if (!error && param->testflag & T_SORT_RECORDS) { @@ -1040,12 +1044,12 @@ static int myisamchk(MI_CHECK *param, char * filename) { if (param->verbose) puts("Table had a compressed index; We must now recreate the index"); - error=mi_repair_by_sort(param,info,filename,1); + error= mi_repair_by_sort(param, info, filename, 1, FALSE); } } } if (!error && param->testflag & T_SORT_INDEX) - error=mi_sort_index(param,info,filename); + error= mi_sort_index(param, info, filename, FALSE); if (!error) share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR); From 48bd8b16fe382be302c6f0b45931be5aa6f29a0e Mon Sep 17 00:00:00 2001 From: Sivert Sorumgard Date: Mon, 22 Aug 2016 14:30:02 +0200 Subject: [PATCH 022/295] Bug#24388753: PRIVILEGE ESCALATION USING MYSQLD_SAFE [This is the 5.5/5.6 version of the bugfix]. The problem was that it was possible to write log files ending in .ini/.cnf that later could be parsed as an options file. This made it possible for users to specify startup options without the permissions to do so. This patch fixes the problem by disallowing general query log and slow query log to be written to files ending in .ini and .cnf. --- sql/log.cc | 89 +++++++++++++++++++++++++++++++++++++++++++++++-- sql/log.h | 10 ++++++ sql/mysqld.cc | 18 +++++++++- sql/sys_vars.cc | 25 +++++++++----- 4 files changed, 131 insertions(+), 11 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index 50d7762af6d7a..493aae8f2ff17 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -2293,6 +2293,77 @@ bool MYSQL_LOG::init_and_set_log_file_name(const char *log_name, } +bool is_valid_log_name(const char *name, size_t len) +{ + if (len > 3) + { + const char *tail= name + len - 4; + if (my_strcasecmp(system_charset_info, tail, ".ini") == 0 || + my_strcasecmp(system_charset_info, tail, ".cnf") == 0) + { + return false; + } + } + return true; +} + + +/** + Get the real log file name, and possibly reopen file. + + Use realpath() to get the path with symbolic links + expanded. Then, close the file, and reopen the real path using the + O_NOFOLLOW flag. This will reject following symbolic links. + + @param file File descriptor. + @param log_file_key Key for P_S instrumentation. + @param open_flags Flags to use for opening the file. + @param opened_file_name Name of the open fd. + + @retval file descriptor to open file with 'real_file_name', or '-1' + in case of errors. +*/ + +#ifndef _WIN32 +static File mysql_file_real_name_reopen(File file, +#ifdef HAVE_PSI_INTERFACE + PSI_file_key log_file_key, +#endif + int open_flags, + const char *opened_file_name) +{ + DBUG_ASSERT(file); + DBUG_ASSERT(opened_file_name); + + /* Buffer for realpath must have capacity for PATH_MAX. */ + char real_file_name[PATH_MAX]; + + /* Get realpath, validate, open realpath with O_NOFOLLOW. */ + if (realpath(opened_file_name, real_file_name) == NULL) + { + (void) mysql_file_close(file, MYF(0)); + return -1; + } + + if (mysql_file_close(file, MYF(0))) + return -1; + + if (strlen(real_file_name) > FN_REFLEN) + return -1; + + if (!is_valid_log_name(real_file_name, strlen(real_file_name))) + { + sql_print_error("Invalid log file name after expanding symlinks: '%s'", + real_file_name); + return -1; + } + + return mysql_file_open(log_file_key, real_file_name, + open_flags | O_NOFOLLOW, + MYF(MY_WME | ME_WAITTANG)); +} +#endif // _WIN32 + /* Open a (new) log file. @@ -2358,8 +2429,22 @@ bool MYSQL_LOG::open( if ((file= mysql_file_open(log_file_key, log_file_name, open_flags, - MYF(MY_WME | ME_WAITTANG))) < 0 || - init_io_cache(&log_file, file, IO_SIZE, io_cache_type, + MYF(MY_WME | ME_WAITTANG))) < 0) + goto err; + +#ifndef _WIN32 + /* Reopen and validate path. */ + if ((log_type_arg == LOG_UNKNOWN || log_type_arg == LOG_NORMAL) && + (file= mysql_file_real_name_reopen(file, +#ifdef HAVE_PSI_INTERFACE + log_file_key, +#endif + open_flags, + log_file_name)) < 0) + goto err; +#endif // _WIN32 + + if (init_io_cache(&log_file, file, IO_SIZE, io_cache_type, mysql_file_tell(file, MYF(MY_WME)), 0, MYF(MY_WME | MY_NABP | ((log_type == LOG_BIN) ? MY_WAIT_IF_FULL : 0)))) diff --git a/sql/log.h b/sql/log.h index b5e751386a60d..d3ecba419645f 100644 --- a/sql/log.h +++ b/sql/log.h @@ -717,6 +717,16 @@ File open_binlog(IO_CACHE *log, const char *log_file_name, char *make_log_name(char *buff, const char *name, const char* log_ext); +/** + Check given log name against certain blacklisted names/extensions. + + @param name Log name to check + @param len Length of log name + + @returns true if name is valid, false otherwise. +*/ +bool is_valid_log_name(const char *name, size_t len); + extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log; extern LOGGER logger; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a2532ceddd396..e979ea1b731a9 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify @@ -3512,6 +3512,22 @@ static int init_common_variables() "--log-slow-queries option, log tables are used. " "To enable logging to files use the --log-output=file option."); + if (opt_logname && + !is_valid_log_name(opt_logname, strlen(opt_logname))) + { + sql_print_error("Invalid value for --general_log_file: %s", + opt_logname); + return 1; + } + + if (opt_slow_logname && + !is_valid_log_name(opt_slow_logname, strlen(opt_slow_logname))) + { + sql_print_error("Invalid value for --slow_query_log_file: %s", + opt_slow_logname); + return 1; + } + #define FIX_LOG_VAR(VAR, ALT) \ if (!VAR || !*VAR) \ { \ diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index b0fa7f9a341fb..d08cb4f8ca838 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2810,6 +2810,14 @@ static bool check_log_path(sys_var *self, THD *thd, set_var *var) if (!var->save_result.string_value.str) return true; + if (!is_valid_log_name(var->save_result.string_value.str, + var->save_result.string_value.length)) + { + my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), + self->name.str, var->save_result.string_value.str); + return true; + } + if (var->save_result.string_value.length > FN_REFLEN) { // path is too long my_error(ER_PATH_LENGTH, MYF(0), self->name.str); @@ -2856,7 +2864,7 @@ static bool check_log_path(sys_var *self, THD *thd, set_var *var) return false; } static bool fix_log(char** logname, const char* default_logname, - const char*ext, bool enabled, void (*reopen)(char*)) + const char*ext, bool enabled, bool (*reopen)(char*)) { if (!*logname) // SET ... = DEFAULT { @@ -2868,16 +2876,17 @@ static bool fix_log(char** logname, const char* default_logname, } logger.lock_exclusive(); mysql_mutex_unlock(&LOCK_global_system_variables); + bool error= false; if (enabled) - reopen(*logname); + error= reopen(*logname); logger.unlock(); mysql_mutex_lock(&LOCK_global_system_variables); - return false; + return error; } -static void reopen_general_log(char* name) +static bool reopen_general_log(char* name) { logger.get_log_file_handler()->close(0); - logger.get_log_file_handler()->open_query_log(name); + return logger.get_log_file_handler()->open_query_log(name); } static bool fix_general_log_file(sys_var *self, THD *thd, enum_var_type type) { @@ -2890,10 +2899,10 @@ static Sys_var_charptr Sys_general_log_path( IN_FS_CHARSET, DEFAULT(0), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_log_path), ON_UPDATE(fix_general_log_file)); -static void reopen_slow_log(char* name) +static bool reopen_slow_log(char* name) { logger.get_slow_log_file_handler()->close(0); - logger.get_slow_log_file_handler()->open_slow_log(name); + return logger.get_slow_log_file_handler()->open_slow_log(name); } static bool fix_slow_log_file(sys_var *self, THD *thd, enum_var_type type) { From 39ec5ac403522ad452a5fe2b2839bd5d85e5ca8f Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Thu, 25 Aug 2016 11:55:54 -0400 Subject: [PATCH 023/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e748b1bda54d4..a82a4e4d77d54 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=27 +MYSQL_VERSION_PATCH=28 From 16702ec95f301d1a21eb5a6f5531387c9254b952 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Thu, 25 Aug 2016 21:19:25 -0400 Subject: [PATCH 024/295] Record wsrep.variables test result (with non-debug galera library). --- mysql-test/suite/wsrep/r/variables.result | 2 -- 1 file changed, 2 deletions(-) diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result index ab2692bed1e32..9c047177ca8ce 100644 --- a/mysql-test/suite/wsrep/r/variables.result +++ b/mysql-test/suite/wsrep/r/variables.result @@ -60,7 +60,6 @@ wsrep_cert_index_size # wsrep_causal_reads # wsrep_cert_interval # wsrep_incoming_addresses # -wsrep_debug_sync_waiters # wsrep_cluster_conf_id # wsrep_cluster_size # wsrep_cluster_state_uuid # @@ -116,7 +115,6 @@ wsrep_cert_index_size # wsrep_causal_reads # wsrep_cert_interval # wsrep_incoming_addresses # -wsrep_debug_sync_waiters # wsrep_cluster_conf_id # wsrep_cluster_size # wsrep_cluster_state_uuid # From 754e7eff2872995e2b6e62f9da7448587a411c7b Mon Sep 17 00:00:00 2001 From: Terje Rosten Date: Fri, 26 Aug 2016 11:25:40 +0200 Subject: [PATCH 025/295] Bug#24464380 PRIVILEGE ESCALATION USING MYSQLD_SAFE Post push fix: Solaris 10 /bin/sh don't understand $(). --- scripts/mysqld_safe.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 11b692ec92874..1b30a3bb15ba2 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -321,7 +321,7 @@ set_malloc_lib() { if [ "$malloc_lib" = tcmalloc ]; then malloc_lib= - for libdir in $(echo $malloc_dirs); do + for libdir in `echo $malloc_dirs`; do for flavor in _minimal '' _and_profiler _debug; do tmp="$libdir/libtcmalloc$flavor.so" #log_notice "DEBUG: Checking for malloc lib '$tmp'" @@ -348,7 +348,7 @@ set_malloc_lib() { fi # Restrict to a the list in $malloc_dirs above - case "$(dirname "$malloc_lib")" in + case "`dirname "$malloc_lib"`" in /usr/lib) ;; /usr/lib64) ;; /usr/lib/i386-linux-gnu) ;; From f81f985f37ccdcf04aa9707fe994a0c87f67b1a8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 29 Aug 2016 11:53:33 +0200 Subject: [PATCH 026/295] fix conpilation on OpenBSD --- sql/signal_handler.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index fd6f62fa100f9..c3f25848e8a16 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -64,13 +64,13 @@ extern "C" sig_handler handle_fatal_signal(int sig) struct tm tm; #ifdef HAVE_STACKTRACE THD *thd; -#endif /* This flag remembers if the query pointer was found invalid. We will try and print the query at the end of the signal handler, in case we're wrong. */ bool print_invalid_query_pointer= false; +#endif if (segfaulted) { @@ -265,6 +265,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) "\"mlockall\" bugs.\n"); } +#ifdef HAVE_STACKTRACE if (print_invalid_query_pointer) { my_safe_printf_stderr( @@ -274,6 +275,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) my_write_stderr(thd->query(), MY_MIN(65536U, thd->query_length())); my_safe_printf_stderr("\n\n"); } +#endif #ifdef HAVE_WRITE_CORE if (test_flags & TEST_CORE_ON_SIGNAL) From 616271b7c9f69596df740a8fec5a4f39b959d064 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Thu, 1 Sep 2016 12:53:44 -0400 Subject: [PATCH 027/295] Cleanup: MDL_context::wsrep_get_thd() is no longer needed --- sql/mdl.cc | 20 ++++++++++---------- sql/mdl.h | 1 - sql/wsrep_mysqld.cc | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/sql/mdl.cc b/sql/mdl.cc index 1e7502aebd8f3..0f1c961aa2913 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -1533,7 +1533,7 @@ void MDL_lock::Ticket_list::add_ticket(MDL_ticket *ticket) DBUG_ASSERT(ticket->get_lock()); #ifdef WITH_WSREP if ((this == &(ticket->get_lock()->m_waiting)) && - wsrep_thd_is_BF((void *)(ticket->get_ctx()->wsrep_get_thd()), false)) + wsrep_thd_is_BF((void *)(ticket->get_ctx()->get_thd()), false)) { Ticket_iterator itw(ticket->get_lock()->m_waiting); Ticket_iterator itg(ticket->get_lock()->m_granted); @@ -1544,11 +1544,11 @@ void MDL_lock::Ticket_list::add_ticket(MDL_ticket *ticket) while ((waiting= itw++) && !added) { - if (!wsrep_thd_is_BF((void *)(waiting->get_ctx()->wsrep_get_thd()), true)) + if (!wsrep_thd_is_BF((void *)(waiting->get_ctx()->get_thd()), true)) { WSREP_DEBUG("MDL add_ticket inserted before: %lu %s", - wsrep_thd_thread_id(waiting->get_ctx()->wsrep_get_thd()), - wsrep_thd_query(waiting->get_ctx()->wsrep_get_thd())); + wsrep_thd_thread_id(waiting->get_ctx()->get_thd()), + wsrep_thd_query(waiting->get_ctx()->get_thd())); /* Insert the ticket before the first non-BF waiting thd. */ m_list.insert_after(prev, ticket); added= true; @@ -1949,12 +1949,12 @@ MDL_lock::can_grant_lock(enum_mdl_type type_arg, ticket->is_incompatible_when_granted(type_arg)) #ifdef WITH_WSREP { - if (wsrep_thd_is_BF((void *)(requestor_ctx->wsrep_get_thd()),false) && + if (wsrep_thd_is_BF((void *)(requestor_ctx->get_thd()),false) && key.mdl_namespace() == MDL_key::GLOBAL) { WSREP_DEBUG("global lock granted for BF: %lu %s", - wsrep_thd_thread_id(requestor_ctx->wsrep_get_thd()), - wsrep_thd_query(requestor_ctx->wsrep_get_thd())); + wsrep_thd_thread_id(requestor_ctx->get_thd()), + wsrep_thd_query(requestor_ctx->get_thd())); can_grant = true; } else if (!wsrep_grant_mdl_exception(requestor_ctx, ticket, &key)) @@ -1990,12 +1990,12 @@ MDL_lock::can_grant_lock(enum_mdl_type type_arg, #ifdef WITH_WSREP else { - if (wsrep_thd_is_BF((void *)(requestor_ctx->wsrep_get_thd()), false) && + if (wsrep_thd_is_BF((void *)(requestor_ctx->get_thd()), false) && key.mdl_namespace() == MDL_key::GLOBAL) { WSREP_DEBUG("global lock granted for BF (waiting queue): %lu %s", - wsrep_thd_thread_id(requestor_ctx->wsrep_get_thd()), - wsrep_thd_query(requestor_ctx->wsrep_get_thd())); + wsrep_thd_thread_id(requestor_ctx->get_thd()), + wsrep_thd_query(requestor_ctx->get_thd())); can_grant = true; } } diff --git a/sql/mdl.h b/sql/mdl.h index 7fc58bcfb8b46..86f681c90f06c 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -933,7 +933,6 @@ class MDL_context public: #ifdef WITH_WSREP - THD *wsrep_get_thd() const { return get_thd(); } bool wsrep_has_explicit_locks(); #endif /* WITH_WSREP */ THD *get_thd() const { return m_owner->get_thd(); } diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 4117d5231f4ba..13b72b65afeb2 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1520,8 +1520,8 @@ wsrep_grant_mdl_exception(MDL_context *requestor_ctx, /* Fallback to the non-wsrep behaviour */ if (!WSREP_ON) return FALSE; - THD *request_thd = requestor_ctx->wsrep_get_thd(); - THD *granted_thd = ticket->get_ctx()->wsrep_get_thd(); + THD *request_thd = requestor_ctx->get_thd(); + THD *granted_thd = ticket->get_ctx()->get_thd(); bool ret = FALSE; const char* schema= key->db_name(); From 3dd88fbd1132ae36c00adb67cbe2fc4ff97a6789 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Thu, 1 Sep 2016 12:59:11 -0400 Subject: [PATCH 028/295] MDEV-10714: Could not execute Delete_rows event on table; wsrep_max_ws_rows exceeded. Error_Code 1180 The wsrep_max_ws_rows related implementation should be skipped when server is running with wsrep disabled. --- mysql-test/suite/wsrep/r/wsrep_rpl.result | 22 ++++++++++ mysql-test/suite/wsrep/t/wsrep_rpl.cnf | 1 + mysql-test/suite/wsrep/t/wsrep_rpl.test | 50 +++++++++++++++++++++ sql/handler.cc | 53 +++++++++++++---------- 4 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 mysql-test/suite/wsrep/r/wsrep_rpl.result create mode 100644 mysql-test/suite/wsrep/t/wsrep_rpl.cnf create mode 100644 mysql-test/suite/wsrep/t/wsrep_rpl.test diff --git a/mysql-test/suite/wsrep/r/wsrep_rpl.result b/mysql-test/suite/wsrep/r/wsrep_rpl.result new file mode 100644 index 0000000000000..fd792c215b3bd --- /dev/null +++ b/mysql-test/suite/wsrep/r/wsrep_rpl.result @@ -0,0 +1,22 @@ +include/master-slave.inc +[connection master] +# +# MDEV-10714: Could not execute Delete_rows event on table; +# wsrep_max_ws_rows exceeded. Error_Code 1180 +# +CREATE TABLE t1(i INT) ENGINE = INNODB; +SET @@GLOBAL.wsrep_max_ws_rows = 1; +INSERT INTO t1 VALUES(1), (2); +SELECT COUNT(*) = 2 FROM t1; +COUNT(*) = 2 +1 +SET @@GLOBAL.wsrep_max_ws_rows = 1; +DELETE FROM t1; +SELECT COUNT(*) = 0 FROM t1; +COUNT(*) = 0 +1 +DROP TABLE t1; +SET @@GLOBAL.wsrep_max_ws_rows = 0; +SET @@GLOBAL.wsrep_max_ws_rows = 0; +include/rpl_end.inc +# End of test. diff --git a/mysql-test/suite/wsrep/t/wsrep_rpl.cnf b/mysql-test/suite/wsrep/t/wsrep_rpl.cnf new file mode 100644 index 0000000000000..56e874f22e1b9 --- /dev/null +++ b/mysql-test/suite/wsrep/t/wsrep_rpl.cnf @@ -0,0 +1 @@ +!include ../../rpl/my.cnf diff --git a/mysql-test/suite/wsrep/t/wsrep_rpl.test b/mysql-test/suite/wsrep/t/wsrep_rpl.test new file mode 100644 index 0000000000000..1cc7214325dc0 --- /dev/null +++ b/mysql-test/suite/wsrep/t/wsrep_rpl.test @@ -0,0 +1,50 @@ +--source include/have_wsrep.inc +--source include/have_innodb.inc +--source include/master-slave.inc + +--echo # +--echo # MDEV-10714: Could not execute Delete_rows event on table; +--echo # wsrep_max_ws_rows exceeded. Error_Code 1180 +--echo # +# Save wsrep_max_ws_rows on master and slave. +connection master; +let $wsrep_max_ws_rows_master = `SELECT @@GLOBAL.wsrep_max_ws_rows`; +connection slave; +let $wsrep_max_ws_rows_slave = `SELECT @@GLOBAL.wsrep_max_ws_rows`; + +connection master; +CREATE TABLE t1(i INT) ENGINE = INNODB; + +# Setting wsrep_max_ws_rows should have no impact on replication master +# unless its a cluster node. +SET @@GLOBAL.wsrep_max_ws_rows = 1; +INSERT INTO t1 VALUES(1), (2); + +sync_slave_with_master; +SELECT COUNT(*) = 2 FROM t1; + +connection slave; +# Setting wsrep_max_ws_rows should have no impact on replication slave +# unless its a cluster node. +SET @@GLOBAL.wsrep_max_ws_rows = 1; + +connection master; +DELETE FROM t1; + +sync_slave_with_master; +SELECT COUNT(*) = 0 FROM t1; + +connection master; +DROP TABLE t1; + +sync_slave_with_master; + +# Restore wsrep_max_ws_rows on master and slave +connection master; +eval SET @@GLOBAL.wsrep_max_ws_rows = $wsrep_max_ws_rows_master; +connection slave; +eval SET @@GLOBAL.wsrep_max_ws_rows = $wsrep_max_ws_rows_slave; + +--source include/rpl_end.inc +--echo # End of test. + diff --git a/sql/handler.cc b/sql/handler.cc index af83e915f0e41..a142bb4e0de93 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6109,14 +6109,17 @@ int handler::ha_write_row(uchar *buf) if (unlikely(error= binlog_log_row(table, 0, buf, log_func))) DBUG_RETURN(error); /* purecov: inspected */ #ifdef WITH_WSREP - current_thd->wsrep_affected_rows++; - if (wsrep_max_ws_rows && - current_thd->wsrep_exec_mode != REPL_RECV && - current_thd->wsrep_affected_rows > wsrep_max_ws_rows) + if (WSREP(current_thd)) { - trans_rollback_stmt(current_thd) || trans_rollback(current_thd); - my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); - DBUG_RETURN(ER_ERROR_DURING_COMMIT); + current_thd->wsrep_affected_rows++; + if (wsrep_max_ws_rows && + current_thd->wsrep_exec_mode != REPL_RECV && + current_thd->wsrep_affected_rows > wsrep_max_ws_rows) + { + trans_rollback_stmt(current_thd) || trans_rollback(current_thd); + my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); + DBUG_RETURN(ER_ERROR_DURING_COMMIT); + } } #endif /* WITH_WSREP */ @@ -6153,14 +6156,17 @@ int handler::ha_update_row(const uchar *old_data, uchar *new_data) if (unlikely(error= binlog_log_row(table, old_data, new_data, log_func))) return error; #ifdef WITH_WSREP - current_thd->wsrep_affected_rows++; - if (wsrep_max_ws_rows && - current_thd->wsrep_exec_mode != REPL_RECV && - current_thd->wsrep_affected_rows > wsrep_max_ws_rows) + if (WSREP(current_thd)) { - trans_rollback_stmt(current_thd) || trans_rollback(current_thd); - my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); - return ER_ERROR_DURING_COMMIT; + current_thd->wsrep_affected_rows++; + if (wsrep_max_ws_rows && + current_thd->wsrep_exec_mode != REPL_RECV && + current_thd->wsrep_affected_rows > wsrep_max_ws_rows) + { + trans_rollback_stmt(current_thd) || trans_rollback(current_thd); + my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); + return ER_ERROR_DURING_COMMIT; + } } #endif /* WITH_WSREP */ return 0; @@ -6191,14 +6197,17 @@ int handler::ha_delete_row(const uchar *buf) if (unlikely(error= binlog_log_row(table, buf, 0, log_func))) return error; #ifdef WITH_WSREP - current_thd->wsrep_affected_rows++; - if (wsrep_max_ws_rows && - current_thd->wsrep_exec_mode != REPL_RECV && - current_thd->wsrep_affected_rows > wsrep_max_ws_rows) - { - trans_rollback_stmt(current_thd) || trans_rollback(current_thd); - my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); - return ER_ERROR_DURING_COMMIT; + if (WSREP(current_thd)) + { + current_thd->wsrep_affected_rows++; + if (wsrep_max_ws_rows && + current_thd->wsrep_exec_mode != REPL_RECV && + current_thd->wsrep_affected_rows > wsrep_max_ws_rows) + { + trans_rollback_stmt(current_thd) || trans_rollback(current_thd); + my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); + return ER_ERROR_DURING_COMMIT; + } } #endif /* WITH_WSREP */ return 0; From b9631e310b7cadf8711eef643e432d7e816680b4 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 10 Nov 2015 12:41:26 +0100 Subject: [PATCH 029/295] MDEV-8833 Crash of server on prepared statement with conversion to semi-join Correct context chain made to allow outer fields pullout. --- mysql-test/r/ps.result | 33 ++++++++++++++++++++++++++++++++- mysql-test/t/ps.test | 29 ++++++++++++++++++++++++++++- sql/item.cc | 23 +++++++++++++++++++++-- 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index 04a19d3840f16..bb8b76faa49b4 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -4072,4 +4072,35 @@ id value deallocate prepare stmt; SET SESSION sql_mode = @save_sql_mode; DROP TABLE t1,t2; -# End of 10.0 tests +# +# MDEV-8833: Crash of server on prepared statement with +# conversion to semi-join +# +CREATE TABLE t1 (column1 INT); +INSERT INTO t1 VALUES (3),(9); +CREATE TABLE t2 (column2 INT); +INSERT INTO t2 VALUES (1),(4); +CREATE TABLE t3 (column3 INT); +INSERT INTO t3 VALUES (6),(8); +CREATE TABLE t4 (column4 INT); +INSERT INTO t4 VALUES (2),(5); +PREPARE stmt FROM "SELECT ( SELECT MAX( table1.column1 ) AS field1 +FROM t1 AS table1 +WHERE table3.column3 IN ( SELECT table2.column2 AS field2 FROM t2 AS table2 ) +) AS sq +FROM t3 AS table3, t4 AS table4"; +EXECUTE stmt; +sq +NULL +NULL +NULL +NULL +EXECUTE stmt; +sq +NULL +NULL +NULL +NULL +deallocate prepare stmt; +drop table t1,t2,t3,t4; +# End of 5.5 tests diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index 2ed5bb11bacbd..1516acca01efd 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -3653,5 +3653,32 @@ deallocate prepare stmt; SET SESSION sql_mode = @save_sql_mode; DROP TABLE t1,t2; +--echo # +--echo # MDEV-8833: Crash of server on prepared statement with +--echo # conversion to semi-join +--echo # + +CREATE TABLE t1 (column1 INT); +INSERT INTO t1 VALUES (3),(9); + +CREATE TABLE t2 (column2 INT); +INSERT INTO t2 VALUES (1),(4); + +CREATE TABLE t3 (column3 INT); +INSERT INTO t3 VALUES (6),(8); + +CREATE TABLE t4 (column4 INT); +INSERT INTO t4 VALUES (2),(5); + +PREPARE stmt FROM "SELECT ( SELECT MAX( table1.column1 ) AS field1 +FROM t1 AS table1 +WHERE table3.column3 IN ( SELECT table2.column2 AS field2 FROM t2 AS table2 ) +) AS sq +FROM t3 AS table3, t4 AS table4"; +EXECUTE stmt; +EXECUTE stmt; +deallocate prepare stmt; +drop table t1,t2,t3,t4; + ---echo # End of 10.0 tests +--echo # End of 5.5 tests diff --git a/sql/item.cc b/sql/item.cc index 5861766371cfd..abcf48fc270a0 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -2778,9 +2778,28 @@ void Item_field::fix_after_pullout(st_select_lex *new_parent, Item **ref) if (context) { Name_resolution_context *ctx= new Name_resolution_context(); - ctx->outer_context= NULL; // We don't build a complete name resolver - ctx->table_list= NULL; // We rely on first_name_resolution_table instead + if (context->select_lex == new_parent) + { + /* + This field was pushed in then pulled out + (for example left part of IN) + */ + ctx->outer_context= context->outer_context; + } + else if (context->outer_context) + { + /* just pull to the upper context */ + ctx->outer_context= context->outer_context->outer_context; + } + else + { + /* No upper context (merging Derived/VIEW where context chain ends) */ + ctx->outer_context= NULL; + } + ctx->table_list= context->first_name_resolution_table; ctx->select_lex= new_parent; + if (context->select_lex == NULL) + ctx->select_lex= NULL; ctx->first_name_resolution_table= context->first_name_resolution_table; ctx->last_name_resolution_table= context->last_name_resolution_table; ctx->error_processor= context->error_processor; From a14f61ef749ad9f9ab2b0f5badf6754ba7443c9e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 5 Sep 2016 12:28:35 +0300 Subject: [PATCH 030/295] MDEV-7142: main.index_merge_innodb fails sporadically in buildbot Attempt to stabilize the testcase. --- mysql-test/include/index_merge2.inc | 1 + mysql-test/r/index_merge_innodb.result | 3 +++ mysql-test/r/index_merge_myisam.result | 3 +++ 3 files changed, 7 insertions(+) diff --git a/mysql-test/include/index_merge2.inc b/mysql-test/include/index_merge2.inc index c50a45a9923d9..03afa49d323f0 100644 --- a/mysql-test/include/index_merge2.inc +++ b/mysql-test/include/index_merge2.inc @@ -341,6 +341,7 @@ while ($1) alter table t1 add index i2(key2); alter table t1 add index i3(key3); update t1 set key2=key1,key3=key1; +analyze table t1; # to test the bug, the following must use "sort_union": --replace_column 9 REF diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 5202c79f3c749..5bf56e213abbb 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -311,6 +311,9 @@ set @d=@d*2; alter table t1 add index i2(key2); alter table t1 add index i3(key3); update t1 set key2=key1,key3=key1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK explain select * from t1 where (key3 > 30 and key3<35) or (key2 >32 and key2 < 40); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index_merge i2,i3 i3,i2 4,4 NULL REF Using sort_union(i3,i2); Using where diff --git a/mysql-test/r/index_merge_myisam.result b/mysql-test/r/index_merge_myisam.result index fcd5eebefa482..c63ed13266255 100644 --- a/mysql-test/r/index_merge_myisam.result +++ b/mysql-test/r/index_merge_myisam.result @@ -1146,6 +1146,9 @@ set @d=@d*2; alter table t1 add index i2(key2); alter table t1 add index i3(key3); update t1 set key2=key1,key3=key1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK explain select * from t1 where (key3 > 30 and key3<35) or (key2 >32 and key2 < 40); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index_merge i2,i3 i3,i2 4,4 NULL REF Using sort_union(i3,i2); Using where From 213765cc222139c05c27774e0555cabeff7c3bbd Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Sep 2016 13:18:04 +0200 Subject: [PATCH 031/295] - Fix MDEV-10496. Memory leak in discovery modified: storage/connect/ha_connect.cc modified: storage/connect/mycat.cc - Fix wrong lrecl calculation for virtual columns modified: storage/connect/reldef.cpp - Typo modified: storage/connect/jdbconn.cpp modified: storage/connect/json.cpp --- storage/connect/ha_connect.cc | 84 ++++++++------- storage/connect/jdbconn.cpp | 195 +--------------------------------- storage/connect/json.cpp | 6 +- storage/connect/mycat.cc | 14 +-- storage/connect/reldef.cpp | 2 +- 5 files changed, 54 insertions(+), 247 deletions(-) diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index ea6fb1b08c16d..cf945a73f4610 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -224,6 +224,7 @@ uint GetWorkSize(void); void SetWorkSize(uint); extern "C" const char *msglang(void); +static void PopUser(PCONNECT xp); static PCONNECT GetUser(THD *thd, PCONNECT xp); static PGLOBAL GetPlug(THD *thd, PCONNECT& lxp); @@ -831,34 +832,43 @@ ha_connect::~ha_connect(void) table ? table->s->table_name.str : "", xp, xp ? xp->count : 0); - if (xp) { - PCONNECT p; + PopUser(xp); +} // end of ha_connect destructor - xp->count--; - for (p= user_connect::to_users; p; p= p->next) - if (p == xp) - break; +/****************************************************************************/ +/* Check whether this user can be removed. */ +/****************************************************************************/ +static void PopUser(PCONNECT xp) +{ + if (xp) { + xp->count--; - if (p && !p->count) { - if (p->next) - p->next->previous= p->previous; + if (!xp->count) { + PCONNECT p; - if (p->previous) - p->previous->next= p->next; - else - user_connect::to_users= p->next; + for (p= user_connect::to_users; p; p= p->next) + if (p == xp) + break; - } // endif p + if (p) { + if (p->next) + p->next->previous= p->previous; - if (!xp->count) { - PlugCleanup(xp->g, true); - delete xp; - } // endif count + if (p->previous) + p->previous->next= p->next; + else + user_connect::to_users= p->next; - } // endif xp + } // endif p -} // end of ha_connect destructor + PlugCleanup(xp->g, true); + delete xp; + } // endif count + + } // endif xp + +} // end of PopUser /****************************************************************************/ @@ -866,7 +876,7 @@ ha_connect::~ha_connect(void) /****************************************************************************/ static PCONNECT GetUser(THD *thd, PCONNECT xp) { - if (!thd) + if (!thd) return NULL; if (xp && thd == xp->thdp) @@ -890,7 +900,6 @@ static PCONNECT GetUser(THD *thd, PCONNECT xp) return xp; } // end of GetUser - /****************************************************************************/ /* Get the global pointer of the user of this handler. */ /****************************************************************************/ @@ -5261,7 +5270,18 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!(shm= (char*)db)) db= table_s->db.str; // Default value - // Check table type + // Save stack and allocation environment and prepare error return + if (g->jump_level == MAX_JUMP) { + strcpy(g->Message, MSG(TOO_MANY_JUMPS)); + goto jer; + } // endif jump_level + + if ((rc= setjmp(g->jumper[++g->jump_level])) != 0) { + my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); + goto err; + } // endif rc + + // Check table type if (ttp == TAB_UNDEF) { topt->type= (src) ? "MYSQL" : (tab) ? "PROXY" : "DOS"; ttp= GetTypeID(topt->type); @@ -5270,20 +5290,9 @@ static int connect_assisted_discovery(handlerton *, THD* thd, } else if (ttp == TAB_NIY) { sprintf(g->Message, "Unsupported table type %s", topt->type); my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); - return HA_ERR_INTERNAL_ERROR; + goto err; } // endif ttp - // Save stack and allocation environment and prepare error return - if (g->jump_level == MAX_JUMP) { - strcpy(g->Message, MSG(TOO_MANY_JUMPS)); - return HA_ERR_INTERNAL_ERROR; - } // endif jump_level - - if ((rc= setjmp(g->jumper[++g->jump_level])) != 0) { - my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); - goto err; - } // endif rc - if (!tab) { if (ttp == TAB_TBL) { // Make tab the first table of the list @@ -5843,6 +5852,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, rc= init_table_share(thd, table_s, create_info, &sql); g->jump_level--; + PopUser(xp); return rc; } // endif ok @@ -5850,7 +5860,9 @@ static int connect_assisted_discovery(handlerton *, THD* thd, err: g->jump_level--; - return HA_ERR_INTERNAL_ERROR; + jer: + PopUser(xp); + return HA_ERR_INTERNAL_ERROR; } // end of connect_assisted_discovery /** diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index 3b8de3e975bfb..952847507a0b3 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -498,145 +498,6 @@ PQRYRES JDBCDrivers(PGLOBAL g, int maxres, bool info) return qrp; } // end of JDBCDrivers -#if 0 -/*************************************************************************/ -/* JDBCDataSources: constructs the result blocks containing all JDBC */ -/* data sources available on the local host. */ -/* Called with info=true to have result column names. */ -/*************************************************************************/ -PQRYRES JDBCDataSources(PGLOBAL g, int maxres, bool info) -{ - int buftyp[] ={ TYPE_STRING, TYPE_STRING }; - XFLD fldtyp[] ={ FLD_NAME, FLD_REM }; - unsigned int length[] ={ 0, 256 }; - bool b[] ={ false, true }; - int i, n = 0, ncol = 2; - PCOLRES crp; - PQRYRES qrp; - JDBConn *jcp = NULL; - - /************************************************************************/ - /* Do an evaluation of the result size. */ - /************************************************************************/ - if (!info) { - jcp = new(g)JDBConn(g, NULL); - n = jcp->GetMaxValue(SQL_MAX_DSN_LENGTH); - length[0] = (n) ? (n + 1) : 256; - - if (!maxres) - maxres = 512; // Estimated max number of data sources - - } else { - length[0] = 256; - maxres = 0; - } // endif info - - if (trace) - htrc("JDBCDataSources: max=%d len=%d\n", maxres, length[0]); - - /************************************************************************/ - /* Allocate the structures used to refer to the result set. */ - /************************************************************************/ - qrp = PlgAllocResult(g, ncol, maxres, IDS_DSRC, - buftyp, fldtyp, length, false, true); - - for (i = 0, crp = qrp->Colresp; crp; i++, crp = crp->Next) - if (b[i]) - crp->Kdata->SetNullable(true); - - /************************************************************************/ - /* Now get the results into blocks. */ - /************************************************************************/ - if (!info && qrp && jcp->GetDataSources(qrp)) - qrp = NULL; - - /************************************************************************/ - /* Return the result pointer for use by GetData routines. */ - /************************************************************************/ - return qrp; -} // end of JDBCDataSources - -/**************************************************************************/ -/* PrimaryKeys: constructs the result blocks containing all the */ -/* JDBC catalog information concerning primary keys. */ -/**************************************************************************/ -PQRYRES JDBCPrimaryKeys(PGLOBAL g, JDBConn *op, char *dsn, char *table) -{ - static int buftyp[] ={ TYPE_STRING, TYPE_STRING, TYPE_STRING, - TYPE_STRING, TYPE_SHORT, TYPE_STRING }; - static unsigned int length[] ={ 0, 0, 0, 0, 6, 128 }; - int n, ncol = 5; - int maxres; - PQRYRES qrp; - JCATPARM *cap; - JDBConn *jcp = op; - - if (!op) { - /**********************************************************************/ - /* Open the connection with the JDBC data source. */ - /**********************************************************************/ - jcp = new(g)JDBConn(g, NULL); - - if (jcp->Open(dsn, 2) < 1) // 2 is openReadOnly - return NULL; - - } // endif op - - /************************************************************************/ - /* Do an evaluation of the result size. */ - /************************************************************************/ - n = jcp->GetMaxValue(SQL_MAX_COLUMNS_IN_TABLE); - maxres = (n) ? (int)n : 250; - n = jcp->GetMaxValue(SQL_MAX_CATALOG_NAME_LEN); - length[0] = (n) ? (n + 1) : 128; - n = jcp->GetMaxValue(SQL_MAX_SCHEMA_NAME_LEN); - length[1] = (n) ? (n + 1) : 128; - n = jcp->GetMaxValue(SQL_MAX_TABLE_NAME_LEN); - length[2] = (n) ? (n + 1) : 128; - n = jcp->GetMaxValue(SQL_MAX_COLUMN_NAME_LEN); - length[3] = (n) ? (n + 1) : 128; - - if (trace) - htrc("JDBCPrimaryKeys: max=%d len=%d,%d,%d\n", - maxres, length[0], length[1], length[2]); - - /************************************************************************/ - /* Allocate the structure used to refer to the result set. */ - /************************************************************************/ - qrp = PlgAllocResult(g, ncol, maxres, IDS_PKEY, - buftyp, NULL, length, false, true); - - if (trace) - htrc("Getting pkey results ncol=%d\n", qrp->Nbcol); - - cap = AllocCatInfo(g, CAT_KEY, NULL, table, qrp); - - /************************************************************************/ - /* Now get the results into blocks. */ - /************************************************************************/ - if ((n = jcp->GetCatInfo(cap)) >= 0) { - qrp->Nblin = n; - // ResetNullValues(cap); - - if (trace) - htrc("PrimaryKeys: NBCOL=%d NBLIN=%d\n", qrp->Nbcol, qrp->Nblin); - - } else - qrp = NULL; - - /************************************************************************/ - /* Close any local connection. */ - /************************************************************************/ - if (!op) - jcp->Close(); - - /************************************************************************/ - /* Return the result pointer for use by GetData routines. */ - /************************************************************************/ - return qrp; -} // end of JDBCPrimaryKeys -#endif // 0 - /***********************************************************************/ /* JDBConn construction/destruction. */ /***********************************************************************/ @@ -739,60 +600,6 @@ bool JDBConn::gmID(PGLOBAL g, jmethodID& mid, const char *name, const char *sig } // end of gmID -#if 0 -/***********************************************************************/ -/* Utility routine. */ -/***********************************************************************/ -PSZ JDBConn::GetStringInfo(ushort infotype) -{ - //ASSERT(m_hdbc != SQL_NULL_HDBC); - char *p, buffer[MAX_STRING_INFO]; - SWORD result; - RETCODE rc; - - rc = SQLGetInfo(m_hdbc, infotype, buffer, sizeof(buffer), &result); - - if (!Check(rc)) { - ThrowDJX(rc, "SQLGetInfo"); // Temporary - // *buffer = '\0'; - } // endif rc - - p = PlugDup(m_G, buffer); - return p; -} // end of GetStringInfo - -/***********************************************************************/ -/* Utility routines. */ -/***********************************************************************/ -void JDBConn::OnSetOptions(HSTMT hstmt) -{ - RETCODE rc; - ASSERT(m_hdbc != SQL_NULL_HDBC); - - if ((signed)m_QueryTimeout != -1) { - // Attempt to set query timeout. Ignore failure - rc = SQLSetStmtOption(hstmt, SQL_QUERY_TIMEOUT, m_QueryTimeout); - - if (!Check(rc)) - // don't attempt it again - m_QueryTimeout = (DWORD)-1; - - } // endif m_QueryTimeout - - if (m_RowsetSize > 0) { - // Attempt to set rowset size. - // In case of failure reset it to 0 to use Fetch. - rc = SQLSetStmtOption(hstmt, SQL_ROWSET_SIZE, m_RowsetSize); - - if (!Check(rc)) - // don't attempt it again - m_RowsetSize = 0; - - } // endif m_RowsetSize - -} // end of OnSetOptions -#endif // 0 - /***********************************************************************/ /* Utility routine. */ /***********************************************************************/ @@ -1007,7 +814,7 @@ int JDBConn::Open(PJPARM sop) #define N 1 #endif - // Java source will be compiled as ajar file installed in the plugin dir + // Java source will be compiled as a jar file installed in the plugin dir jpop->Append(sep); jpop->Append(GetPluginDir()); jpop->Append("JdbcInterface.jar"); diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index 75bf277b25bf5..c45630129f120 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -595,7 +595,7 @@ PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty) fputs(EL, fs); fclose(fs); str = (err) ? NULL : strcpy(g->Message, "Ok"); - } else if (!err) { + } else if (!err) { str = ((JOUTSTR*)jp)->Strp; jp->WriteChr('\0'); PlugSubAlloc(g, NULL, ((JOUTSTR*)jp)->N); @@ -767,7 +767,7 @@ bool JOUTSTR::Escape(const char *s) { WriteChr('"'); - for (unsigned int i = 0; i < strlen(s); i++) + for (unsigned int i = 0; s[i]; i++) switch (s[i]) { case '"': case '\\': @@ -816,7 +816,7 @@ bool JOUTFILE::Escape(const char *s) // This is temporary fputc('"', Stream); - for (unsigned int i = 0; i < strlen(s); i++) + for (unsigned int i = 0; s[i]; i++) switch (s[i]) { case '"': fputs("\\\"", Stream); break; case '\\': fputs("\\\\", Stream); break; diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index da8be20723796..b4b03e6ba4a22 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -109,19 +109,7 @@ PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info); /***********************************************************************/ char *GetPluginDir(void) { - char *plugin_dir; - -#if defined(_WIN64) - plugin_dir = (char *)GetProcAddress(GetModuleHandle(NULL), - "?opt_plugin_dir@@3PADEA"); -#elif defined(_WIN32) - plugin_dir = (char*)GetProcAddress(GetModuleHandle(NULL), - "?opt_plugin_dir@@3PADA"); -#else - plugin_dir = opt_plugin_dir; -#endif - - return plugin_dir; + return opt_plugin_dir; } // end of GetPluginDir /***********************************************************************/ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 2c8ada52e6f5c..8ad6e203d519c 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -294,7 +294,7 @@ int TABDEF::GetColCatInfo(PGLOBAL g) nlg+= nof; case TAB_DIR: case TAB_XML: - poff= loff + 1; + poff= loff + (pcf->Flags & U_VIRTUAL ? 0 : 1); break; case TAB_INI: case TAB_MAC: From 6c74ef8ae9cdfedb20827694362cad0fccaa5880 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 7 Sep 2016 09:30:02 +1000 Subject: [PATCH 032/295] MDEV-10707: Fix tokudb test rows-32m-rand-insert (#231) MDEV-10757: Fix tokudb test rows-32m-rand-insert --- storage/tokudb/mysql-test/tokudb/r/rows-32m-rand-insert.result | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/tokudb/mysql-test/tokudb/r/rows-32m-rand-insert.result b/storage/tokudb/mysql-test/tokudb/r/rows-32m-rand-insert.result index 5c1c53946a464..b287c70469e08 100644 --- a/storage/tokudb/mysql-test/tokudb/r/rows-32m-rand-insert.result +++ b/storage/tokudb/mysql-test/tokudb/r/rows-32m-rand-insert.result @@ -1009,6 +1009,7 @@ Table Op Msg_type Msg_text test.t check status OK optimize table t; Table Op Msg_type Msg_text +test.t optimize note Table does not support optimize, doing recreate + analyze instead test.t optimize status OK check table t; Table Op Msg_type Msg_text From de7f87708a156659e518bf4ccdb051e92a0a7521 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 9 Sep 2016 08:40:24 +0400 Subject: [PATCH 033/295] MDEV-10702 Crash in SET STATEMENT FOR EXECUTE --- mysql-test/r/ps_ddl.result | 29 +++++++++++++++++++++++ mysql-test/t/ps_ddl.test | 24 +++++++++++++++++++ sql/sql_prepare.cc | 48 +++++++++++++++++++++++++++++++------- 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/ps_ddl.result b/mysql-test/r/ps_ddl.result index dec0d12c45501..fc1832260c59b 100644 --- a/mysql-test/r/ps_ddl.result +++ b/mysql-test/r/ps_ddl.result @@ -2542,3 +2542,32 @@ EXECUTE stmt3; EXECUTE stmt3; DEALLOCATE PREPARE stmt3; DROP TEMPORARY TABLES tm, t1; +# +# Start of 10.1 tests +# +# +# MDEV-10702 Crash in SET STATEMENT FOR EXECUTE +# +CREATE TABLE t1 (a INT); +PREPARE stmt FROM 'INSERT INTO t1 VALUES (@@max_sort_length)'; +SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; +SELECT * FROM t1; +a +2048 +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW SET NEW.a=NEW.a + 1; +SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; +SELECT * FROM t1; +a +2048 +1025 +DROP TRIGGER tr1; +SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; +SELECT * FROM t1; +a +2048 +1025 +1024 +DROP TABLE t1; +# +# End of 10.1 tests +# diff --git a/mysql-test/t/ps_ddl.test b/mysql-test/t/ps_ddl.test index 21355ca42b7a1..90226d379bfb1 100644 --- a/mysql-test/t/ps_ddl.test +++ b/mysql-test/t/ps_ddl.test @@ -2259,3 +2259,27 @@ EXECUTE stmt3; EXECUTE stmt3; DEALLOCATE PREPARE stmt3; DROP TEMPORARY TABLES tm, t1; + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-10702 Crash in SET STATEMENT FOR EXECUTE +--echo # +CREATE TABLE t1 (a INT); +PREPARE stmt FROM 'INSERT INTO t1 VALUES (@@max_sort_length)'; +SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; +SELECT * FROM t1; +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW SET NEW.a=NEW.a + 1; +SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; +SELECT * FROM t1; +DROP TRIGGER tr1; +SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; +SELECT * FROM t1; +DROP TABLE t1; + + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 0db1daa378e52..d381825851d6d 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -3030,7 +3030,36 @@ void mysql_sql_stmt_execute(THD *thd) DBUG_PRINT("info",("stmt: 0x%lx", (long) stmt)); + /* + thd->free_list can already have some Items, + e.g. for a query like this: + PREPARE stmt FROM 'INSERT INTO t1 VALUES (@@max_sort_length)'; + SET STATEMENT max_sort_length=2048 FOR EXECUTE stmt; + thd->free_list contains a pointer to Item_int corresponding to 2048. + + If Prepared_statement::execute() notices that the table metadata for "t1" + has changed since PREPARE, it returns an error asking the calling + Prepared_statement::execute_loop() to re-prepare the statement. + Before returning the error, Prepared_statement::execute() + calls Prepared_statement::cleanup_stmt(), + which calls thd->cleanup_after_query(), + which calls Query_arena::free_items(). + + We hide "external" Items, e.g. those created while parsing the + "SET STATEMENT" part of the query, + so they don't get freed in case of re-prepare. + See MDEV-10702 Crash in SET STATEMENT FOR EXECUTE + */ + Item *free_list_backup= thd->free_list; + thd->free_list= NULL; // Hide the external (e.g. "SET STATEMENT") Items (void) stmt->execute_loop(&expanded_query, FALSE, NULL, NULL); + thd->free_items(); // Free items created by execute_loop() + /* + Now restore the "external" (e.g. "SET STATEMENT") Item list. + It will be freed normaly in THD::cleanup_after_query(). + */ + thd->free_list= free_list_backup; + stmt->lex->restore_set_statement_var(); DBUG_VOID_RETURN; } @@ -3853,9 +3882,14 @@ Prepared_statement::execute_loop(String *expanded_query, Reprepare_observer reprepare_observer; bool error; int reprepare_attempt= 0; -#ifndef DBUG_OFF - Item *free_list_state= thd->free_list; -#endif + + /* + - In mysql_sql_stmt_execute() we hide all "external" Items + e.g. those created in the "SET STATEMENT" part of the "EXECUTE" query. + - In case of mysqld_stmt_execute() there should not be "external" Items. + */ + DBUG_ASSERT(thd->free_list == NULL); + thd->select_number= select_number_after_prepare; /* Check if we got an error when sending long data */ if (state == Query_arena::STMT_ERROR) @@ -3877,12 +3911,8 @@ Prepared_statement::execute_loop(String *expanded_query, #endif reexecute: - /* - If the free_list is not empty, we'll wrongly free some externally - allocated items when cleaning up after validation of the prepared - statement. - */ - DBUG_ASSERT(thd->free_list == free_list_state); + // Make sure that reprepare() did not create any new Items. + DBUG_ASSERT(thd->free_list == NULL); /* Install the metadata observer. If some metadata version is From 577f3c1dce2011f51a01811fabd4ebd4e6f4d1ed Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sat, 10 Sep 2016 17:50:32 +0200 Subject: [PATCH 034/295] Fix use of `require` in mysql-test-run. The motivation for this is that Perl is moving towards not having current directory ./ in @INC by default. This is causing mysql-test-run.pl to fail in latest Debian Unstable: https://lists.debian.org/debian-devel-announce/2016/08/msg00013.html However, we have `use "lib"`, there is no need for current directory in @INC, except for a gross hack. In mtr_cases.pm, there is a `require "mtr_misc.pl"`, which hides mtr_misc.pl away in mtr_cases namespace. And things only work because mysql-test-run.pl loads it with a different name, `require "lib/mtr_misc.pl"`! (Perl will `require` only once for each unique filename). Fix this by only using `require` in main program, and referencing functions with :: scope from other namespaces. For multi-use in different namespaces, proper `use` modules should be used. Signed-off-by: Kristian Nielsen --- mysql-test/lib/mtr_cases.pm | 4 +--- mysql-test/lib/mtr_report.pm | 3 +-- mysql-test/mysql-test-run.pl | 10 +++++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm index 10e5fd5c33725..38c52b705f6d8 100644 --- a/mysql-test/lib/mtr_cases.pm +++ b/mysql-test/lib/mtr_cases.pm @@ -60,8 +60,6 @@ use My::Test; use My::Find; use My::Suite; -require "mtr_misc.pl"; - # locate plugin suites, depending on whether it's a build tree or installed my @plugin_suitedirs; my $plugin_suitedir_regex; @@ -1122,7 +1120,7 @@ sub get_tags_from_file($$) { $file_to_tags{$file}= $tags; $file_to_master_opts{$file}= $master_opts; $file_to_slave_opts{$file}= $slave_opts; - $file_combinations{$file}= [ uniq(@combinations) ]; + $file_combinations{$file}= [ ::uniq(@combinations) ]; $file_in_overlay{$file} = 1 if $in_overlay; return @{$tags}; } diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm index 9ab82c454ed49..97ace54f0fbcc 100644 --- a/mysql-test/lib/mtr_report.pm +++ b/mysql-test/lib/mtr_report.pm @@ -34,7 +34,6 @@ use mtr_match; use My::Platform; use POSIX qw[ _exit ]; use IO::Handle qw[ flush ]; -require "mtr_io.pl"; use mtr_results; my $tot_real_time= 0; @@ -92,7 +91,7 @@ sub mtr_report_test_passed ($) { my $timer_str= ""; if ( $timer and -f "$::opt_vardir/log/timer" ) { - $timer_str= mtr_fromfile("$::opt_vardir/log/timer"); + $timer_str= ::mtr_fromfile("$::opt_vardir/log/timer"); $tinfo->{timer}= $timer_str; resfile_test_info('duration', $timer_str) if $::opt_resfile; } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index f3b733a1eac99..9bfea2577c604 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -102,11 +102,11 @@ BEGIN use IO::Socket::INET; use IO::Select; -require "lib/mtr_process.pl"; -require "lib/mtr_io.pl"; -require "lib/mtr_gcov.pl"; -require "lib/mtr_gprof.pl"; -require "lib/mtr_misc.pl"; +require "mtr_process.pl"; +require "mtr_io.pl"; +require "mtr_gcov.pl"; +require "mtr_gprof.pl"; +require "mtr_misc.pl"; $SIG{INT}= sub { mtr_error("Got ^C signal"); }; $SIG{HUP}= sub { mtr_error("Hangup detected on controlling terminal"); }; From af3dc4825b09bf3de3aa092b480fff6514d1e0f8 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sat, 10 Sep 2016 20:42:20 +0200 Subject: [PATCH 035/295] Attempt to fix strange rpm dependency issue following prior patch --- cmake/cpack_rpm.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index fa7c563e57db2..d684761187ff3 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -220,6 +220,9 @@ SETA(CPACK_RPM_test_PACKAGE_PROVIDES "perl(mtr_io.pl)" "perl(mtr_match)" "perl(mtr_misc.pl)" + "perl(mtr_gcov.pl)" + "perl(mtr_gprof.pl)" + "perl(mtr_process.pl)" "perl(mtr_report)" "perl(mtr_results)" "perl(mtr_unique)") From b34d7fba31c4b18f12d400c247a09bce0ca635be Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sun, 11 Sep 2016 11:18:27 +0200 Subject: [PATCH 036/295] Debian bug#837369 - test failures on hppa ENOTEMPTY is 247 on hppa, not 39 like on Linux, according to this report: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=837369 So add another replacement for this to rpl.rpl_drop_db and binlog.binlog_databasae tests (there were already a couple similar replacements for other platforms). Signed-off-by: Kristian Nielsen --- mysql-test/extra/binlog_tests/database.test | 2 +- mysql-test/suite/rpl/t/rpl_drop_db.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/extra/binlog_tests/database.test b/mysql-test/extra/binlog_tests/database.test index 82e8b3963570d..f111a028642e4 100644 --- a/mysql-test/extra/binlog_tests/database.test +++ b/mysql-test/extra/binlog_tests/database.test @@ -52,7 +52,7 @@ eval SELECT 'hello' INTO OUTFILE 'fake_file.$prefix'; # Use '/' instead of '\' in the error message. On windows platform, dir is # formed with '\'. ---replace_regex /\\testing_1\\*/\/testing_1\// /66/39/ /17/39/ /File exists/Directory not empty/ +--replace_regex /\\testing_1\\*/\/testing_1\// /66/39/ /17/39/ /247/39/ /File exists/Directory not empty/ --error 1010 DROP DATABASE testing_1; let $wait_binlog_event= DROP TABLE IF EXIST; diff --git a/mysql-test/suite/rpl/t/rpl_drop_db.test b/mysql-test/suite/rpl/t/rpl_drop_db.test index a67850a66ddbf..dae1651dc93ae 100644 --- a/mysql-test/suite/rpl/t/rpl_drop_db.test +++ b/mysql-test/suite/rpl/t/rpl_drop_db.test @@ -13,7 +13,7 @@ insert into mysqltest1.t1 values (1); select * from mysqltest1.t1 into outfile 'mysqltest1/f1.txt'; create table mysqltest1.t2 (n int); create table mysqltest1.t3 (n int); ---replace_result \\ / 66 39 17 39 "File exists" "Directory not empty" +--replace_result \\ / 66 39 17 39 247 39 "File exists" "Directory not empty" --error 1010 drop database mysqltest1; use mysqltest1; @@ -30,7 +30,7 @@ while ($1) } --enable_query_log ---replace_result \\ / 66 39 17 39 "File exists" "Directory not empty" +--replace_result \\ / 66 39 17 39 247 39 "File exists" "Directory not empty" --error 1010 drop database mysqltest1; use mysqltest1; From a2290919533df16afdfdbd0679f80734b5a36109 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 11 Sep 2016 20:52:00 +0200 Subject: [PATCH 037/295] potential signedness issue different fix for 07a33cdcef: Bug #23296299 : HANDLE_FATAL_SIGNAL (SIG=11) IN MY_TOSORT_UTF32 --- mysql-test/r/ctype_utf32.result | 3 +++ mysql-test/t/ctype_utf32.test | 5 +++++ strings/ctype-ucs2.c | 14 +++++++------- strings/ctype-utf8.c | 8 ++++---- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result index 1f316b7b68fe8..ae55f2c101ed9 100644 --- a/mysql-test/r/ctype_utf32.result +++ b/mysql-test/r/ctype_utf32.result @@ -1269,6 +1269,9 @@ CHAR_LENGTH(TRIM(BOTH 0x61 FROM _utf32 0x00000061)) SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061)); CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061)) 1 +select hex(lower(cast(0xffff0000 as char character set utf32))) as c; +c +FFFF0000 # # End of 5.5 tests # diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test index 1be8925873c41..8cbb8e2e55e19 100644 --- a/mysql-test/t/ctype_utf32.test +++ b/mysql-test/t/ctype_utf32.test @@ -876,6 +876,11 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _utf32 0x00000061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _utf32 0x00000061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061)); +# +# potential signedness issue +# +select hex(lower(cast(0xffff0000 as char character set utf32))) as c; + --echo # --echo # End of 5.5 tests --echo # diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c index a79f5899ec542..ca6f53f3f8d7c 100644 --- a/strings/ctype-ucs2.c +++ b/strings/ctype-ucs2.c @@ -1,5 +1,5 @@ /* Copyright (c) 2003, 2013, Oracle and/or its affiliates - Copyright (c) 2009, 2014, SkySQL Ab. + Copyright (c) 2009, 2016, MariaDB This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -1098,7 +1098,7 @@ my_uni_utf16(CHARSET_INFO *cs __attribute__((unused)), static inline void my_tolower_utf16(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256 && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].tolower; } @@ -1107,7 +1107,7 @@ my_tolower_utf16(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) static inline void my_toupper_utf16(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256 && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].toupper; } @@ -1116,7 +1116,7 @@ my_toupper_utf16(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) static inline void my_tosort_utf16(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256) { if (uni_plane[page]) @@ -1727,7 +1727,7 @@ my_uni_utf32(CHARSET_INFO *cs __attribute__((unused)), static inline void my_tolower_utf32(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256 && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].tolower; } @@ -1736,7 +1736,7 @@ my_tolower_utf32(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) static inline void my_toupper_utf32(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256 && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].toupper; } @@ -1745,7 +1745,7 @@ my_toupper_utf32(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) static inline void my_tosort_utf32(MY_UNICASE_INFO *const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256) { if (uni_plane[page]) diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c index 2dd7f5e6b921e..f2782657bea70 100644 --- a/strings/ctype-utf8.c +++ b/strings/ctype-utf8.c @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2009, 2013, Monty Program Ab + Copyright (c) 2009, 2016, MariaDB This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -1939,7 +1939,7 @@ MY_UNICASE_INFO *const my_unicase_turkish[256]= static inline void my_tosort_unicode(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256) { if (uni_plane[page]) @@ -5024,7 +5024,7 @@ my_wc_mb_utf8mb4_no_range(CHARSET_INFO *cs __attribute__((unused)), static inline void my_tolower_utf8mb4(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256 && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].tolower; } @@ -5033,7 +5033,7 @@ my_tolower_utf8mb4(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) static inline void my_toupper_utf8mb4(MY_UNICASE_INFO * const* uni_plane, my_wc_t *wc) { - int page= *wc >> 8; + uint page= *wc >> 8; if (page < 256 && uni_plane[page]) *wc= uni_plane[page][*wc & 0xFF].toupper; } From 611dc0dcf4b39c670daf229f10e43b8b33f6e8c3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 11 Sep 2016 20:53:16 +0200 Subject: [PATCH 038/295] missing element in prelocked_mode_name[] array different fix for a63a250d40: BUG#23509275 :DBUG_PRINT in THD::decide_logging_format prints incorrectly, access out-of-bound --- sql/sql_class.cc | 2 ++ sql/sql_class.h | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 05a8ee8091c47..62339b2690a62 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4736,9 +4736,11 @@ int THD::decide_logging_format(TABLE_LIST *tables) { static const char *prelocked_mode_name[] = { "NON_PRELOCKED", + "LOCK_TABLES", "PRELOCKED", "PRELOCKED_UNDER_LOCK_TABLES", }; + compile_time_assert(array_elements(prelocked_mode_name) == LTM_always_last); DBUG_PRINT("debug", ("prelocked_mode: %s", prelocked_mode_name[locked_tables_mode])); } diff --git a/sql/sql_class.h b/sql/sql_class.h index da83382d5e93b..27bc40e3761ec 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1075,7 +1075,8 @@ enum enum_locked_tables_mode LTM_NONE= 0, LTM_LOCK_TABLES, LTM_PRELOCKED, - LTM_PRELOCKED_UNDER_LOCK_TABLES + LTM_PRELOCKED_UNDER_LOCK_TABLES, + LTM_always_last }; From 347eeefbfc658c8531878218487d729f4e020805 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 11 Sep 2016 20:55:11 +0200 Subject: [PATCH 039/295] don't use my_copystat in the server it was supposed to be used in command-line tools only. Different fix for 4e5473862e: Bug#24388746: PRIVILEGE ESCALATION AND RACE CONDITION USING CREATE TABLE --- include/my_sys.h | 4 ++-- mysys/my_redel.c | 7 ++++--- mysys/my_static.c | 1 + sql/mysqld.cc | 1 + 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 7e37fe598bd3e..001769a0b76fc 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2013, Monty Program Ab. + Copyright (c) 2010, 2016, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -246,7 +246,7 @@ extern my_bool my_use_symdir; extern ulong my_default_record_cache_size; extern my_bool my_disable_locking, my_disable_async_io, my_disable_flush_key_blocks, my_disable_symlinks; -extern my_bool my_disable_sync; +extern my_bool my_disable_sync, my_disable_copystat_in_redel; extern char wild_many,wild_one,wild_prefix; extern const char *charsets_dir; /* from default.c */ diff --git a/mysys/my_redel.c b/mysys/my_redel.c index b285bb25e2e88..e5e4f48d9d5ec 100644 --- a/mysys/my_redel.c +++ b/mysys/my_redel.c @@ -1,5 +1,5 @@ -/* - Copyright (c) 2000, 2010, Oracle and/or its affiliates +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates + Copyright (c) 2009, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -49,7 +49,8 @@ int my_redel(const char *org_name, const char *tmp_name, DBUG_PRINT("my",("org_name: '%s' tmp_name: '%s' MyFlags: %d", org_name,tmp_name,MyFlags)); - if (my_copystat(org_name,tmp_name,MyFlags) < 0) + if (!my_disable_copystat_in_redel && + my_copystat(org_name,tmp_name,MyFlags) < 0) goto end; if (MyFlags & MY_REDEL_MAKE_BACKUP) { diff --git a/mysys/my_static.c b/mysys/my_static.c index fdc01b1248b22..48b1e5b8dd9a5 100644 --- a/mysys/my_static.c +++ b/mysys/my_static.c @@ -99,6 +99,7 @@ my_bool my_disable_sync=0; my_bool my_disable_async_io=0; my_bool my_disable_flush_key_blocks=0; my_bool my_disable_symlinks=0; +my_bool my_disable_copystat_in_redel=0; /* Note that PSI_hook and PSI_server are unconditionally diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9b8f964629d59..be9e21d6746d8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3455,6 +3455,7 @@ static int init_common_variables() max_system_variables.pseudo_thread_id= (ulong)~0; server_start_time= flush_status_time= my_time(0); + my_disable_copystat_in_redel= 1; rpl_filter= new Rpl_filter; binlog_filter= new Rpl_filter; From 0da39caceea7733a94d898427d63ba2670160af4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 12 Sep 2016 16:18:07 +0200 Subject: [PATCH 040/295] fix BIGINT+MEDIUMINT type aggregation --- mysql-test/r/type_uint.result | 19 +++++++++++++++++++ mysql-test/t/type_uint.test | 8 ++++++++ sql/field.cc | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/type_uint.result b/mysql-test/r/type_uint.result index e08605fb237e0..d67c735f067a8 100644 --- a/mysql-test/r/type_uint.result +++ b/mysql-test/r/type_uint.result @@ -14,3 +14,22 @@ this 0 4294967295 drop table t1; +create table t1 (a bigint unsigned, b mediumint unsigned); +insert t1 values (1,2),(0xffffffffffffffff,0xffffff); +select coalesce(a,b), coalesce(b,a) from t1; +coalesce(a,b) coalesce(b,a) +1 2 +18446744073709551615 16777215 +create table t2 as select a from t1 union select b from t1; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` bigint(20) unsigned DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +select * from t2; +a +1 +18446744073709551615 +2 +16777215 +drop table t1, t2; diff --git a/mysql-test/t/type_uint.test b/mysql-test/t/type_uint.test index a9212183cb61c..14e5e3bd6213c 100644 --- a/mysql-test/t/type_uint.test +++ b/mysql-test/t/type_uint.test @@ -15,3 +15,11 @@ select * from t1; drop table t1; # End of 4.1 tests + +create table t1 (a bigint unsigned, b mediumint unsigned); +insert t1 values (1,2),(0xffffffffffffffff,0xffffff); +select coalesce(a,b), coalesce(b,a) from t1; +create table t2 as select a from t1 union select b from t1; +show create table t2; +select * from t2; +drop table t1, t2; diff --git a/sql/field.cc b/sql/field.cc index a0686fb2f1974..878e3d305af9f 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -359,7 +359,7 @@ static enum_field_types field_types_merge_rules [FIELDTYPE_NUM][FIELDTYPE_NUM]= //MYSQL_TYPE_NULL MYSQL_TYPE_TIMESTAMP MYSQL_TYPE_LONGLONG, MYSQL_TYPE_VARCHAR, //MYSQL_TYPE_LONGLONG MYSQL_TYPE_INT24 - MYSQL_TYPE_LONGLONG, MYSQL_TYPE_LONG, + MYSQL_TYPE_LONGLONG, MYSQL_TYPE_LONGLONG, //MYSQL_TYPE_DATE MYSQL_TYPE_TIME MYSQL_TYPE_VARCHAR, MYSQL_TYPE_VARCHAR, //MYSQL_TYPE_DATETIME MYSQL_TYPE_YEAR From 6e02d426d5f3970f4a92e2501f410468faa6ef2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 13 Sep 2016 13:16:11 +0200 Subject: [PATCH 041/295] Fix compilation failure of TokuDB on BSD-like systems mincore is defined differently in BSD mincore(void *, size_t, char *) vs linux variant of: mincore(void *, size_t, unsigned char *). Account for this difference in TokuDB. --- storage/tokudb/PerconaFT/portability/huge_page_detection.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/tokudb/PerconaFT/portability/huge_page_detection.cc b/storage/tokudb/PerconaFT/portability/huge_page_detection.cc index bc48e93937da4..8e73c56a6c549 100644 --- a/storage/tokudb/PerconaFT/portability/huge_page_detection.cc +++ b/storage/tokudb/PerconaFT/portability/huge_page_detection.cc @@ -90,7 +90,13 @@ static bool check_huge_pages_in_practice(void) const long pagesize = 4096; const long n_pages = TWO_MB/pagesize; +#ifdef __linux__ + // On linux mincore is defined as mincore(void *, size_t, unsigned char *) unsigned char vec[n_pages]; +#else + // On BSD (OS X included) it is defined as mincore(void *, size_t, char *) + char vec[n_pages]; +#endif { int r = mincore(second, TWO_MB, vec); if (r!=0 && errno==ENOMEM) { From b3f7a8019dae01ed03353856f62543248e6f9cd9 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 13 Sep 2016 11:12:54 -0400 Subject: [PATCH 042/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index acabf9b42d0ca..d44c8b2800612 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=52 +MYSQL_VERSION_PATCH=53 MYSQL_VERSION_EXTRA= From 7d596c9ff526bc912769490023c44e9a5b2fa743 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 16 Sep 2016 22:14:14 +0200 Subject: [PATCH 043/295] - Working on MDEV-10525. Lrecl mismatch on DBF files modified: storage/connect/filamdbf.cpp modified: storage/connect/filamdbf.h modified: storage/connect/reldef.cpp --- storage/connect/filamdbf.cpp | 86 ++++++++++++++++++++++++++++-------- storage/connect/filamdbf.h | 2 +- storage/connect/reldef.cpp | 6 ++- 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/storage/connect/filamdbf.cpp b/storage/connect/filamdbf.cpp index 8afda72357809..a4557facbd869 100644 --- a/storage/connect/filamdbf.cpp +++ b/storage/connect/filamdbf.cpp @@ -383,7 +383,7 @@ DBFBASE::DBFBASE(DBFBASE *txfp) /* and header length. Set Records, check that Reclen is equal to lrecl and */ /* return the header length or 0 in case of error. */ /****************************************************************************/ -int DBFBASE::ScanHeader(PGLOBAL g, PSZ fname, int lrecl, char *defpath) +int DBFBASE::ScanHeader(PGLOBAL g, PSZ fn, int lrecl, int *rln, char *defpath) { int rc; char filename[_MAX_PATH]; @@ -393,7 +393,7 @@ int DBFBASE::ScanHeader(PGLOBAL g, PSZ fname, int lrecl, char *defpath) /************************************************************************/ /* Open the input file. */ /************************************************************************/ - PlugSetPath(filename, fname, defpath); + PlugSetPath(filename, fn, defpath); if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "rb"))) return 0; // Assume file does not exist @@ -410,11 +410,7 @@ int DBFBASE::ScanHeader(PGLOBAL g, PSZ fname, int lrecl, char *defpath) } else if (rc == RC_FX) return -1; - if ((int)header.Reclen() != lrecl) { - sprintf(g->Message, MSG(BAD_LRECL), lrecl, header.Reclen()); - return -1; - } // endif Lrecl - + *rln = (int)header.Reclen(); Records = (int)header.Records(); return (int)header.Headlen(); } // end of ScanHeader @@ -431,9 +427,27 @@ int DBFFAM::Cardinality(PGLOBAL g) if (!g) return 1; - if (!Headlen) - if ((Headlen = ScanHeader(g, To_File, Lrecl, Tdbp->GetPath())) < 0) - return -1; // Error in ScanHeader + if (!Headlen) { + int rln = 0; // Record length in the file header + + Headlen = ScanHeader(g, To_File, Lrecl, &rln, Tdbp->GetPath()); + + if (Headlen < 0) + return -1; // Error in ScanHeader + + if (rln && Lrecl != rln) { + // This happens always on some Linux platforms + sprintf(g->Message, MSG(BAD_LRECL), Lrecl, rln); + + if (Accept) { + Lrecl = rln; + PushWarning(g, Tdbp); + } else + return -1; + + } // endif rln + + } // endif Headlen // Set number of blocks for later use Block = (Records > 0) ? (Records + Nrec - 1) / Nrec : 0; @@ -565,7 +579,13 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g) if (Lrecl != reclen) { sprintf(g->Message, MSG(BAD_LRECL), Lrecl, reclen); - return true; + + if (Accept) { + Lrecl = reclen; + PushWarning(g, Tdbp); + } else + return true; + } // endif Lrecl hlen = HEADLEN * (n + 1) + 2; @@ -641,8 +661,14 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g) if ((rc = dbfhead(g, Stream, Tdbp->GetFile(g), &header)) == RC_OK) { if (Lrecl != (int)header.Reclen()) { sprintf(g->Message, MSG(BAD_LRECL), Lrecl, header.Reclen()); - return true; - } // endif Lrecl + + if (Accept) { + Lrecl = header.Reclen(); + PushWarning(g, Tdbp); + } else + return true; + + } // endif Lrecl Records = (int)header.Records(); Headlen = (int)header.Headlen(); @@ -916,9 +942,27 @@ int DBMFAM::Cardinality(PGLOBAL g) if (!g) return 1; - if (!Headlen) - if ((Headlen = ScanHeader(g, To_File, Lrecl, Tdbp->GetPath())) < 0) - return -1; // Error in ScanHeader + if (!Headlen) { + int rln = 0; // Record length in the file header + + Headlen = ScanHeader(g, To_File, Lrecl, &rln, Tdbp->GetPath()); + + if (Headlen < 0) + return -1; // Error in ScanHeader + + if (rln && Lrecl != rln) { + // This happens always on some Linux platforms + sprintf(g->Message, MSG(BAD_LRECL), Lrecl, rln); + + if (Accept) { + Lrecl = rln; + PushWarning(g, Tdbp); + } else + return -1; + + } // endif rln + + } // endif Headlen // Set number of blocks for later use Block = (Records > 0) ? (Records + Nrec - 1) / Nrec : 0; @@ -961,8 +1005,14 @@ bool DBMFAM::AllocateBuffer(PGLOBAL g) if (Lrecl != (int)hp->Reclen()) { sprintf(g->Message, MSG(BAD_LRECL), Lrecl, hp->Reclen()); - return true; - } // endif Lrecl + + if (Accept) { + Lrecl = hp->Reclen(); + PushWarning(g, Tdbp); + } else + return true; + + } // endif Lrecl Records = (int)hp->Records(); Headlen = (int)hp->Headlen(); diff --git a/storage/connect/filamdbf.h b/storage/connect/filamdbf.h index da84d7685a8dd..66458a10eaad9 100644 --- a/storage/connect/filamdbf.h +++ b/storage/connect/filamdbf.h @@ -31,7 +31,7 @@ class DllExport DBFBASE { DBFBASE(PDBF txfp); // Implementation - int ScanHeader(PGLOBAL g, PSZ fname, int lrecl, char *defpath); + int ScanHeader(PGLOBAL g, PSZ fname, int lrecl, int *rlen, char *defpath); protected: // Default constructor, not to be used diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 8ad6e203d519c..ac2327212e0fa 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -440,7 +440,11 @@ int TABDEF::GetColCatInfo(PGLOBAL g) } // endswitch tc // lrecl must be at least recln to avoid buffer overflow - recln= MY_MAX(recln, Hc->GetIntegerOption("Lrecl")); + if (trace) + htrc("Lrecl: Calculated=%d defined=%d\n", + recln, Hc->GetIntegerOption("Lrecl")); + + recln = MY_MAX(recln, Hc->GetIntegerOption("Lrecl")); Hc->SetIntegerOption("Lrecl", recln); ((PDOSDEF)this)->SetLrecl(recln); } // endif Lrecl From fd0c114c5dbd506b7bb795dd3674b942e90e7458 Mon Sep 17 00:00:00 2001 From: iangilfillan Date: Mon, 12 Sep 2016 14:57:32 +0200 Subject: [PATCH 044/295] Update contributors --- CREDITS | 1 + mysql-test/r/contributors.result | 1 + sql/contributors.h | 1 + 3 files changed, 3 insertions(+) diff --git a/CREDITS b/CREDITS index f0e6de7f08fae..35ab4d48a8f79 100644 --- a/CREDITS +++ b/CREDITS @@ -10,6 +10,7 @@ Visma http://visma.com (2015 - 2016) Acronis http://acronis.com (2016) Nexedi https://www.nexedi.com (2016) Automattic https://automattic.com (2014 - 2016) +Tencent Game DBA http://tencentdba.com/about (2016) Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2016) Virtuozzo https://virtuozzo.com (2016) diff --git a/mysql-test/r/contributors.result b/mysql-test/r/contributors.result index 918ceaa496fe1..f3f5e227d3a98 100644 --- a/mysql-test/r/contributors.result +++ b/mysql-test/r/contributors.result @@ -9,6 +9,7 @@ Acronis http://www.acronis.com Silver Sponsor of the MariaDB Foundation Auttomattic https://automattic.com Bronze Sponsor of the MariaDB Foundation Verkkokauppa.com https://virtuozzo.com Bronze Sponsor of the MariaDB Foundation Virtuozzo https://virtuozzo.com/ Bronze Sponsor of the MariaDB Foundation +Tencent Game DBA http://tencentdba.com/about/ Bronze Sponsor of the MariaDB Foundation Google USA Sponsoring encryption, parallel replication and GTID Facebook USA Sponsoring non-blocking API, LIMIT ROWS EXAMINED etc Ronald Bradford Brisbane, Australia EFF contribution for UC2006 Auction diff --git a/sql/contributors.h b/sql/contributors.h index f52d3243453ae..0359ec5402289 100644 --- a/sql/contributors.h +++ b/sql/contributors.h @@ -46,6 +46,7 @@ struct show_table_contributors_st show_table_contributors[]= { {"Auttomattic", "https://automattic.com", "Bronze Sponsor of the MariaDB Foundation"}, {"Verkkokauppa.com", "https://virtuozzo.com", "Bronze Sponsor of the MariaDB Foundation"}, {"Virtuozzo", "https://virtuozzo.com/", "Bronze Sponsor of the MariaDB Foundation"}, + {"Tencent Game DBA", "http://tencentdba.com/about/", "Bronze Sponsor of the MariaDB Foundation"}, /* Sponsors of important features */ {"Google", "USA", "Sponsoring encryption, parallel replication and GTID"}, From 83d5b963bd38e327a673c5d4f4d70c8223f45dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Mon, 19 Sep 2016 17:15:18 +0200 Subject: [PATCH 045/295] Fix tokudb jemalloc linking Linking tokudb with jemalloc privately causes problems on library load/unload. To prevent dangling destructor pointers, link with the same library as the server is using. --- storage/tokudb/PerconaFT/portability/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/tokudb/PerconaFT/portability/CMakeLists.txt b/storage/tokudb/PerconaFT/portability/CMakeLists.txt index 9f84d9b03df2a..4793db63cc1e8 100644 --- a/storage/tokudb/PerconaFT/portability/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/portability/CMakeLists.txt @@ -14,12 +14,11 @@ set(tokuportability_srcs ) add_library(${LIBTOKUPORTABILITY} SHARED ${tokuportability_srcs}) -target_link_libraries(${LIBTOKUPORTABILITY} LINK_PRIVATE ${LIBJEMALLOC}) target_link_libraries(${LIBTOKUPORTABILITY} LINK_PUBLIC ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS}) add_library(tokuportability_static_conv STATIC ${tokuportability_srcs}) set_target_properties(tokuportability_static_conv PROPERTIES POSITION_INDEPENDENT_CODE ON) -set(tokuportability_source_libs tokuportability_static_conv ${LIBJEMALLOC} ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS}) +set(tokuportability_source_libs tokuportability_static_conv ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS}) toku_merge_static_libs(${LIBTOKUPORTABILITY}_static ${LIBTOKUPORTABILITY}_static "${tokuportability_source_libs}") maybe_add_gcov_to_libraries(${LIBTOKUPORTABILITY} tokuportability_static_conv) From 6eca463cae8e2e61469fcefa227acd4ae7b771c8 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 20 Sep 2016 15:17:57 -0400 Subject: [PATCH 046/295] Fix typo in valgrind.supp --- mysql-test/valgrind.supp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index b2b264e17fa8f..8b9c671d19d48 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1310,7 +1310,7 @@ } { -g codership/mysql-wsrep/issues#176 + codership/mysql-wsrep/issues#176 Memcheck:Leak fun:_Z16wsrep_set_paramsRN6galera10ReplicatorEPKc } From 8b51bacfd63ffcffa06e2815dd5ee72a45b5de79 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 20 Sep 2016 21:32:53 -0400 Subject: [PATCH 047/295] MDEV-10735: Valgrind warnings around Galera SST While copying the received state Id (uuid:seqno) to an uninitialized buffer, it was not properly null-terminated. --- sql/wsrep_sst.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index d88263d75cbb3..7c36643547284 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -476,13 +476,11 @@ static void* sst_joiner_thread (void* a) } else { // Scan state ID first followed by wsrep_gtid_domain_id. - char uuid[512]; unsigned long int domain_id; - size_t len= pos - out + 1; - if (len > sizeof(uuid)) goto err; // safety check - memcpy(uuid, out, len); // including '\0' - err= sst_scan_uuid_seqno (uuid, &ret_uuid, &ret_seqno); + // Null-terminate the state-id. + out[pos - out]= 0; + err= sst_scan_uuid_seqno (out, &ret_uuid, &ret_seqno); if (err) { From ec7e0b7b30ecd301da5990495cdf18b39425a7c6 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 21 Sep 2016 09:13:33 +0400 Subject: [PATCH 048/295] MDEV-10556 Assertion `0' failed in virtual void Item_sum_field::set_result_field(Field*) --- mysql-test/r/func_group.result | 9 +++++++++ mysql-test/r/innodb_group.result | 13 +++++++++++++ mysql-test/t/func_group.test | 9 +++++++++ mysql-test/t/innodb_group.test | 22 ++++++++++++++++++++++ sql/item_sum.h | 1 - 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 mysql-test/r/innodb_group.result create mode 100644 mysql-test/t/innodb_group.test diff --git a/mysql-test/r/func_group.result b/mysql-test/r/func_group.result index 10d5019335289..e0e870d1573c8 100644 --- a/mysql-test/r/func_group.result +++ b/mysql-test/r/func_group.result @@ -2432,5 +2432,14 @@ c1 3 drop table t1,t2; # +# MDEV-10556 Assertion `0' failed in virtual void Item_sum_field::set_result_field(Field*) +# +CREATE TABLE t1 (i INT, KEY(i)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (10),(20),(30); +SELECT DISTINCT STDDEV(1) FROM t1 GROUP BY i ORDER BY BENCHMARK(0, BIT_XOR(i)); +STDDEV(1) +0.0000 +DROP TABLE t1; +# # End of 10.1 tests # diff --git a/mysql-test/r/innodb_group.result b/mysql-test/r/innodb_group.result new file mode 100644 index 0000000000000..58bd75e0baf09 --- /dev/null +++ b/mysql-test/r/innodb_group.result @@ -0,0 +1,13 @@ +# +# Start of 10.1 tests +# +# +# MDEV-10556 Assertion `0' failed in virtual void Item_sum_field::set_result_field(Field*) +# +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +SELECT DISTINCT STDDEV(1) FROM t1 GROUP BY i ORDER BY BENCHMARK(0, BIT_XOR(i)); +STDDEV(1) +DROP TABLE t1; +# +# End of 10.1 tests +# diff --git a/mysql-test/t/func_group.test b/mysql-test/t/func_group.test index 7e342928ef869..1e75099a1fe92 100644 --- a/mysql-test/t/func_group.test +++ b/mysql-test/t/func_group.test @@ -1679,6 +1679,15 @@ select c1 from t1 having c1 >= (select t.c1 as c from t2 t order by (select min( select c1 from t1 having c1 >= (select t.c1 as c from t2 t order by (select min(t1.c1+tt.c1) from t2 tt)); drop table t1,t2; +--echo # +--echo # MDEV-10556 Assertion `0' failed in virtual void Item_sum_field::set_result_field(Field*) +--echo # + +CREATE TABLE t1 (i INT, KEY(i)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (10),(20),(30); +SELECT DISTINCT STDDEV(1) FROM t1 GROUP BY i ORDER BY BENCHMARK(0, BIT_XOR(i)); +DROP TABLE t1; + --echo # --echo # End of 10.1 tests --echo # diff --git a/mysql-test/t/innodb_group.test b/mysql-test/t/innodb_group.test new file mode 100644 index 0000000000000..56c8d54003e3e --- /dev/null +++ b/mysql-test/t/innodb_group.test @@ -0,0 +1,22 @@ +# +# Tests involving GROUP BY, aggregate functions and InnoDB +# + + +--source include/have_innodb.inc + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-10556 Assertion `0' failed in virtual void Item_sum_field::set_result_field(Field*) +--echo # + +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +SELECT DISTINCT STDDEV(1) FROM t1 GROUP BY i ORDER BY BENCHMARK(0, BIT_XOR(i)); +DROP TABLE t1; + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/item_sum.h b/sql/item_sum.h index 8568eaae907f7..7ad037c046bd7 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -1084,7 +1084,6 @@ class Item_sum_field :public Item fixed= true; } table_map used_tables() const { return (table_map) 1L; } - void set_result_field(Field *) { DBUG_ASSERT(0); } void save_in_result_field(bool no_conversions) { DBUG_ASSERT(0); } }; From 7e4eb990adb71920cc10393194d7317ba07e3f91 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 22 Sep 2016 07:00:10 +0400 Subject: [PATCH 049/295] MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BASE64('TEST')) Problem N1: MDEV-10425 Item_func_{md5|sha|sha2}::fix_length_and_dec() changed args[0]->collation to force binary comparison in args[0]->eq(). It was done to treat e.g. MD5('a') and MD5('A') as different values. It is wrong for a Item_func_xxx to modify its arguments. Item_func_conv_charset did not expect that and crashed on assert. Problem N2: MDEV-10850 Item_func_to_base64, Item_func_password, Item_func_hex are also case sensitive hash functions, but they did not compare their arguments as binary. Solution: - Removing the code changing args[0]->collation - Introducing Item_str_ascii_checksum_func as a common parent for Item_func_{md5|sha|sha2|password|hex|to_base64} and overriding its eq() method to compare arguments binary. --- .../include/func_str_ascii_checksum.inc | 24 +++++++ mysql-test/r/func_crypt.result | 62 +++++++++++++++++++ mysql-test/r/func_digest.result | 32 ++++++++++ mysql-test/r/func_str.result | 56 +++++++++++++++++ mysql-test/t/func_crypt.test | 14 +++++ mysql-test/t/func_digest.test | 26 ++++++++ mysql-test/t/func_str.test | 6 ++ sql/item_strfunc.cc | 30 --------- sql/item_strfunc.h | 51 ++++++++++----- 9 files changed, 257 insertions(+), 44 deletions(-) create mode 100644 mysql-test/include/func_str_ascii_checksum.inc diff --git a/mysql-test/include/func_str_ascii_checksum.inc b/mysql-test/include/func_str_ascii_checksum.inc new file mode 100644 index 0000000000000..8e51c92c1ac93 --- /dev/null +++ b/mysql-test/include/func_str_ascii_checksum.inc @@ -0,0 +1,24 @@ +--echo # Start of func_str_ascii_checksum.inc + +--echo # +--echo # MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BASE64('TEST')) +--echo # + +--eval CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(255), UNIQUE KEY k1 (f1,f2)) +--eval INSERT INTO t1 VALUES ('test',$func('test')), ('TEST', $func('TEST')) +--eval SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= $func("test") OR f2= $func("TEST")) +--eval SELECT * FROM t1 WHERE f1='test' AND (f2= $func("test") OR f2= $func("TEST")) +--eval SELECT * FROM t1 WHERE f1='test' AND (f2= $func("TEST") OR f2= $func("test")) +DROP TABLE t1; + + +--echo # +--echo # MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +--echo # + +--eval PREPARE stmt FROM "SELECT $func(CONVERT('foo' USING latin1))" +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + + +--echo # End of func_str_ascii_checksum.inc diff --git a/mysql-test/r/func_crypt.result b/mysql-test/r/func_crypt.result index 1eda56ac1143c..c3d7bf67859c9 100644 --- a/mysql-test/r/func_crypt.result +++ b/mysql-test/r/func_crypt.result @@ -106,3 +106,65 @@ OLD_PASSWORD(c1) PASSWORD(c1) 77023ffe214c04ff *82E58A2C08AAFE72C8EB523069CD8ADB33F78F58 DROP TABLE t1; End of 5.0 tests +# +# Start of 10.1 tests +# +# Start of func_str_ascii_checksum.inc +# +# MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BASE64('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(255), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',password('test')), ('TEST', password('TEST')); +SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= password("test") OR f2= password("TEST")); +f1 f2 +test *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 +TEST *47A6B0EA08A36FAEBE4305B373FE37E3CF27C357 +SELECT * FROM t1 WHERE f1='test' AND (f2= password("test") OR f2= password("TEST")); +f1 f2 +TEST *47A6B0EA08A36FAEBE4305B373FE37E3CF27C357 +test *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 +SELECT * FROM t1 WHERE f1='test' AND (f2= password("TEST") OR f2= password("test")); +f1 f2 +TEST *47A6B0EA08A36FAEBE4305B373FE37E3CF27C357 +test *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 +DROP TABLE t1; +# +# MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +# +PREPARE stmt FROM "SELECT password(CONVERT('foo' USING latin1))"; +EXECUTE stmt; +password(CONVERT('foo' USING latin1)) +*F3A2A51A9B0F2BE2468926B4132313728C250DBF +DEALLOCATE PREPARE stmt; +# End of func_str_ascii_checksum.inc +# Start of func_str_ascii_checksum.inc +# +# MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BASE64('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(255), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',old_password('test')), ('TEST', old_password('TEST')); +SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= old_password("test") OR f2= old_password("TEST")); +f1 f2 +test 378b243e220ca493 +TEST 06df397e084be793 +SELECT * FROM t1 WHERE f1='test' AND (f2= old_password("test") OR f2= old_password("TEST")); +f1 f2 +TEST 06df397e084be793 +test 378b243e220ca493 +SELECT * FROM t1 WHERE f1='test' AND (f2= old_password("TEST") OR f2= old_password("test")); +f1 f2 +TEST 06df397e084be793 +test 378b243e220ca493 +DROP TABLE t1; +# +# MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +# +PREPARE stmt FROM "SELECT old_password(CONVERT('foo' USING latin1))"; +EXECUTE stmt; +old_password(CONVERT('foo' USING latin1)) +7c786c222596437b +DEALLOCATE PREPARE stmt; +# End of func_str_ascii_checksum.inc +# +# End of 10.1 tests +# diff --git a/mysql-test/r/func_digest.result b/mysql-test/r/func_digest.result index 095b69363cec7..6821c84d8d8e8 100644 --- a/mysql-test/r/func_digest.result +++ b/mysql-test/r/func_digest.result @@ -1426,3 +1426,35 @@ Catalog Database Table Table_alias Column Column_alias Type Length Max length Is def sha2('1',224) 253 56 56 Y 0 31 8 sha2('1',224) e25388fde8290dc286a6164fa2d97e551b53498dcbf7bc378eb1f178 +# +# Start of 10.1 tests +# +# +# MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BAS E64('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(64), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',SHA2('test',224)), ('TEST', SHA2('TEST',224)); +SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= SHA2("test",224) OR f2= SHA2("TEST",224)); +f1 f2 +test 90a3ed9e32b2aaf4c61c410eb925426119e1a9dc53d4286ade99a809 +TEST 917ecca24f3e6ceaf52375d8083381f1f80a21e6e49fbadc40afeb8e +SELECT * FROM t1 WHERE f1='test' AND (f2= SHA2("test",224) OR f2= SHA2("TEST",224)); +f1 f2 +test 90a3ed9e32b2aaf4c61c410eb925426119e1a9dc53d4286ade99a809 +TEST 917ecca24f3e6ceaf52375d8083381f1f80a21e6e49fbadc40afeb8e +SELECT * FROM t1 WHERE f1='test' AND (f2= SHA2("TEST",224) OR f2= SHA2("test",224)); +f1 f2 +test 90a3ed9e32b2aaf4c61c410eb925426119e1a9dc53d4286ade99a809 +TEST 917ecca24f3e6ceaf52375d8083381f1f80a21e6e49fbadc40afeb8e +DROP TABLE t1; +# +# MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +# +PREPARE stmt FROM "SELECT SHA2(CONVERT('foo' USING latin1), 224)"; +EXECUTE stmt; +SHA2(CONVERT('foo' USING latin1), 224) +0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db +DEALLOCATE PREPARE stmt; +# +# End of 10.1 tests +# diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result index f196571218f6f..04c6a2bfc0abe 100644 --- a/mysql-test/r/func_str.result +++ b/mysql-test/r/func_str.result @@ -4581,6 +4581,62 @@ SELECT COUNT(*) FROM t1, t1 t2 GROUP BY INSERT('', t2.a, t1.a, @@global.max_binl COUNT(*) 25 DROP TABLE t1; +# Start of func_str_ascii_checksum.inc +# +# MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BASE64('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(255), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',hex('test')), ('TEST', hex('TEST')); +SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= hex("test") OR f2= hex("TEST")); +f1 f2 +test 74657374 +TEST 54455354 +SELECT * FROM t1 WHERE f1='test' AND (f2= hex("test") OR f2= hex("TEST")); +f1 f2 +TEST 54455354 +test 74657374 +SELECT * FROM t1 WHERE f1='test' AND (f2= hex("TEST") OR f2= hex("test")); +f1 f2 +TEST 54455354 +test 74657374 +DROP TABLE t1; +# +# MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +# +PREPARE stmt FROM "SELECT hex(CONVERT('foo' USING latin1))"; +EXECUTE stmt; +hex(CONVERT('foo' USING latin1)) +666F6F +DEALLOCATE PREPARE stmt; +# End of func_str_ascii_checksum.inc +# Start of func_str_ascii_checksum.inc +# +# MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BASE64('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(255), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',to_base64('test')), ('TEST', to_base64('TEST')); +SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= to_base64("test") OR f2= to_base64("TEST")); +f1 f2 +test dGVzdA== +TEST VEVTVA== +SELECT * FROM t1 WHERE f1='test' AND (f2= to_base64("test") OR f2= to_base64("TEST")); +f1 f2 +test dGVzdA== +TEST VEVTVA== +SELECT * FROM t1 WHERE f1='test' AND (f2= to_base64("TEST") OR f2= to_base64("test")); +f1 f2 +test dGVzdA== +TEST VEVTVA== +DROP TABLE t1; +# +# MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +# +PREPARE stmt FROM "SELECT to_base64(CONVERT('foo' USING latin1))"; +EXECUTE stmt; +to_base64(CONVERT('foo' USING latin1)) +Zm9v +DEALLOCATE PREPARE stmt; +# End of func_str_ascii_checksum.inc # # End of 10.1 tests # diff --git a/mysql-test/t/func_crypt.test b/mysql-test/t/func_crypt.test index ca6e712f45ce4..1504051d91e84 100644 --- a/mysql-test/t/func_crypt.test +++ b/mysql-test/t/func_crypt.test @@ -70,3 +70,17 @@ SELECT OLD_PASSWORD(c1), PASSWORD(c1) FROM t1; DROP TABLE t1; --echo End of 5.0 tests + + +--echo # +--echo # Start of 10.1 tests +--echo # + +--let func=password +--source include/func_str_ascii_checksum.inc +--let func=old_password +--source include/func_str_ascii_checksum.inc + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/mysql-test/t/func_digest.test b/mysql-test/t/func_digest.test index 81f19c7e091d7..39edb64fef24a 100644 --- a/mysql-test/t/func_digest.test +++ b/mysql-test/t/func_digest.test @@ -494,3 +494,29 @@ SET NAMES latin1; SELECT sha2('1',224); --disable_metadata +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-10850 Wrong result for WHERE .. (f2=TO_BASE64('test') OR f2=TO_BAS E64('TEST')) +--echo # + +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(64), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',SHA2('test',224)), ('TEST', SHA2('TEST',224)); +SELECT * FROM t1 IGNORE INDEX(k1) WHERE f1='test' AND (f2= SHA2("test",224) OR f2= SHA2("TEST",224)); +SELECT * FROM t1 WHERE f1='test' AND (f2= SHA2("test",224) OR f2= SHA2("TEST",224)); +SELECT * FROM t1 WHERE f1='test' AND (f2= SHA2("TEST",224) OR f2= SHA2("test",224)); +DROP TABLE t1; + +--echo # +--echo # MDEV-10425 Assertion `collation.derivation == DERIVATION_IMPLICIT' failed in Item_func_conv_charset::fix_length_and_dec() +--echo # + +PREPARE stmt FROM "SELECT SHA2(CONVERT('foo' USING latin1), 224)"; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test index 2645417f3e5be..98af37053f29e 100644 --- a/mysql-test/t/func_str.test +++ b/mysql-test/t/func_str.test @@ -1783,6 +1783,12 @@ INSERT INTO t1 VALUES (0),(0),(1),(0),(0); SELECT COUNT(*) FROM t1, t1 t2 GROUP BY INSERT('', t2.a, t1.a, @@global.max_binlog_size); DROP TABLE t1; +--let func=hex +--source include/func_str_ascii_checksum.inc + +--let func=to_base64 +--source include/func_str_ascii_checksum.inc + --echo # --echo # End of 10.1 tests --echo # diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 75b034cbd4faa..43770a88cdcfa 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -165,31 +165,6 @@ String *Item_func_md5::val_str_ascii(String *str) } -/* - The MD5()/SHA() functions treat their parameter as being a case sensitive. - Thus we set binary collation on it so different instances of MD5() will be - compared properly. -*/ -static CHARSET_INFO *get_checksum_charset(const char *csname) -{ - CHARSET_INFO *cs= get_charset_by_csname(csname, MY_CS_BINSORT, MYF(0)); - if (!cs) - { - // Charset has no binary collation: use my_charset_bin. - cs= &my_charset_bin; - } - return cs; -} - - -void Item_func_md5::fix_length_and_dec() -{ - CHARSET_INFO *cs= get_checksum_charset(args[0]->collation.collation->csname); - args[0]->collation.set(cs, DERIVATION_COERCIBLE); - fix_length_and_charset(32, default_charset()); -} - - String *Item_func_sha::val_str_ascii(String *str) { DBUG_ASSERT(fixed == 1); @@ -215,8 +190,6 @@ String *Item_func_sha::val_str_ascii(String *str) void Item_func_sha::fix_length_and_dec() { - CHARSET_INFO *cs= get_checksum_charset(args[0]->collation.collation->csname); - args[0]->collation.set(cs, DERIVATION_COERCIBLE); // size of hex representation of hash fix_length_and_charset(SHA1_HASH_SIZE * 2, default_charset()); } @@ -346,9 +319,6 @@ void Item_func_sha2::fix_length_and_dec() "sha2"); } - CHARSET_INFO *cs= get_checksum_charset(args[0]->collation.collation->csname); - args[0]->collation.set(cs, DERIVATION_COERCIBLE); - #else THD *thd= current_thd; push_warning_printf(thd, diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 65c286819d33a..db0509e17d735 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -96,40 +96,63 @@ class Item_str_ascii_func :public Item_str_func }; -class Item_func_md5 :public Item_str_ascii_func +/** + Functions returning a checksum or a hash of the argument. +*/ +class Item_str_ascii_checksum_func: public Item_str_ascii_func +{ +public: + Item_str_ascii_checksum_func(THD *thd, Item *a) + :Item_str_ascii_func(thd, a) { } + Item_str_ascii_checksum_func(THD *thd, Item *a, Item *b) + :Item_str_ascii_func(thd, a, b) { } + bool eq(const Item *item, bool binary_cmp) const + { + // Always use binary argument comparison: MD5('x') != MD5('X') + return Item_func::eq(item, true); + } +}; + + +class Item_func_md5 :public Item_str_ascii_checksum_func { String tmp_value; public: - Item_func_md5(THD *thd, Item *a): Item_str_ascii_func(thd, a) {} + Item_func_md5(THD *thd, Item *a): Item_str_ascii_checksum_func(thd, a) {} String *val_str_ascii(String *); - void fix_length_and_dec(); + void fix_length_and_dec() + { + fix_length_and_charset(32, default_charset()); + } const char *func_name() const { return "md5"; } }; -class Item_func_sha :public Item_str_ascii_func +class Item_func_sha :public Item_str_ascii_checksum_func { public: - Item_func_sha(THD *thd, Item *a): Item_str_ascii_func(thd, a) {} + Item_func_sha(THD *thd, Item *a): Item_str_ascii_checksum_func(thd, a) {} String *val_str_ascii(String *); void fix_length_and_dec(); const char *func_name() const { return "sha"; } }; -class Item_func_sha2 :public Item_str_ascii_func +class Item_func_sha2 :public Item_str_ascii_checksum_func { public: - Item_func_sha2(THD *thd, Item *a, Item *b): Item_str_ascii_func(thd, a, b) {} + Item_func_sha2(THD *thd, Item *a, Item *b) + :Item_str_ascii_checksum_func(thd, a, b) {} String *val_str_ascii(String *); void fix_length_and_dec(); const char *func_name() const { return "sha2"; } }; -class Item_func_to_base64 :public Item_str_ascii_func +class Item_func_to_base64 :public Item_str_ascii_checksum_func { String tmp_value; public: - Item_func_to_base64(THD *thd, Item *a): Item_str_ascii_func(thd, a) {} + Item_func_to_base64(THD *thd, Item *a) + :Item_str_ascii_checksum_func(thd, a) {} String *val_str_ascii(String *); void fix_length_and_dec(); const char *func_name() const { return "to_base64"; } @@ -431,7 +454,7 @@ class Item_func_rtrim :public Item_func_trim authentication procedure works, see comments in password.c. */ -class Item_func_password :public Item_str_ascii_func +class Item_func_password :public Item_str_ascii_checksum_func { public: enum PW_Alg {OLD, NEW}; @@ -441,9 +464,9 @@ class Item_func_password :public Item_str_ascii_func bool deflt; public: Item_func_password(THD *thd, Item *a): - Item_str_ascii_func(thd, a), alg(NEW), deflt(1) {} + Item_str_ascii_checksum_func(thd, a), alg(NEW), deflt(1) {} Item_func_password(THD *thd, Item *a, PW_Alg al): - Item_str_ascii_func(thd, a), alg(al), deflt(0) {} + Item_str_ascii_checksum_func(thd, a), alg(al), deflt(0) {} String *val_str_ascii(String *str); bool fix_fields(THD *thd, Item **ref); void fix_length_and_dec() @@ -803,12 +826,12 @@ class Item_func_conv :public Item_str_func }; -class Item_func_hex :public Item_str_ascii_func +class Item_func_hex :public Item_str_ascii_checksum_func { String tmp_value; public: Item_func_hex(THD *thd, Item *a): - Item_str_ascii_func(thd, a) {} + Item_str_ascii_checksum_func(thd, a) {} const char *func_name() const { return "hex"; } String *val_str_ascii(String *); void fix_length_and_dec() From 9f837c6e1a54ea05be92112fe7520ffe0134b260 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 22 Sep 2016 10:03:12 +0400 Subject: [PATCH 050/295] MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) --- mysql-test/r/func_compress.result | 24 ++++++++++ mysql-test/r/func_crypt.result | 18 +++++++ mysql-test/r/func_str.result | 18 +++++++ mysql-test/t/func_compress.test | 19 ++++++++ mysql-test/t/func_crypt.test | 12 +++++ mysql-test/t/func_str.test | 12 +++++ sql/item_strfunc.h | 80 +++++++++++++++++++++++-------- 7 files changed, 163 insertions(+), 20 deletions(-) diff --git a/mysql-test/r/func_compress.result b/mysql-test/r/func_compress.result index e9c8193685bb5..1f15c74ff0da6 100644 --- a/mysql-test/r/func_compress.result +++ b/mysql-test/r/func_compress.result @@ -160,3 +160,27 @@ set global max_allowed_packet=default; # # End of 5.5 tests # +# +# Start of 10.1 tests +# +# +# MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(64), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',compress('test')), ('TEST', compress('TEST')); +SELECT f1,HEX(f2) FROM t1 ignore index(k1) WHERE f1='test' AND (f2= compress("test") OR f2= compress("TEST")); +f1 HEX(f2) +test 04000000789C2B492D2E0100045D01C1 +TEST 04000000789C0B710D0E0100031D0141 +SELECT f1,HEX(f2) FROM t1 WHERE f1='test' AND (f2= compress("test") OR f2= compress("TEST")); +f1 HEX(f2) +TEST 04000000789C0B710D0E0100031D0141 +test 04000000789C2B492D2E0100045D01C1 +SELECT f1,HEX(f2) FROM t1 WHERE f1='test' AND (f2= compress("TEST") OR f2= compress("test")); +f1 HEX(f2) +TEST 04000000789C0B710D0E0100031D0141 +test 04000000789C2B492D2E0100045D01C1 +DROP TABLE t1; +# +# End of 10.1 tests +# diff --git a/mysql-test/r/func_crypt.result b/mysql-test/r/func_crypt.result index c3d7bf67859c9..a1c33f827c7fb 100644 --- a/mysql-test/r/func_crypt.result +++ b/mysql-test/r/func_crypt.result @@ -166,5 +166,23 @@ old_password(CONVERT('foo' USING latin1)) DEALLOCATE PREPARE stmt; # End of func_str_ascii_checksum.inc # +# MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(64), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',encrypt('test','key')), ('TEST', encrypt('TEST','key')); +SELECT f1 FROM t1 ignore index(k1) WHERE f1='test' AND (f2= encrypt('test','key') OR f2= encrypt('TEST','key')); +f1 +test +TEST +SELECT f1 FROM t1 WHERE f1='test' AND (f2= encrypt('test','key') OR f2= encrypt('TEST','key')); +f1 +TEST +test +SELECT f1 FROM t1 WHERE f1='test' AND (f2= encrypt('TEST','key') OR f2= encrypt('test','key')); +f1 +TEST +test +DROP TABLE t1; +# # End of 10.1 tests # diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result index 04c6a2bfc0abe..d32efe642bbc1 100644 --- a/mysql-test/r/func_str.result +++ b/mysql-test/r/func_str.result @@ -4638,5 +4638,23 @@ Zm9v DEALLOCATE PREPARE stmt; # End of func_str_ascii_checksum.inc # +# MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) +# +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(128), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('YQ==',from_base64('YQ==')), ('Yq==', from_base64('Yq==')); +SELECT f1,HEX(f2) FROM t1 ignore index(k1) WHERE f1='YQ==' AND (f2= from_base64("YQ==") OR f2= from_base64("Yq==")); +f1 HEX(f2) +YQ== 61 +Yq== 62 +SELECT f1,HEX(f2) FROM t1 WHERE f1='YQ==' AND (f2= from_base64("YQ==") OR f2= from_base64("Yq==")); +f1 HEX(f2) +YQ== 61 +Yq== 62 +SELECT f1,HEX(f2) FROM t1 WHERE f1='YQ==' AND (f2= from_base64("Yq==") OR f2= from_base64("YQ==")); +f1 HEX(f2) +YQ== 61 +Yq== 62 +DROP TABLE t1; +# # End of 10.1 tests # diff --git a/mysql-test/t/func_compress.test b/mysql-test/t/func_compress.test index 2a06769b2ef60..6305df3952dcb 100644 --- a/mysql-test/t/func_compress.test +++ b/mysql-test/t/func_compress.test @@ -152,3 +152,22 @@ set global max_allowed_packet=default; --echo # --echo # End of 5.5 tests --echo # + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) +--echo # + +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(64), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',compress('test')), ('TEST', compress('TEST')); +SELECT f1,HEX(f2) FROM t1 ignore index(k1) WHERE f1='test' AND (f2= compress("test") OR f2= compress("TEST")); +SELECT f1,HEX(f2) FROM t1 WHERE f1='test' AND (f2= compress("test") OR f2= compress("TEST")); +SELECT f1,HEX(f2) FROM t1 WHERE f1='test' AND (f2= compress("TEST") OR f2= compress("test")); +DROP TABLE t1; + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/mysql-test/t/func_crypt.test b/mysql-test/t/func_crypt.test index 1504051d91e84..dc4bf4663d56f 100644 --- a/mysql-test/t/func_crypt.test +++ b/mysql-test/t/func_crypt.test @@ -81,6 +81,18 @@ DROP TABLE t1; --let func=old_password --source include/func_str_ascii_checksum.inc +--echo # +--echo # MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) +--echo # + +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(64), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('test',encrypt('test','key')), ('TEST', encrypt('TEST','key')); +SELECT f1 FROM t1 ignore index(k1) WHERE f1='test' AND (f2= encrypt('test','key') OR f2= encrypt('TEST','key')); +SELECT f1 FROM t1 WHERE f1='test' AND (f2= encrypt('test','key') OR f2= encrypt('TEST','key')); +SELECT f1 FROM t1 WHERE f1='test' AND (f2= encrypt('TEST','key') OR f2= encrypt('test','key')); +DROP TABLE t1; + + --echo # --echo # End of 10.1 tests --echo # diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test index 98af37053f29e..48872edcd4baa 100644 --- a/mysql-test/t/func_str.test +++ b/mysql-test/t/func_str.test @@ -1789,6 +1789,18 @@ DROP TABLE t1; --let func=to_base64 --source include/func_str_ascii_checksum.inc +--echo # +--echo # MDEV-10864 Wrong result for WHERE .. (f2=COMPRESS('test') OR f2=COMPRESS('TEST')) +--echo # + +CREATE TABLE t1 (f1 VARCHAR(4), f2 VARCHAR(128), UNIQUE KEY k1 (f1,f2)); +INSERT INTO t1 VALUES ('YQ==',from_base64('YQ==')), ('Yq==', from_base64('Yq==')); +SELECT f1,HEX(f2) FROM t1 ignore index(k1) WHERE f1='YQ==' AND (f2= from_base64("YQ==") OR f2= from_base64("Yq==")); +SELECT f1,HEX(f2) FROM t1 WHERE f1='YQ==' AND (f2= from_base64("YQ==") OR f2= from_base64("Yq==")); +SELECT f1,HEX(f2) FROM t1 WHERE f1='YQ==' AND (f2= from_base64("Yq==") OR f2= from_base64("YQ==")); +DROP TABLE t1; + + --echo # --echo # End of 10.1 tests --echo # diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index db0509e17d735..a56e100a956b8 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -97,7 +97,9 @@ class Item_str_ascii_func :public Item_str_func /** - Functions returning a checksum or a hash of the argument. + Functions that return a checksum or a hash of the argument, + or somehow else encode or decode the argument, + returning an ASCII-repertoire string. */ class Item_str_ascii_checksum_func: public Item_str_ascii_func { @@ -114,6 +116,29 @@ class Item_str_ascii_checksum_func: public Item_str_ascii_func }; +/** + Functions that return a checksum or a hash of the argument, + or somehow else encode or decode the argument, + returning a binary string. +*/ +class Item_str_binary_checksum_func: public Item_str_func +{ +public: + Item_str_binary_checksum_func(THD *thd, Item *a) + :Item_str_func(thd, a) { } + Item_str_binary_checksum_func(THD *thd, Item *a, Item *b) + :Item_str_func(thd, a, b) { } + bool eq(const Item *item, bool binary_cmp) const + { + /* + Always use binary argument comparison: + FROM_BASE64('test') != FROM_BASE64('TEST') + */ + return Item_func::eq(item, true); + } +}; + + class Item_func_md5 :public Item_str_ascii_checksum_func { String tmp_value; @@ -158,11 +183,12 @@ class Item_func_to_base64 :public Item_str_ascii_checksum_func const char *func_name() const { return "to_base64"; } }; -class Item_func_from_base64 :public Item_str_func +class Item_func_from_base64 :public Item_str_binary_checksum_func { String tmp_value; public: - Item_func_from_base64(THD *thd, Item *a): Item_str_func(thd, a) {} + Item_func_from_base64(THD *thd, Item *a) + :Item_str_binary_checksum_func(thd, a) { } String *val_str(String *); void fix_length_and_dec(); const char *func_name() const { return "from_base64"; } @@ -170,7 +196,7 @@ class Item_func_from_base64 :public Item_str_func #include -class Item_aes_crypt :public Item_str_func +class Item_aes_crypt :public Item_str_binary_checksum_func { enum { AES_KEY_LENGTH = 128 }; void create_key(String *user_key, uchar* key); @@ -178,7 +204,8 @@ class Item_aes_crypt :public Item_str_func protected: int what; public: - Item_aes_crypt(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} + Item_aes_crypt(THD *thd, Item *a, Item *b) + :Item_str_binary_checksum_func(thd, a, b) {} String *val_str(String *); }; @@ -484,12 +511,14 @@ class Item_func_password :public Item_str_ascii_checksum_func -class Item_func_des_encrypt :public Item_str_func +class Item_func_des_encrypt :public Item_str_binary_checksum_func { String tmp_value,tmp_arg; public: - Item_func_des_encrypt(THD *thd, Item *a): Item_str_func(thd, a) {} - Item_func_des_encrypt(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} + Item_func_des_encrypt(THD *thd, Item *a) + :Item_str_binary_checksum_func(thd, a) {} + Item_func_des_encrypt(THD *thd, Item *a, Item *b) + :Item_str_binary_checksum_func(thd, a, b) {} String *val_str(String *); void fix_length_and_dec() { @@ -500,12 +529,14 @@ class Item_func_des_encrypt :public Item_str_func const char *func_name() const { return "des_encrypt"; } }; -class Item_func_des_decrypt :public Item_str_func +class Item_func_des_decrypt :public Item_str_binary_checksum_func { String tmp_value; public: - Item_func_des_decrypt(THD *thd, Item *a): Item_str_func(thd, a) {} - Item_func_des_decrypt(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} + Item_func_des_decrypt(THD *thd, Item *a) + :Item_str_binary_checksum_func(thd, a) {} + Item_func_des_decrypt(THD *thd, Item *a, Item *b) + :Item_str_binary_checksum_func(thd, a, b) {} String *val_str(String *); void fix_length_and_dec() { @@ -518,7 +549,13 @@ class Item_func_des_decrypt :public Item_str_func const char *func_name() const { return "des_decrypt"; } }; -class Item_func_encrypt :public Item_str_func + +/** + QQ: Item_func_encrypt should derive from Item_str_ascii_checksum_func. + However, it should be fixed to handle UCS2, UTF16, UTF32 properly first, + as the underlying crypt() call expects a null-terminated input string. +*/ +class Item_func_encrypt :public Item_str_binary_checksum_func { String tmp_value; @@ -528,11 +565,12 @@ class Item_func_encrypt :public Item_str_func collation.set(&my_charset_bin); } public: - Item_func_encrypt(THD *thd, Item *a): Item_str_func(thd, a) + Item_func_encrypt(THD *thd, Item *a): Item_str_binary_checksum_func(thd, a) { constructor_helper(); } - Item_func_encrypt(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) + Item_func_encrypt(THD *thd, Item *a, Item *b) + :Item_str_binary_checksum_func(thd, a, b) { constructor_helper(); } @@ -548,7 +586,7 @@ class Item_func_encrypt :public Item_str_func #include "sql_crypt.h" -class Item_func_encode :public Item_str_func +class Item_func_encode :public Item_str_binary_checksum_func { private: /** Whether the PRNG has already been seeded. */ @@ -557,7 +595,7 @@ class Item_func_encode :public Item_str_func SQL_CRYPT sql_crypt; public: Item_func_encode(THD *thd, Item *a, Item *seed_arg): - Item_str_func(thd, a, seed_arg) {} + Item_str_binary_checksum_func(thd, a, seed_arg) {} String *val_str(String *); void fix_length_and_dec(); const char *func_name() const { return "encode"; } @@ -1172,21 +1210,23 @@ class Item_func_uncompressed_length : public Item_int_func #define ZLIB_DEPENDED_FUNCTION { null_value=1; return 0; } #endif -class Item_func_compress: public Item_str_func +class Item_func_compress: public Item_str_binary_checksum_func { String buffer; public: - Item_func_compress(THD *thd, Item *a): Item_str_func(thd, a) {} + Item_func_compress(THD *thd, Item *a) + :Item_str_binary_checksum_func(thd, a) {} void fix_length_and_dec(){max_length= (args[0]->max_length*120)/100+12;} const char *func_name() const{return "compress";} String *val_str(String *) ZLIB_DEPENDED_FUNCTION }; -class Item_func_uncompress: public Item_str_func +class Item_func_uncompress: public Item_str_binary_checksum_func { String buffer; public: - Item_func_uncompress(THD *thd, Item *a): Item_str_func(thd, a) {} + Item_func_uncompress(THD *thd, Item *a) + :Item_str_binary_checksum_func(thd, a) {} void fix_length_and_dec(){ maybe_null= 1; max_length= MAX_BLOB_WIDTH; } const char *func_name() const{return "uncompress";} String *val_str(String *) ZLIB_DEPENDED_FUNCTION From e56a53920b0075f9a534610032ee05f2e249e3ae Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Fri, 1 Jul 2016 13:57:18 +0400 Subject: [PATCH 051/295] MDEV-10315 - Online ALTER TABLE may get stuck in tdc_remove_table There was race condition between online ALTER TABLE and statements performing TABLE_SHARE release without marking it flushed (e.g. in case of table cache overflow, SET @@global.table_open_cache, manager thread purging table cache). The reason was missing mysql_cond_broadcast(). --- sql/table_cache.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/table_cache.cc b/sql/table_cache.cc index 097f37d26d835..bdb7914c32b97 100644 --- a/sql/table_cache.cc +++ b/sql/table_cache.cc @@ -876,6 +876,8 @@ void tdc_release_share(TABLE_SHARE *share) } if (--share->tdc.ref_count) { + if (!share->is_view) + mysql_cond_broadcast(&share->tdc.COND_release); mysql_mutex_unlock(&share->tdc.LOCK_table_share); mysql_mutex_unlock(&LOCK_unused_shares); DBUG_VOID_RETURN; From e387bfafbbb01ccfabeb2beb86efb199ca2ca3ac Mon Sep 17 00:00:00 2001 From: Seamus Lee Date: Thu, 22 Sep 2016 19:14:40 +1000 Subject: [PATCH 052/295] MDEV-10830 - Fix undefined database test error when running mysql_install_db (#234) * Fix undefined database test error when running mysql_install_db When using mariaDb in docker mode it can fail as it calls mysql_install_db but as we are going through a slightly different install process the test database has not been created, therefore we should fall back to the mysql database as per https://mariadb.com/kb/en/mariadb/mariadb-10112-mysql_install_db-aborts-on-unkown-file-test/ * Also fix mysql_install_db.pl.in --- scripts/mysql_install_db.pl.in | 2 +- scripts/mysql_install_db.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mysql_install_db.pl.in b/scripts/mysql_install_db.pl.in index d1b299d9ef5e3..8b99b3564e4f2 100644 --- a/scripts/mysql_install_db.pl.in +++ b/scripts/mysql_install_db.pl.in @@ -508,7 +508,7 @@ if ( open(PIPE, "| $mysqld_install_cmd_line") ) # FIXME > /dev/null ? if ( open(PIPE, "| $mysqld_install_cmd_line") ) { - print PIPE "use test;\n"; + print PIPE "use mysql;\n"; while ( ) { print PIPE $_; diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index cd479d959c19c..1daa1bf58eed1 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -473,7 +473,7 @@ else fi s_echo "Creating OpenGIS required SP-s..." -if { echo "use test;"; cat "$maria_add_gis_sp"; } | mysqld_install_cmd_line > /dev/null +if { echo "use mysql;"; cat "$maria_add_gis_sp"; } | mysqld_install_cmd_line > /dev/null then s_echo "OK" else From 2bedc3978b90bf5abe1029df393c63ced1849bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 22 Sep 2016 16:32:26 +0300 Subject: [PATCH 053/295] MDEV-9931: InnoDB reads first page of every .ibd file at startup Analysis: By design InnoDB was reading first page of every .ibd file at startup to find out is tablespace encrypted or not. This is because tablespace could have been encrypted always, not encrypted newer or encrypted based on configuration and this information can be find realible only from first page of .ibd file. Fix: Do not read first page of every .ibd file at startup. Instead whenever tablespace is first time accedded we will read the first page to find necessary information about tablespace encryption status. TODO: Add support for SYS_TABLEOPTIONS where all table options encryption information included will be stored. --- .../encryption/r/innodb-bad-key-change.result | 7 +- .../r/innodb_encryption_row_compressed.result | 153 ++++++++++ .../encryption/r/innodb_lotoftables.result | 154 ++++++++++ .../t/innodb_encryption_row_compressed.opt | 4 + .../t/innodb_encryption_row_compressed.test | 125 ++++++++ .../suite/encryption/t/innodb_lotoftables.opt | 3 + .../encryption/t/innodb_lotoftables.test | 274 ++++++++++++++++++ .../r/innodb_monitor_disable_basic.result | 1 + .../r/innodb_monitor_enable_basic.result | 1 + storage/innobase/btr/btr0btr.cc | 38 ++- storage/innobase/dict/dict0load.cc | 13 +- storage/innobase/fil/fil0crypt.cc | 76 ++++- storage/innobase/fil/fil0fil.cc | 70 ++++- storage/innobase/handler/ha_innodb.cc | 2 + storage/innobase/include/btr0btr.ic | 4 +- storage/innobase/include/dict0mem.h | 2 + storage/innobase/include/fil0crypt.h | 3 +- storage/innobase/include/fil0fil.h | 14 +- storage/innobase/include/page0page.ic | 27 +- storage/innobase/include/srv0mon.h | 1 + storage/innobase/include/srv0srv.h | 6 +- storage/innobase/row/row0mysql.cc | 8 +- storage/innobase/srv/srv0mon.cc | 11 + storage/innobase/srv/srv0srv.cc | 3 +- storage/innobase/srv/srv0start.cc | 25 +- storage/xtradb/btr/btr0btr.cc | 39 ++- storage/xtradb/dict/dict0load.cc | 13 +- storage/xtradb/fil/fil0crypt.cc | 76 ++++- storage/xtradb/fil/fil0fil.cc | 71 ++++- storage/xtradb/handler/ha_innodb.cc | 2 + storage/xtradb/include/btr0btr.ic | 4 +- storage/xtradb/include/dict0mem.h | 2 + storage/xtradb/include/fil0crypt.h | 3 +- storage/xtradb/include/fil0fil.h | 15 +- storage/xtradb/include/srv0mon.h | 1 + storage/xtradb/include/srv0srv.h | 6 +- storage/xtradb/row/row0mysql.cc | 8 +- storage/xtradb/srv/srv0mon.cc | 11 + storage/xtradb/srv/srv0srv.cc | 3 +- storage/xtradb/srv/srv0start.cc | 26 +- 40 files changed, 1173 insertions(+), 132 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result create mode 100644 mysql-test/suite/encryption/r/innodb_lotoftables.result create mode 100644 mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt create mode 100644 mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test create mode 100644 mysql-test/suite/encryption/t/innodb_lotoftables.opt create mode 100644 mysql-test/suite/encryption/t/innodb_lotoftables.test diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index cf9791887cc83..ec8dee89230ec 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -36,10 +36,13 @@ SELECT * FROM t1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t1' -Warning 192 Table test/t1 is encrypted but encryption service or used key_id 2 is not available. Can't continue reading table. +Warning 192 Table test/t1 in tablespace 6 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DROP TABLE t1; +Warnings: +Warning 192 Table in tablespace 6 encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. +Warning 192 Table in tablespace 6 encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. # Start server with keys.txt CREATE TABLE t2 (c VARCHAR(8), id int not null primary key, b int, key(b)) ENGINE=InnoDB ENCRYPTED=YES; INSERT INTO t2 VALUES ('foobar',1,2); diff --git a/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result b/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result new file mode 100644 index 0000000000000..355271ef9caf3 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result @@ -0,0 +1,153 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create table innodb_compressed1(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed encrypted=yes; +create table innodb_compressed2(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=1 encrypted=yes; +create table innodb_compressed3(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=2 encrypted=yes; +create table innodb_compressed4(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=4 encrypted=yes; +insert into innodb_compressed1 values (1, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (2, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (3, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (4, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (5, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (6, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (7, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (8, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (9, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (10, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed2 select * from innodb_compressed1; +insert into innodb_compressed3 select * from innodb_compressed1; +insert into innodb_compressed4 select * from innodb_compressed1; +# t1 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed4.ibd +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +select * from innodb_compressed1 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed1 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +select * from innodb_compressed2 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed2 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +select * from innodb_compressed3 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed3 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +select * from innodb_compressed4 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed4 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +update innodb_compressed1 set d = d + 10 where d = 30; +update innodb_compressed2 set d = d + 10 where d = 30; +update innodb_compressed3 set d = d + 10 where d = 30; +update innodb_compressed4 set d = d + 10 where d = 30; +insert into innodb_compressed1 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed2 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed3 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed4 values (20, 60, 'newprivate', 'newevenmoreprivate'); +# t1 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed4.ibd +select * from innodb_compressed1 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed1 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +select * from innodb_compressed2 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed2 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +select * from innodb_compressed3 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed3 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +select * from innodb_compressed4 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed4 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +# t1 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed4.ibd +drop table innodb_compressed1; +drop table innodb_compressed2; +drop table innodb_compressed3; +drop table innodb_compressed4; diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result new file mode 100644 index 0000000000000..5e3eaef550f41 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -0,0 +1,154 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SHOW VARIABLES LIKE 'innodb_encrypt%'; +Variable_name Value +innodb_encrypt_log OFF +innodb_encrypt_tables OFF +innodb_encryption_rotate_key_age 1 +innodb_encryption_rotation_iops 100 +innodb_encryption_threads 0 +create database innodb_encrypted_1; +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 1 +set autocommit=0; +set autocommit=1; +commit work; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 1 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +create database innodb_encrypted_2; +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +set autocommit=0; +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +create database innodb_encrypted_3; +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +set autocommit=0; +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +# should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +use test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +SET GLOBAL innodb_encrypt_tables = on; +SET GLOBAL innodb_encryption_threads=4; +# Wait until all encrypted tables have been encrypted +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +# Success! +# Restart mysqld --innodb_encrypt_tables=0 --innodb_encryption_threads=0 +# Restart Success! +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 103 +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 103 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 203 +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 203 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 203 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +SET GLOBAL innodb_encrypt_tables = off; +SET GLOBAL innodb_encryption_threads=4; +# Wait until all default encrypted tables have been decrypted +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +# should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 303 +use test; +drop database innodb_encrypted_1; +drop database innodb_encrypted_2; +drop database innodb_encrypted_3; diff --git a/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt new file mode 100644 index 0000000000000..7ebf81a07f378 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt @@ -0,0 +1,4 @@ +--innodb-encrypt-tables=ON +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test new file mode 100644 index 0000000000000..0a28c1690a21a --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test @@ -0,0 +1,125 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +-- source include/not_embedded.inc + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +create table innodb_compressed1(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed encrypted=yes; +create table innodb_compressed2(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=1 encrypted=yes; +create table innodb_compressed3(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=2 encrypted=yes; +create table innodb_compressed4(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=4 encrypted=yes; + +insert into innodb_compressed1 values (1, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (2, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (3, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (4, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (5, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (6, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (7, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (8, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (9, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (10, 20, 'private', 'evenmoreprivate'); + +insert into innodb_compressed2 select * from innodb_compressed1; +insert into innodb_compressed3 select * from innodb_compressed1; +insert into innodb_compressed4 select * from innodb_compressed1; + +--source include/restart_mysqld.inc + +--let $MYSQLD_DATADIR=`select @@datadir` +--let t1_IBD = $MYSQLD_DATADIR/test/innodb_compressed1.ibd +--let t2_IBD = $MYSQLD_DATADIR/test/innodb_compressed2.ibd +--let t3_IBD = $MYSQLD_DATADIR/test/innodb_compressed3.ibd +--let t4_IBD = $MYSQLD_DATADIR/test/innodb_compressed4.ibd +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=private +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +select * from innodb_compressed1 where d = 20; +select * from innodb_compressed1 where d = 30; +select * from innodb_compressed2 where d = 20; +select * from innodb_compressed2 where d = 30; +select * from innodb_compressed3 where d = 20; +select * from innodb_compressed3 where d = 30; +select * from innodb_compressed4 where d = 20; +select * from innodb_compressed4 where d = 30; + +update innodb_compressed1 set d = d + 10 where d = 30; +update innodb_compressed2 set d = d + 10 where d = 30; +update innodb_compressed3 set d = d + 10 where d = 30; +update innodb_compressed4 set d = d + 10 where d = 30; + +insert into innodb_compressed1 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed2 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed3 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed4 values (20, 60, 'newprivate', 'newevenmoreprivate'); + +--let SEARCH_PATTERN=private +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +--source include/restart_mysqld.inc + +select * from innodb_compressed1 where d = 40; +select * from innodb_compressed1 where d = 60; +select * from innodb_compressed2 where d = 40; +select * from innodb_compressed2 where d = 60; +select * from innodb_compressed3 where d = 40; +select * from innodb_compressed3 where d = 60; +select * from innodb_compressed4 where d = 40; +select * from innodb_compressed4 where d = 60; + +--let SEARCH_PATTERN=private +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +drop table innodb_compressed1; +drop table innodb_compressed2; +drop table innodb_compressed3; +drop table innodb_compressed4; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_lotoftables.opt b/mysql-test/suite/encryption/t/innodb_lotoftables.opt new file mode 100644 index 0000000000000..ffb5a2957f89e --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_lotoftables.opt @@ -0,0 +1,3 @@ +--innodb-tablespaces-encryption +--innodb-encrypt-tables=off +--innodb-encryption-threads=0 diff --git a/mysql-test/suite/encryption/t/innodb_lotoftables.test b/mysql-test/suite/encryption/t/innodb_lotoftables.test new file mode 100644 index 0000000000000..cad3cb54326e3 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_lotoftables.test @@ -0,0 +1,274 @@ +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +-- source include/big_test.inc + +# embedded does not support restart +-- source include/not_embedded.inc + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $innodb_encryption_threads_orig = `SELECT @@global.innodb_encryption_threads`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +SHOW VARIABLES LIKE 'innodb_encrypt%'; + +# +# This will create 100 tables where that could be +# encrypted an unencrypt +# +create database innodb_encrypted_1; +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +set autocommit=0; +let $tables = 100; + +--disable_query_log +while ($tables) +{ + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb; + commit; + let $rows = 100; + while($rows) + { + eval insert into t_$tables values ($rows, substring(MD5(RAND()), -64)); + dec $rows; + } + commit; + dec $tables; +} +--enable_query_log + +set autocommit=1; +commit work; +show status like 'innodb_pages0_read%'; +# +# Verify +# +--echo # should be 100 + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; + +# +# This will create 100 tables that are encrypted always +# +create database innodb_encrypted_2; +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +set autocommit=0; + +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb encrypted=yes; + commit; + let $rows = 100; + while($rows) + { + eval insert into t_$tables values ($rows, substring(MD5(RAND()), -64)); + dec $rows; + } + commit; + dec $tables; +} +--enable_query_log + +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +# +# Verify +# +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; + +# +# This will create 100 tables that are not encrypted +# +create database innodb_encrypted_3; +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +set autocommit=0; + +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb encrypted=no; + commit; + let $rows = 100; + while($rows) + { + eval insert into t_$tables values ($rows, substring(MD5(RAND()), -64)); + dec $rows; + } + commit; + dec $tables; +} +--enable_query_log + +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +# +# Verify +# +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +--echo # should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; + +use test; +show status like 'innodb_pages0_read%'; + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; + +SET GLOBAL innodb_encrypt_tables = on; +SET GLOBAL innodb_encryption_threads=4; + +--echo # Wait until all encrypted tables have been encrypted +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 100 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +show status like 'innodb_pages0_read%'; + +--echo # Success! +--echo # Restart mysqld --innodb_encrypt_tables=0 --innodb_encryption_threads=0 +-- let $restart_parameters=--innodb_encrypt_tables=0 --innodb_encryption_threads=0 +-- source include/restart_mysqld.inc + +--echo # Restart Success! +show status like 'innodb_pages0_read%'; + +show status like 'innodb_pages0_read%'; +use test; +show status like 'innodb_pages0_read%'; +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; + +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +--disable_result_log +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval select * from t_$tables; + dec $tables; +} +--enable_query_log +--enable_result_log + +show status like 'innodb_pages0_read%'; + +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; + +--disable_result_log +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval select * from t_$tables; + dec $tables; +} +--enable_query_log +--enable_result_log + +show status like 'innodb_pages0_read%'; + +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +--disable_result_log +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval select * from t_$tables; + dec $tables; +} +--enable_query_log +--enable_result_log + +show status like 'innodb_pages0_read%'; + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; + +SET GLOBAL innodb_encrypt_tables = off; +SET GLOBAL innodb_encryption_threads=4; + +--echo # Wait until all default encrypted tables have been decrypted +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 100 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} + +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +--echo # should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +show status like 'innodb_pages0_read%'; + +# +# Cleanup +# +use test; +drop database innodb_encrypted_1; +drop database innodb_encrypted_2; +drop database innodb_encrypted_3; + +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +EVAL SET GLOBAL innodb_encryption_threads = $innodb_encryption_threads_orig; +--enable_query_log diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result index 6c7051dc3d076..f7a02ed354865 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result index 6c7051dc3d076..f7a02ed354865 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 33ca57c96548b..470825fa24628 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation +Copyright (c) 2014, 2016, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -737,14 +737,16 @@ btr_root_block_get( block = btr_block_get(space, zip_size, root_page_no, mode, (dict_index_t*)index, mtr); if (!block) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; - - ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name, space); + if (index && index->table) { + index->table->is_encrypted = TRUE; + index->table->corrupted = FALSE; + + ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + index->table->name, space); + } return NULL; } @@ -1819,6 +1821,12 @@ btr_free_but_not_root( root = btr_page_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, &mtr); + + if (!root) { + mtr_commit(&mtr); + return; + } + #ifdef UNIV_BTR_DEBUG ut_a(btr_root_fseg_validate(FIL_PAGE_DATA + PAGE_BTR_SEG_LEAF + root, space)); @@ -1875,15 +1883,17 @@ btr_free_root( block = btr_block_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, mtr); - btr_search_drop_page_hash_index(block); + if (block) { + btr_search_drop_page_hash_index(block); - header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; + header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; #ifdef UNIV_BTR_DEBUG - ut_a(btr_root_fseg_validate(header, space)); + ut_a(btr_root_fseg_validate(header, space)); #endif /* UNIV_BTR_DEBUG */ - while (!fseg_free_step(header, mtr)) { - /* Free the entire segment in small steps. */ + while (!fseg_free_step(header, mtr)) { + /* Free the entire segment in small steps. */ + } } } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 3c72efdb6f33e..8baab7ad53685 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1154,11 +1154,14 @@ dict_check_tablespaces_and_store_max_id( space_id, name); } - /* We need to read page 0 to get (optional) IV - regardless if encryptions is turned on or not, - since if it's off we should decrypt a potentially - already encrypted table */ - bool read_page_0 = true; + /* We could read page 0 to get (optional) IV + if encryption is turned on, if it's off + we will read the page 0 later and find out + if we should decrypt a potentially + already encrypted table + bool read_page_0 = srv_encrypt_tables; */ + + bool read_page_0 = false; /* We set the 2nd param (fix_dict = true) here because we already have an x-lock on diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index ceffa950739fb..bedaf7cfe4967 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -258,20 +258,6 @@ fil_space_read_crypt_data( } if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { -#ifdef UNIV_DEBUG - ib_logf(IB_LOG_LEVEL_WARN, - "Found potentially bogus bytes on " - "page 0 offset %lu for space %lu : " - "[ %.2x %.2x %.2x %.2x %.2x %.2x ]. " - "Assuming space is not encrypted!.", - offset, space, - page[offset + 0], - page[offset + 1], - page[offset + 2], - page[offset + 3], - page[offset + 4], - page[offset + 5]); -#endif /* Crypt data is not stored. */ return NULL; } @@ -666,6 +652,61 @@ fil_space_encrypt( byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame); +#ifdef UNIV_DEBUG + if (tmp) { + /* Verify that encrypted buffer is not corrupted */ + byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + dberr_t err = DB_SUCCESS; + byte* src = src_frame; + bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); + byte* comp_mem = NULL; + byte* uncomp_mem = NULL; + ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + + if (page_compressed_encrypted) { + comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); + fil_decompress_page(uncomp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + src = uncomp_mem; + } + + bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); + bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + + /* Need to decompress the page if it was also compressed */ + if (page_compressed_encrypted) { + memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); + fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + } + + bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); + bool different = memcmp(src, tmp_mem, size); + + if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { + fprintf(stderr, "JAN: ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "JAN1: src_frame\n"); + buf_page_print(src_frame, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN2: encrypted_frame\n"); + buf_page_print(tmp, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN1: decrypted_frame\n"); + buf_page_print(tmp_mem, zip_size, BUF_PAGE_PRINT_NO_CRASH); + ut_error; + } + + free(tmp_mem); + + if (comp_mem) { + free(comp_mem); + } + + if (uncomp_mem) { + free(uncomp_mem); + } + } + +#endif /* UNIV_DEBUG */ + return tmp; } @@ -2426,7 +2467,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space) /*!< in: Space id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data) /*!< in: crypt_data or NULL */ { if (!fil_crypt_threads_inited) { return; @@ -2434,7 +2476,9 @@ fil_space_crypt_mark_space_closing( mutex_enter(&fil_crypt_threads_mutex); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (!crypt_data) { + crypt_data = fil_space_get_crypt_data(space); + } if (crypt_data == NULL) { mutex_exit(&fil_crypt_threads_mutex); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index b92ac02da1058..43aecf45b6052 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1156,7 +1156,8 @@ fil_space_create( ulint id, /*!< in: space id */ ulint flags, /*!< in: tablespace flags */ ulint purpose,/*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data) /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table) /*!< in: true if create table */ { fil_space_t* space; @@ -1240,6 +1241,21 @@ fil_space_create( space->magic_n = FIL_SPACE_MAGIC_N; space->printed_compression_failure = false; + space->crypt_data = crypt_data; + + /* In create table we write page 0 so we have already + "read" it and for system tablespaces we have read + crypt data at startup. */ + if (create_table || crypt_data != NULL) { + space->page_0_crypt_read = true; + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Created tablespace for space %lu name %s key_id %u encryption %d\n", + space->id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0); rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); @@ -1251,7 +1267,6 @@ fil_space_create( UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); - space->crypt_data = crypt_data; mutex_exit(&fil_system->mutex); @@ -2020,6 +2035,8 @@ fil_read_first_page( os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); + srv_stats.page0_read.add(1); + /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave *flags and *space_id as they were read from the first file and @@ -2039,6 +2056,7 @@ fil_read_first_page( ulint space = fsp_header_get_space_id(page); ulint offset = fsp_header_get_crypt_offset( fsp_flags_get_zip_size(*flags), NULL); + cdata = fil_space_read_crypt_data(space, page, offset); if (crypt_data) { @@ -3595,7 +3613,7 @@ fil_create_new_single_table_tablespace( } success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE, - crypt_data); + crypt_data, true); if (!success || !fil_node_create(path, size, space_id, FALSE)) { err = DB_ERROR; @@ -3832,6 +3850,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = def.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3871,6 +3890,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = remote.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3910,6 +3930,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = dict.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -4081,7 +4102,7 @@ fil_open_single_table_tablespace( if (err != DB_SUCCESS) { ; // Don't load the tablespace into the cache } else if (!fil_space_create(tablename, id, flags, FIL_TABLESPACE, - crypt_data)) { + crypt_data, false)) { err = DB_ERROR; } else { /* We do not measure the size of the file, that is why @@ -4698,7 +4719,7 @@ fil_load_single_table_tablespace( #endif /* UNIV_HOTBACKUP */ ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, - fsp->crypt_data); + fsp->crypt_data, false); if (!file_space_create_success) { if (srv_force_recovery > 0) { @@ -7224,7 +7245,46 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); if (space != NULL) { + /* If we have not yet read the page0 + of this tablespace we will do it now. */ + if (!space->crypt_data && !space->page_0_crypt_read) { + ulint flags; + ulint space_id; + lsn_t min_flushed_lsn; + lsn_t max_flushed_lsn; + fil_node_t* node; + + ut_a(space->crypt_data == NULL); + node = UT_LIST_GET_FIRST(space->chain); + + fil_node_prepare_for_io(node, fil_system, space); + + const char* msg = fil_read_first_page(node->handle, + false, + &flags, + &space_id, + &min_flushed_lsn, + &max_flushed_lsn, + &space->crypt_data); + + fil_node_complete_io(node, fil_system, OS_FILE_READ); + + ib_logf(IB_LOG_LEVEL_INFO, + "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", + space_id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0, + node->handle); + + ut_a(space->id == space_id); + + space->page_0_crypt_read = true; + } + crypt_data = space->crypt_data; + + ut_ad(space->page_0_crypt_read); } mutex_exit(&fil_system->mutex); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 0eaf8d3546392..fd15092d96cb6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -943,6 +943,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_created, SHOW_LONG}, {"pages_read", (char*) &export_vars.innodb_pages_read, SHOW_LONG}, + {"pages0_read", + (char*) &export_vars.innodb_page0_read, SHOW_LONG}, {"pages_written", (char*) &export_vars.innodb_pages_written, SHOW_LONG}, {"row_lock_current_waits", diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index 64b3d5a0975c1..e94103102130f 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -60,7 +60,9 @@ btr_block_get_func( NULL, BUF_GET, file, line, mtr, &err); if (err == DB_DECRYPTION_FAILED) { - index->table->is_encrypted = true; + if (index && index->table) { + index->table->is_encrypted = true; + } } if (block) { diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index f964447fb8f20..f2d72de39e5ce 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1030,6 +1030,8 @@ struct dict_table_t{ mem_heap_t* heap; /*!< memory heap */ char* name; /*!< table name */ void* thd; /*!< thd */ + bool page_0_read; /*!< true if page 0 has + been already read */ fil_space_crypt_t *crypt_data; /*!< crypt data if present */ const char* dir_path_of_temp_table;/*!< NULL or the directory path where a TEMPORARY table that was explicitly diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index fdc413e752023..8ffa4e2073fcd 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -316,7 +316,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space); /*!< in: tablespace id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */ /********************************************************************* Wait for crypt threads to stop accessing space */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index c97143235bc98..ae8224d77bb25 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -330,10 +330,17 @@ struct fil_space_t { bool printed_compression_failure; /*!< true if we have already printed compression failure */ + fil_space_crypt_t* crypt_data; + /*!< tablespace crypt data or NULL */ + bool page_0_crypt_read; + /*!< tablespace crypt data has been + read */ + ulint file_block_size; + /*!< file system block size */ + UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - fil_space_crypt_t* crypt_data; - ulint file_block_size;/*!< file system block size */ + ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -470,7 +477,8 @@ fil_space_create( ulint zip_size,/*!< in: compressed page size, or 0 for uncompressed tablespaces */ ulint purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table); /*!< in: true if create table */ /*******************************************************************//** Assigns a new space id for a new single-table tablespace. This works simply by diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index cde3cad33f06f..d7f1db828581f 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -52,6 +52,7 @@ page_align( { return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE)); } + #ifndef UNIV_INNOCHECKSUM /************************************************************//** Gets the offset within a page. @@ -230,18 +231,6 @@ page_header_reset_last_insert( #endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ -/************************************************************//** -Determine whether the page is in new-style compact format. -@return nonzero if the page is in compact format, zero if it is in -old-style format */ -UNIV_INLINE -ulint -page_is_comp( -/*=========*/ - const page_t* page) /*!< in: index page */ -{ - return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); -} #ifndef UNIV_INNOCHECKSUM /************************************************************//** @@ -973,6 +962,20 @@ page_rec_get_base_extra_size( } #endif /* !UNIV_INNOCHECKSUM */ + +/************************************************************//** +Determine whether the page is in new-style compact format. +@return nonzero if the page is in compact format, zero if it is in +old-style format */ +UNIV_INLINE +ulint +page_is_comp( +/*=========*/ + const page_t* page) /*!< in: index page */ +{ + return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); +} + /************************************************************//** Returns the sum of the sizes of the records in the record list, excluding the infimum and supremum records. diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 422cfc3eaf6d4..d15110726b9b2 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -167,6 +167,7 @@ enum monitor_id_t { MONITOR_OVLD_INDEX_PAGES_WRITTEN, MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN, MONITOR_OVLD_PAGES_READ, + MONITOR_OVLD_PAGES0_READ, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS_AVOIDED, MONITOR_OVLD_BYTE_READ, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 04c8cbecf9988..d75f60944cc8d 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -179,6 +179,9 @@ struct srv_stats_t { /** Number of times prefix optimization avoided triggering cluster lookup */ ulint_ctr_64_t n_sec_rec_cluster_reads_avoided; + + /** Number of times page 0 is read from tablespace */ + ulint_ctr_64_t page0_read; }; extern const char* srv_main_thread_op_info; @@ -950,7 +953,8 @@ struct export_var_t{ ulint innodb_os_log_pending_fsyncs; /*!< fil_n_pending_log_flushes */ ulint innodb_page_size; /*!< UNIV_PAGE_SIZE */ ulint innodb_pages_created; /*!< buf_pool->stat.n_pages_created */ - ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read */ + ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read*/ + ulint innodb_page0_read; /*!< srv_stats.page0_read */ ulint innodb_pages_written; /*!< buf_pool->stat.n_pages_written */ ulint innodb_row_lock_waits; /*!< srv_n_lock_wait_count */ ulint innodb_row_lock_current_waits; /*!< srv_n_lock_wait_current_count */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index a5ab4f4911ea2..b2c96a7ed7b31 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3279,7 +3279,7 @@ fil_wait_crypt_bg_threads( uint last = start; if (table->space != 0) { - fil_space_crypt_mark_space_closing(table->space); + fil_space_crypt_mark_space_closing(table->space, table->crypt_data); } while (table->n_ref_count > 0) { @@ -4211,6 +4211,12 @@ row_drop_table_for_mysql( rw_lock_x_unlock(dict_index_get_lock(index)); } + /* If table has not yet have crypt_data, try to read it to + make freeing the table easier. */ + if (!table->crypt_data) { + table->crypt_data = fil_space_get_crypt_data(table->space); + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 3375ea406587d..6bf4ad8af01a7 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -309,6 +309,12 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_READ}, + {"buffer_pages0_read", "buffer", + "Number of page 0 read (innodb_pages0_read)", + static_cast( + MONITOR_EXISTING | MONITOR_DEFAULT_ON), + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES0_READ}, + {"buffer_index_sec_rec_cluster_reads", "buffer", "Number of secondary record reads triggered cluster read", static_cast( @@ -1718,6 +1724,11 @@ srv_mon_process_existing_counter( value = stat.n_pages_read; break; + /* innodb_pages0_read */ + case MONITOR_OVLD_PAGES0_READ: + value = srv_stats.page0_read; + break; + /* Number of times secondary index lookup triggered cluster lookup */ case MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS: value = srv_stats.n_sec_rec_cluster_reads; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e17b27b44fc56..076bc36613281 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1549,6 +1549,7 @@ srv_export_innodb_status(void) export_vars.innodb_pages_created = stat.n_pages_created; export_vars.innodb_pages_read = stat.n_pages_read; + export_vars.innodb_page0_read = srv_stats.page0_read; export_vars.innodb_pages_written = stat.n_pages_written; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 0b81ad86f1c54..135846384f689 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2015, MariaDB Corporation +Copyright (c) 2013, 2016, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -675,7 +675,8 @@ create_log_files( logfilename, SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* this is create */); ut_a(fil_validate()); logfile0 = fil_node_create( @@ -813,7 +814,7 @@ open_or_create_data_files( ulint space; ulint rounded_size_pages; char name[10000]; - fil_space_crypt_t* crypt_data; + fil_space_crypt_t* crypt_data=NULL; if (srv_n_data_files >= 1000) { @@ -1150,18 +1151,20 @@ open_or_create_data_files( } *sum_of_new_sizes += srv_data_file_sizes[i]; - - crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } ret = os_file_close(files[i]); ut_a(ret); if (i == 0) { + if (!crypt_data) { + crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); + } + flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + fil_space_create(name, 0, flags, FIL_TABLESPACE, - crypt_data); - crypt_data = NULL; + crypt_data, (*create_new_db) == true); } ut_a(fil_validate()); @@ -1308,7 +1311,8 @@ srv_undo_tablespace_open( /* Set the compressed page size to 0 (non-compressed) */ flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); fil_space_create(name, space, flags, FIL_TABLESPACE, - NULL /* no encryption */); + NULL /* no encryption */, + true /* create */); ut_a(fil_validate()); @@ -2293,7 +2297,8 @@ innobase_start_or_create_for_mysql(void) SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* create */); ut_a(fil_validate()); @@ -2315,7 +2320,7 @@ innobase_start_or_create_for_mysql(void) /* Create the file space object for archived logs. Under MySQL, no archiving ever done. */ fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1, - 0, FIL_LOG); + 0, FIL_LOG, NULL, true); #endif /* UNIV_LOG_ARCHIVE */ log_group_init(0, i, srv_log_file_size * UNIV_PAGE_SIZE, SRV_LOG_SPACE_FIRST_ID, diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index 00a04e75c4992..c2a70cce7aa01 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation +Copyright (c) 2014, 2016, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -743,14 +743,16 @@ btr_root_block_get( block = btr_block_get(space, zip_size, root_page_no, mode, (dict_index_t*)index, mtr); if (!block) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; - - ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name, space); + if (index && index->table) { + index->table->is_encrypted = TRUE; + index->table->corrupted = FALSE; + + ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + index->table->name, space); + } return NULL; } @@ -1840,6 +1842,11 @@ btr_free_but_not_root( root = btr_page_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, &mtr); + if (!root) { + mtr_commit(&mtr); + return; + } + SRV_CORRUPT_TABLE_CHECK(root, { mtr_commit(&mtr); @@ -1909,17 +1916,19 @@ btr_free_root( block = btr_block_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, mtr); - SRV_CORRUPT_TABLE_CHECK(block, return;); + if (block) { + SRV_CORRUPT_TABLE_CHECK(block, return;); - btr_search_drop_page_hash_index(block); + btr_search_drop_page_hash_index(block); - header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; + header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; #ifdef UNIV_BTR_DEBUG - ut_a(btr_root_fseg_validate(header, space)); + ut_a(btr_root_fseg_validate(header, space)); #endif /* UNIV_BTR_DEBUG */ - while (!fseg_free_step(header, mtr)) { - /* Free the entire segment in small steps. */ + while (!fseg_free_step(header, mtr)) { + /* Free the entire segment in small steps. */ + } } } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 3dcdfc7c61a85..8b7034e5ebaf5 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1154,11 +1154,14 @@ dict_check_tablespaces_and_store_max_id( space_id, name); } - /* We need to read page 0 to get (optional) IV - regardless if encryptions is turned on or not, - since if it's off we should decrypt a potentially - already encrypted table */ - bool read_page_0 = true; + /* We could read page 0 to get (optional) IV + if encryption is turned on, if it's off + we will read the page 0 later and find out + if we should decrypt a potentially + already encrypted table + bool read_page_0 = srv_encrypt_tables; */ + + bool read_page_0 = false; /* We set the 2nd param (fix_dict = true) here because we already have an x-lock on diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index ceffa950739fb..bedaf7cfe4967 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -258,20 +258,6 @@ fil_space_read_crypt_data( } if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { -#ifdef UNIV_DEBUG - ib_logf(IB_LOG_LEVEL_WARN, - "Found potentially bogus bytes on " - "page 0 offset %lu for space %lu : " - "[ %.2x %.2x %.2x %.2x %.2x %.2x ]. " - "Assuming space is not encrypted!.", - offset, space, - page[offset + 0], - page[offset + 1], - page[offset + 2], - page[offset + 3], - page[offset + 4], - page[offset + 5]); -#endif /* Crypt data is not stored. */ return NULL; } @@ -666,6 +652,61 @@ fil_space_encrypt( byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame); +#ifdef UNIV_DEBUG + if (tmp) { + /* Verify that encrypted buffer is not corrupted */ + byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + dberr_t err = DB_SUCCESS; + byte* src = src_frame; + bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); + byte* comp_mem = NULL; + byte* uncomp_mem = NULL; + ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + + if (page_compressed_encrypted) { + comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); + fil_decompress_page(uncomp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + src = uncomp_mem; + } + + bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); + bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + + /* Need to decompress the page if it was also compressed */ + if (page_compressed_encrypted) { + memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); + fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + } + + bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); + bool different = memcmp(src, tmp_mem, size); + + if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { + fprintf(stderr, "JAN: ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "JAN1: src_frame\n"); + buf_page_print(src_frame, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN2: encrypted_frame\n"); + buf_page_print(tmp, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN1: decrypted_frame\n"); + buf_page_print(tmp_mem, zip_size, BUF_PAGE_PRINT_NO_CRASH); + ut_error; + } + + free(tmp_mem); + + if (comp_mem) { + free(comp_mem); + } + + if (uncomp_mem) { + free(uncomp_mem); + } + } + +#endif /* UNIV_DEBUG */ + return tmp; } @@ -2426,7 +2467,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space) /*!< in: Space id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data) /*!< in: crypt_data or NULL */ { if (!fil_crypt_threads_inited) { return; @@ -2434,7 +2476,9 @@ fil_space_crypt_mark_space_closing( mutex_enter(&fil_crypt_threads_mutex); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (!crypt_data) { + crypt_data = fil_space_get_crypt_data(space); + } if (crypt_data == NULL) { mutex_exit(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 58e08f117787f..1db342d2bc741 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1191,7 +1191,8 @@ fil_space_create( ulint id, /*!< in: space id */ ulint flags, /*!< in: tablespace flags */ ulint purpose,/*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data) /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table) /*!< in: true if create table */ { fil_space_t* space; @@ -1285,10 +1286,23 @@ fil_space_create( space->is_in_unflushed_spaces = false; space->is_corrupt = FALSE; + space->crypt_data = crypt_data; - UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); + /* In create table we write page 0 so we have already + "read" it and for system tablespaces we have read + crypt data at startup. */ + if (create_table || crypt_data != NULL) { + space->page_0_crypt_read = true; + } - space->crypt_data = crypt_data; + ib_logf(IB_LOG_LEVEL_INFO, + "Created tablespace for space %lu name %s key_id %u encryption %d\n", + space->id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0); + + UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); mutex_exit(&fil_system->mutex); @@ -2057,6 +2071,8 @@ fil_read_first_page( os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); + srv_stats.page0_read.add(1); + /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave *flags and *space_id as they were read from the first file and @@ -2077,6 +2093,7 @@ fil_read_first_page( ulint space = fsp_header_get_space_id(page); ulint offset = fsp_header_get_crypt_offset( fsp_flags_get_zip_size(*flags), NULL); + cdata = fil_space_read_crypt_data(space, page, offset); if (crypt_data) { @@ -3627,7 +3644,7 @@ fil_create_new_single_table_tablespace( } success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE, - crypt_data); + crypt_data, true); if (!success || !fil_node_create(path, size, space_id, FALSE)) { err = DB_ERROR; @@ -3861,6 +3878,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = def.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3897,6 +3915,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = remote.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3933,6 +3952,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = dict.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -4104,7 +4124,7 @@ fil_open_single_table_tablespace( if (err != DB_SUCCESS) { ; // Don't load the tablespace into the cache } else if (!fil_space_create(tablename, id, flags, FIL_TABLESPACE, - crypt_data)) { + crypt_data, false)) { err = DB_ERROR; } else { /* We do not measure the size of the file, that is why @@ -4710,7 +4730,7 @@ fil_load_single_table_tablespace( #endif /* UNIV_HOTBACKUP */ ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, - fsp->crypt_data); + fsp->crypt_data, false); if (!file_space_create_success) { if (srv_force_recovery > 0) { @@ -7323,7 +7343,46 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); if (space != NULL) { + /* If we have not yet read the page0 + of this tablespace we will do it now. */ + if (!space->crypt_data && !space->page_0_crypt_read) { + ulint flags; + ulint space_id; + lsn_t min_flushed_lsn; + lsn_t max_flushed_lsn; + fil_node_t* node; + + ut_a(space->crypt_data == NULL); + node = UT_LIST_GET_FIRST(space->chain); + + fil_node_prepare_for_io(node, fil_system, space); + + const char* msg = fil_read_first_page(node->handle, + false, + &flags, + &space_id, + &min_flushed_lsn, + &max_flushed_lsn, + &space->crypt_data); + + fil_node_complete_io(node, fil_system, OS_FILE_READ); + + ib_logf(IB_LOG_LEVEL_INFO, + "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", + space_id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0, + node->handle); + + ut_a(space->id == space_id); + + space->page_0_crypt_read = true; + } + crypt_data = space->crypt_data; + + ut_ad(space->page_0_crypt_read); } mutex_exit(&fil_system->mutex); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 868ece7dbe926..14870659b0e28 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1116,6 +1116,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_created, SHOW_LONG}, {"pages_read", (char*) &export_vars.innodb_pages_read, SHOW_LONG}, + {"pages0_read", + (char*) &export_vars.innodb_page0_read, SHOW_LONG}, {"pages_written", (char*) &export_vars.innodb_pages_written, SHOW_LONG}, {"purge_trx_id", diff --git a/storage/xtradb/include/btr0btr.ic b/storage/xtradb/include/btr0btr.ic index 8c9c3bead0972..34e0d36e230b2 100644 --- a/storage/xtradb/include/btr0btr.ic +++ b/storage/xtradb/include/btr0btr.ic @@ -60,7 +60,9 @@ btr_block_get_func( NULL, BUF_GET, file, line, mtr, &err); if (err == DB_DECRYPTION_FAILED) { - index->table->is_encrypted = true; + if (index && index->table) { + index->table->is_encrypted = true; + } } if (block) { diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 24219ac8799ba..29f5b8fe8145d 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -1046,6 +1046,8 @@ struct dict_table_t{ mem_heap_t* heap; /*!< memory heap */ char* name; /*!< table name */ void* thd; /*!< thd */ + bool page_0_read; /*!< true if page 0 has + been already read */ fil_space_crypt_t *crypt_data; /*!< crypt data if present */ const char* dir_path_of_temp_table;/*!< NULL or the directory path where a TEMPORARY table that was explicitly diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 5deed1f001c95..b656cd3985d76 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -316,7 +316,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space); /*!< in: tablespace id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */ /********************************************************************* Wait for crypt threads to stop accessing space */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 20e8303f76417..95011ae61250d 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -321,13 +321,21 @@ struct fil_space_t { /*!< true if this space is currently in unflushed_spaces */ ibool is_corrupt; + /*!< true if tablespace corrupted */ bool printed_compression_failure; /*!< true if we have already printed compression failure */ + fil_space_crypt_t* crypt_data; + /*!< tablespace crypt data or NULL */ + bool page_0_crypt_read; + /*!< tablespace crypt data has been + read */ + ulint file_block_size; + /*!< file system block size */ + UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - fil_space_crypt_t* crypt_data; - ulint file_block_size;/*!< file system block size */ + ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -471,7 +479,8 @@ fil_space_create( ulint zip_size,/*!< in: compressed page size, or 0 for uncompressed tablespaces */ ulint purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table); /*!< in: true if create table */ /*******************************************************************//** Assigns a new space id for a new single-table tablespace. This works simply by diff --git a/storage/xtradb/include/srv0mon.h b/storage/xtradb/include/srv0mon.h index 33ae7749ca51d..3b030d56d2908 100644 --- a/storage/xtradb/include/srv0mon.h +++ b/storage/xtradb/include/srv0mon.h @@ -167,6 +167,7 @@ enum monitor_id_t { MONITOR_OVLD_INDEX_PAGES_WRITTEN, MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN, MONITOR_OVLD_PAGES_READ, + MONITOR_OVLD_PAGES0_READ, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS_AVOIDED, MONITOR_OVLD_BYTE_READ, diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index bd7a8731bdf54..d95adf00814a3 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -186,6 +186,9 @@ struct srv_stats_t { /** Number of lock waits that have been up to max time (i.e.) lock wait timeout */ ulint_ctr_1_t n_lock_max_wait_time; + + /** Number of times page 0 is read from tablespace */ + ulint_ctr_64_t page0_read; }; extern const char* srv_main_thread_op_info; @@ -1161,7 +1164,8 @@ struct export_var_t{ ulint innodb_os_log_pending_fsyncs; /*!< fil_n_pending_log_flushes */ ulint innodb_page_size; /*!< UNIV_PAGE_SIZE */ ulint innodb_pages_created; /*!< buf_pool->stat.n_pages_created */ - ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read */ + ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read*/ + ulint innodb_page0_read; /*!< srv_stats.page0_read */ ulint innodb_pages_written; /*!< buf_pool->stat.n_pages_written */ ib_int64_t innodb_purge_trx_id; ib_int64_t innodb_purge_undo_no; diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index ba2e0047fe934..059254a3c4913 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -3293,7 +3293,7 @@ fil_wait_crypt_bg_threads( uint last = start; if (table->space != 0) { - fil_space_crypt_mark_space_closing(table->space); + fil_space_crypt_mark_space_closing(table->space, table->crypt_data); } while (table->n_ref_count > 0) { @@ -4225,6 +4225,12 @@ row_drop_table_for_mysql( rw_lock_x_unlock(dict_index_get_lock(index)); } + /* If table has not yet have crypt_data, try to read it to + make freeing the table easier. */ + if (!table->crypt_data) { + table->crypt_data = fil_space_get_crypt_data(table->space); + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index e72868c645047..1e0d21d4a9ede 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -309,6 +309,12 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_READ}, + {"buffer_pages0_read", "buffer", + "Number of page 0 read (innodb_pages0_read)", + static_cast( + MONITOR_EXISTING | MONITOR_DEFAULT_ON), + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES0_READ}, + {"buffer_index_sec_rec_cluster_reads", "buffer", "Number of secondary record reads triggered cluster read", static_cast( @@ -1715,6 +1721,11 @@ srv_mon_process_existing_counter( value = stat.n_pages_read; break; + /* innodb_pages0_read */ + case MONITOR_OVLD_PAGES0_READ: + value = srv_stats.page0_read; + break; + /* Number of times secondary index lookup triggered cluster lookup */ case MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS: value = srv_stats.n_sec_rec_cluster_reads; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 58b442e1aa80f..a836442eb7041 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1951,6 +1951,7 @@ srv_export_innodb_status(void) export_vars.innodb_pages_created = stat.n_pages_created; export_vars.innodb_pages_read = stat.n_pages_read; + export_vars.innodb_page0_read = srv_stats.page0_read; export_vars.innodb_pages_written = stat.n_pages_written; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ca1eb194b0036..6b1e8c39b3818 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -705,7 +705,8 @@ create_log_files( logfilename, SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* this is create */); ut_a(fil_validate()); logfile0 = fil_node_create( @@ -727,7 +728,7 @@ create_log_files( #ifdef UNIV_LOG_ARCHIVE /* Create the file space object for archived logs. */ fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1, - 0, FIL_LOG, NULL /* no encryption yet */); + 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); #endif log_group_init(0, srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE, @@ -853,7 +854,7 @@ open_or_create_data_files( ulint space; ulint rounded_size_pages; char name[10000]; - fil_space_crypt_t* crypt_data; + fil_space_crypt_t* crypt_data=NULL; if (srv_n_data_files >= 1000) { @@ -1184,18 +1185,20 @@ open_or_create_data_files( } *sum_of_new_sizes += srv_data_file_sizes[i]; - - crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } ret = os_file_close(files[i]); ut_a(ret); if (i == 0) { + if (!crypt_data) { + crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); + } + flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + fil_space_create(name, 0, flags, FIL_TABLESPACE, - crypt_data); - crypt_data = NULL; + crypt_data, (*create_new_db) == true); } ut_a(fil_validate()); @@ -1342,7 +1345,8 @@ srv_undo_tablespace_open( /* Set the compressed page size to 0 (non-compressed) */ flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); fil_space_create(name, space, flags, FIL_TABLESPACE, - NULL /* no encryption */); + NULL /* no encryption */, + true /* create */); ut_a(fil_validate()); @@ -2371,7 +2375,8 @@ innobase_start_or_create_for_mysql(void) SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* create */); ut_a(fil_validate()); @@ -2393,7 +2398,8 @@ innobase_start_or_create_for_mysql(void) /* Create the file space object for archived logs. Under MySQL, no archiving ever done. */ fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1, - 0, FIL_LOG, NULL /* no encryption yet */); + 0, FIL_LOG, NULL /* no encryption yet */, + true /* create */); #endif /* UNIV_LOG_ARCHIVE */ log_group_init(0, i, srv_log_file_size * UNIV_PAGE_SIZE, SRV_LOG_SPACE_FIRST_ID, From 1d55cfce10fc78c386f4444b20a21c664a81a297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 22 Sep 2016 20:57:15 +0300 Subject: [PATCH 054/295] Do not use os_file_read() directly for reading first page of the tablespace. Instead use fil_read() with syncronous setting. Fix test failures and mask tablespace number as it could change in concurrent mtr runs. --- .../encryption/r/innodb-bad-key-change.result | 4 +-- .../encryption/r/innodb_lotoftables.result | 6 ++-- .../encryption/t/innodb-bad-key-change.test | 3 +- .../suite/innodb/r/innodb_monitor.result | 1 + .../r/innodb_skip_innodb_is_tables.result | 1 + .../r/innodb_monitor_reset_all_basic.result | 1 + .../r/innodb_monitor_reset_basic.result | 1 + storage/innobase/fil/fil0fil.cc | 30 ++++++++----------- storage/xtradb/fil/fil0fil.cc | 30 ++++++++----------- 9 files changed, 37 insertions(+), 40 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index ec8dee89230ec..0d73aa520ab7e 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -36,7 +36,7 @@ SELECT * FROM t1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 192 Table test/t1 in tablespace 6 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table test/t1 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DROP TABLE t1; @@ -52,7 +52,7 @@ SELECT * FROM t2; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 192 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table test/t2 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SELECT * FROM t2 where id = 1; diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result index 5e3eaef550f41..34f2684253e34 100644 --- a/mysql-test/suite/encryption/r/innodb_lotoftables.result +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -11,13 +11,13 @@ create database innodb_encrypted_1; use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 3 set autocommit=0; set autocommit=1; commit work; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 3 # should be 100 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; COUNT(*) @@ -127,7 +127,7 @@ Variable_name Value Innodb_pages0_read 203 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 203 +Innodb_pages0_read 303 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; COUNT(*) 100 diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index 9180fb12085ac..bc5e3b459d654 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -54,6 +54,7 @@ SELECT * FROM t1; --error ER_GET_ERRMSG SELECT * FROM t1; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt @@ -79,7 +80,7 @@ INSERT INTO t2 VALUES ('foobar',1,2); --error ER_GET_ERRMSG SELECT * FROM t2; ---replace_regex /.*tablespace [0-9]*// +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG SELECT * FROM t2 where id = 1; diff --git a/mysql-test/suite/innodb/r/innodb_monitor.result b/mysql-test/suite/innodb/r/innodb_monitor.result index 8c580348e1acf..33acfbdf996a0 100644 --- a/mysql-test/suite/innodb/r/innodb_monitor.result +++ b/mysql-test/suite/innodb/r/innodb_monitor.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result index 3280726729fa0..7568dc9e369ac 100644 --- a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result +++ b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result @@ -75,6 +75,7 @@ buffer_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NUL buffer_index_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of index pages written (innodb_index_pages_written) buffer_non_index_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of non index pages written (innodb_non_index_pages_written) buffer_pages_read buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages read (innodb_pages_read) +buffer_pages0_read buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of page 0 read (innodb_pages0_read) buffer_index_sec_rec_cluster_reads buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of secondary record reads triggered cluster read buffer_index_sec_rec_cluster_reads_avoided buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of secondary record reads avoided triggering cluster read buffer_data_reads buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Amount of data read in bytes (innodb_data_reads) diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result index 6c7051dc3d076..f7a02ed354865 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result index 6c7051dc3d076..f7a02ed354865 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 43aecf45b6052..d73539cb84d4c 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -669,6 +669,7 @@ fil_node_open_file( page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); + srv_stats.page0_read.add(1); space_id = fsp_header_get_space_id(page); flags = fsp_header_get_flags(page); @@ -7244,30 +7245,27 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); + mutex_exit(&fil_system->mutex); + if (space != NULL) { /* If we have not yet read the page0 of this tablespace we will do it now. */ if (!space->crypt_data && !space->page_0_crypt_read) { - ulint flags; - ulint space_id; - lsn_t min_flushed_lsn; - lsn_t max_flushed_lsn; + ulint space_id = space->id; fil_node_t* node; ut_a(space->crypt_data == NULL); node = UT_LIST_GET_FIRST(space->chain); - fil_node_prepare_for_io(node, fil_system, space); - - const char* msg = fil_read_first_page(node->handle, - false, - &flags, - &space_id, - &min_flushed_lsn, - &max_flushed_lsn, - &space->crypt_data); - - fil_node_complete_io(node, fil_system, OS_FILE_READ); + byte *buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + byte *page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); + fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, + NULL, NULL); + ulint flags = fsp_header_get_flags(page); + ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(flags), NULL); + space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); + ut_free(buf); ib_logf(IB_LOG_LEVEL_INFO, "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", @@ -7287,8 +7285,6 @@ fil_space_get_crypt_data( ut_ad(space->page_0_crypt_read); } - mutex_exit(&fil_system->mutex); - return(crypt_data); } diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 1db342d2bc741..1dbc33340cfd2 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -672,6 +672,7 @@ fil_node_open_file( page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); + srv_stats.page0_read.add(1); space_id = fsp_header_get_space_id(page); flags = fsp_header_get_flags(page); @@ -7342,30 +7343,27 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); + mutex_exit(&fil_system->mutex); + if (space != NULL) { /* If we have not yet read the page0 of this tablespace we will do it now. */ if (!space->crypt_data && !space->page_0_crypt_read) { - ulint flags; - ulint space_id; - lsn_t min_flushed_lsn; - lsn_t max_flushed_lsn; + ulint space_id = space->id; fil_node_t* node; ut_a(space->crypt_data == NULL); node = UT_LIST_GET_FIRST(space->chain); - fil_node_prepare_for_io(node, fil_system, space); - - const char* msg = fil_read_first_page(node->handle, - false, - &flags, - &space_id, - &min_flushed_lsn, - &max_flushed_lsn, - &space->crypt_data); - - fil_node_complete_io(node, fil_system, OS_FILE_READ); + byte *buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + byte *page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); + fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, + NULL, NULL); + ulint flags = fsp_header_get_flags(page); + ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(flags), NULL); + space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); + ut_free(buf); ib_logf(IB_LOG_LEVEL_INFO, "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", @@ -7385,8 +7383,6 @@ fil_space_get_crypt_data( ut_ad(space->page_0_crypt_read); } - mutex_exit(&fil_system->mutex); - return(crypt_data); } From e136aa1ba7d6a0d4cc38e12a4cdcffaa5f0f01b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 23 Sep 2016 09:11:11 +0300 Subject: [PATCH 055/295] Fix test failure. Need to mask more tablespace numbers as they are not consistent on parallel mtr runs. --- .../encryption/r/innodb-bad-key-change.result | 8 ++++++-- .../encryption/t/innodb-bad-key-change.test | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index 0d73aa520ab7e..b613cca47be6b 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -41,8 +41,12 @@ Warning 192 Table test/t1 is encrypted but encryption service or used key_id is Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DROP TABLE t1; Warnings: -Warning 192 Table in tablespace 6 encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. -Warning 192 Table in tablespace 6 encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. +Warning 192 Table in tablespace encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. +Warning 192 Table in tablespace encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. +SHOW WARNINGS; +Level Code Message +Warning 192 Table in tablespace encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. +Warning 192 Table in tablespace encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. # Start server with keys.txt CREATE TABLE t2 (c VARCHAR(8), id int not null primary key, b int, key(b)) ENGINE=InnoDB ENCRYPTED=YES; INSERT INTO t2 VALUES ('foobar',1,2); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index bc5e3b459d654..f4f3ecf69ea79 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -59,7 +59,11 @@ SHOW WARNINGS; -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt -- source include/restart_mysqld.inc + +--replace_regex /tablespace [0-9]*/tablespace / DROP TABLE t1; +--replace_regex /tablespace [0-9]*/tablespace / +SHOW WARNINGS; # # MDEV-8591: Database page corruption on disk or a failed space, Assertion failure in file buf0buf.cc @@ -84,34 +88,45 @@ SELECT * FROM t2; SHOW WARNINGS; --error ER_GET_ERRMSG SELECT * FROM t2 where id = 1; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG SELECT * FROM t2 where b = 1; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG INSERT INTO t2 VALUES ('tmp',3,3); +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG DELETE FROM t2 where b = 3; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG DELETE FROM t2 where id = 3; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG UPDATE t2 set b = b +1; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; OPTIMIZE TABLE t2; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG ALTER TABLE t2 ADD COLUMN c INT; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; ANALYZE TABLE t2; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG TRUNCATE TABLE t2; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --error ER_GET_ERRMSG DROP TABLE t2; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; --echo @@ -120,4 +135,5 @@ SHOW WARNINGS; -- source include/restart_mysqld.inc DROP TABLE t2; +--replace_regex /tablespace [0-9]*/tablespace / SHOW WARNINGS; From 5d001d13c25c1de1178cc9109e829ef577c9ed73 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 23 Sep 2016 17:28:38 +1000 Subject: [PATCH 056/295] MDEV-10832 - Out of tree build: mysql_install_db to see all .sql files (#237) * Out of tree build: mysql_install_db to see all .sql files Since MDEV-7875 (da0991c6), not all sql source files are in the source directory, maria_add_gis_sp_bootstrap.sql is in the build directory. This corrects mysql_install_db{.sh} to be aware of the differing locations. Signed-off-by: Daniel Black * Out of tree build: scripts/mysql_install_db.pl.in Signed-off-by: Daniel Black --- scripts/mysql_install_db.pl.in | 46 +++++++++++++++++----------------- scripts/mysql_install_db.sh | 19 ++++++++------ 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/mysql_install_db.pl.in b/scripts/mysql_install_db.pl.in index 8b99b3564e4f2..3ad733884fbd3 100644 --- a/scripts/mysql_install_db.pl.in +++ b/scripts/mysql_install_db.pl.in @@ -297,17 +297,19 @@ parse_arguments($opt, 'PICK-ARGS-FROM-ARGV', @ARGV); # ---------------------------------------------------------------------- # FIXME $extra_bindir is not used -my ($bindir,$extra_bindir,$mysqld,$pkgdatadir,$mysqld_opt,$scriptdir); +my ($bindir,$extra_bindir,$mysqld,$srcpkgdatadir,$buildpkgdatadir,$mysqld_opt, + $scriptdir); if ( $opt->{srcdir} ) { - $opt->{basedir} = $opt->{builddir}; - $bindir = "$opt->{basedir}/client"; - $extra_bindir = "$opt->{basedir}/extra"; - $mysqld = "$opt->{basedir}/sql/mysqld"; - $mysqld_opt = "--language=$opt->{srcdir}/sql/share/english"; - $pkgdatadir = "$opt->{srcdir}/scripts"; - $scriptdir = "$opt->{srcdir}/scripts"; + $opt->{basedir} = $opt->{builddir}; + $bindir = "$opt->{basedir}/client"; + $extra_bindir = "$opt->{basedir}/extra"; + $mysqld = "$opt->{basedir}/sql/mysqld"; + $mysqld_opt = "--language=$opt->{srcdir}/sql/share/english"; + $srcpkgdatadir = "$opt->{srcdir}/scripts"; + $buildpkgdatadir = "$opt->{builddir}/scripts"; + $scriptdir = "$opt->{srcdir}/scripts"; } elsif ( $opt->{basedir} ) { @@ -317,18 +319,20 @@ elsif ( $opt->{basedir} ) "libexec","sbin","bin") || # ,"sql" find_in_basedir($opt,"file","mysqld-nt", "bin"); # ,"sql" - $pkgdatadir = find_in_basedir($opt,"dir","fill_help_tables.sql", + $srcpkgdatadir = find_in_basedir($opt,"dir","fill_help_tables.sql", "share","share/mysql"); # ,"scripts" + $buildpkgdir = $srcpkgdatadir; $scriptdir = "$opt->{basedir}/scripts"; } else { - $opt->{basedir} = '@prefix@'; - $bindir = '@bindir@'; - $extra_bindir = $bindir; - $mysqld = '@libexecdir@/mysqld'; - $pkgdatadir = '@pkgdatadir@'; - $scriptdir = '@scriptdir@'; + $opt->{basedir} = '@prefix@'; + $bindir = '@bindir@'; + $extra_bindir = $bindir; + $mysqld = '@libexecdir@/mysqld'; + $srcpkgdatadir = '@pkgdatadir@'; + $buildpkgdatadir = '@pkgdatadir@'; + $scriptdir = '@scriptdir@'; } unless ( $opt->{ldata} ) @@ -336,19 +340,15 @@ unless ( $opt->{ldata} ) $opt->{ldata} = '@localstatedir@'; } -if ( $opt->{srcdir} ) -{ - $pkgdatadir = "$opt->{srcdir}/scripts"; -} # ---------------------------------------------------------------------- # Set up paths to SQL scripts required for bootstrap # ---------------------------------------------------------------------- -my $fill_help_tables = "$pkgdatadir/fill_help_tables.sql"; -my $create_system_tables = "$pkgdatadir/mysql_system_tables.sql"; -my $fill_system_tables = "$pkgdatadir/mysql_system_tables_data.sql"; -my $maria_add_gis_sp = "$pkgdatadir/maria_add_gis_sp_bootstrap.sql"; +my $fill_help_tables = "$srcpkgdatadir/fill_help_tables.sql"; +my $create_system_tables = "$srcpkgdatadir/mysql_system_tables.sql"; +my $fill_system_tables = "$srcpkgdatadir/mysql_system_tables_data.sql"; +my $maria_add_gis_sp = "$buildpkgdatadir/maria_add_gis_sp_bootstrap.sql"; foreach my $f ( $fill_help_tables,$create_system_tables,$fill_system_tables,$maria_add_gis_sp ) { diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 1daa1bf58eed1..b03c1059605d0 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -270,7 +270,8 @@ then extra_bindir="$basedir/extra" mysqld="$basedir/sql/mysqld" langdir="$basedir/sql/share/english" - pkgdatadir="$srcdir/scripts" + srcpkgdatadir="$srcdir/scripts" + buildpkgdatadir="$builddir/scripts" scriptdir="$srcdir/scripts" elif test -n "$basedir" then @@ -288,7 +289,8 @@ then cannot_find_file errmsg.sys $basedir/share/english $basedir/share/mysql/english exit 1 fi - pkgdatadir=`find_in_basedir --dir fill_help_tables.sql share share/mysql` + srcpkgdatadir=`find_in_basedir --dir fill_help_tables.sql share share/mysql` + buildpkgdatadir=$srcpkgdatadir if test -z "$pkgdatadir" then cannot_find_file fill_help_tables.sql $basedir/share $basedir/share/mysql @@ -300,16 +302,17 @@ else bindir="@bindir@" extra_bindir="$bindir" mysqld="@libexecdir@/mysqld" - pkgdatadir="@pkgdatadir@" + srcpkgdatadir="@pkgdatadir@" + buildpkgdatadir="@pkgdatadir@" scriptdir="@scriptdir@" fi # Set up paths to SQL scripts required for bootstrap -fill_help_tables="$pkgdatadir/fill_help_tables.sql" -create_system_tables="$pkgdatadir/mysql_system_tables.sql" -create_system_tables2="$pkgdatadir/mysql_performance_tables.sql" -fill_system_tables="$pkgdatadir/mysql_system_tables_data.sql" -maria_add_gis_sp="$pkgdatadir/maria_add_gis_sp_bootstrap.sql" +fill_help_tables="$srcpkgdatadir/fill_help_tables.sql" +create_system_tables="$srcpkgdatadir/mysql_system_tables.sql" +create_system_tables2="$srcpkgdatadir/mysql_performance_tables.sql" +fill_system_tables="$srcpkgdatadir/mysql_system_tables_data.sql" +maria_add_gis_sp="$buildpkgdatadir/maria_add_gis_sp_bootstrap.sql" for f in "$fill_help_tables" "$create_system_tables" "$create_system_tables2" "$fill_system_tables" "$maria_add_gis_sp" do From 677c44f0c37973ad70550d9b807781e688764fae Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 23 Sep 2016 20:27:58 +0200 Subject: [PATCH 057/295] MDEV-10775 System table in InnoDB format allowed in MariaDB could lead to crash when opening a system table for a SELECT-like read, pretend (for the sake of engines) it's SQLCOM_SELECT --- mysql-test/suite/innodb/r/system_tables.result | 8 ++++++++ mysql-test/suite/innodb/t/system_tables.test | 12 ++++++++++++ sql/sql_base.cc | 1 + 3 files changed, 21 insertions(+) create mode 100644 mysql-test/suite/innodb/r/system_tables.result create mode 100644 mysql-test/suite/innodb/t/system_tables.test diff --git a/mysql-test/suite/innodb/r/system_tables.result b/mysql-test/suite/innodb/r/system_tables.result new file mode 100644 index 0000000000000..79a24f7e4556e --- /dev/null +++ b/mysql-test/suite/innodb/r/system_tables.result @@ -0,0 +1,8 @@ +alter table mysql.time_zone_name engine=InnoDB; +create table envois3 (starttime datetime) engine=InnoDB; +insert envois3 values ('2008-08-11 22:43:00'); +select convert_tz(starttime,'UTC','Europe/Moscow') starttime from envois3; +starttime +2008-08-12 02:43:00 +drop table envois3; +alter table mysql.time_zone_name engine=MyISAM; diff --git a/mysql-test/suite/innodb/t/system_tables.test b/mysql-test/suite/innodb/t/system_tables.test new file mode 100644 index 0000000000000..90cb8c59fbd1d --- /dev/null +++ b/mysql-test/suite/innodb/t/system_tables.test @@ -0,0 +1,12 @@ +--source include/have_innodb.inc + +# +# MDEV-10775 System table in InnoDB format allowed in MariaDB could lead to crash +# +alter table mysql.time_zone_name engine=InnoDB; +create table envois3 (starttime datetime) engine=InnoDB; +insert envois3 values ('2008-08-11 22:43:00'); +--source include/restart_mysqld.inc +select convert_tz(starttime,'UTC','Europe/Moscow') starttime from envois3; +drop table envois3; +alter table mysql.time_zone_name engine=MyISAM; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 6ec9401636607..b9cc4e5d69a51 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -9223,6 +9223,7 @@ open_system_tables_for_read(THD *thd, TABLE_LIST *table_list, */ lex->reset_n_backup_query_tables_list(&query_tables_list_backup); thd->reset_n_backup_open_tables_state(backup); + thd->lex->sql_command= SQLCOM_SELECT; if (open_and_lock_tables(thd, table_list, FALSE, MYSQL_OPEN_IGNORE_FLUSH | From 7d7b92c107aaa36ac5e2dc142eb9e6a6181ff35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 24 Sep 2016 14:21:27 +0300 Subject: [PATCH 058/295] Disable encryption info and first page read info for every tablespace on product builds. --- storage/innobase/fil/fil0fil.cc | 8 ++++++-- storage/xtradb/fil/fil0fil.cc | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d73539cb84d4c..f1fdb0f281eeb 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1251,12 +1251,14 @@ fil_space_create( space->page_0_crypt_read = true; } +#ifdef UNIV_DEBUG ib_logf(IB_LOG_LEVEL_INFO, - "Created tablespace for space %lu name %s key_id %u encryption %d\n", + "Created tablespace for space %lu name %s key_id %u encryption %d.", space->id, space->name, space->crypt_data ? space->crypt_data->key_id : 0, space->crypt_data ? space->crypt_data->encryption : 0); +#endif rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); @@ -7267,13 +7269,15 @@ fil_space_get_crypt_data( space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); +#ifdef UNIV_DEBUG ib_logf(IB_LOG_LEVEL_INFO, - "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", + "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d.", space_id, space->name, space->crypt_data ? space->crypt_data->key_id : 0, space->crypt_data ? space->crypt_data->encryption : 0, node->handle); +#endif ut_a(space->id == space_id); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 1dbc33340cfd2..81f26b276623a 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1296,12 +1296,14 @@ fil_space_create( space->page_0_crypt_read = true; } +#ifdef UNIV_DEBUG ib_logf(IB_LOG_LEVEL_INFO, - "Created tablespace for space %lu name %s key_id %u encryption %d\n", + "Created tablespace for space %lu name %s key_id %u encryption %d.", space->id, space->name, space->crypt_data ? space->crypt_data->key_id : 0, space->crypt_data ? space->crypt_data->encryption : 0); +#endif UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); @@ -7365,13 +7367,15 @@ fil_space_get_crypt_data( space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); +#ifdef UNIV_DEBUG ib_logf(IB_LOG_LEVEL_INFO, - "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", + "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d.", space_id, space->name, space->crypt_data ? space->crypt_data->key_id : 0, space->crypt_data ? space->crypt_data->encryption : 0, node->handle); +#endif ut_a(space->id == space_id); From 943443137460cab6499c032f0e5a03256daa9571 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sat, 24 Sep 2016 13:50:42 +0200 Subject: [PATCH 059/295] Fix free() after my_malloc() (should be my_free()). Signed-off-by: Kristian Nielsen --- tests/async_queries.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/async_queries.c b/tests/async_queries.c index 75229eec4b43c..c91edf5bb3b24 100644 --- a/tests/async_queries.c +++ b/tests/async_queries.c @@ -425,7 +425,7 @@ main(int argc, char *argv[]) event_dispatch(); - free(sds); + my_free(sds); mysql_library_end(); From d30809a3cde823ad696304a941afe5a562bfa3ed Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 26 Sep 2016 09:40:47 +0400 Subject: [PATCH 060/295] MDEV-10832 Out of tree build: mysql_install_db to see all .sql files. One line in mysql_install_db.sh was left unfixed. --- scripts/mysql_install_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index b03c1059605d0..dbb220c3a0dae 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -291,7 +291,7 @@ then fi srcpkgdatadir=`find_in_basedir --dir fill_help_tables.sql share share/mysql` buildpkgdatadir=$srcpkgdatadir - if test -z "$pkgdatadir" + if test -z "$srcpkgdatadir" then cannot_find_file fill_help_tables.sql $basedir/share $basedir/share/mysql exit 1 From f620da194befe4506679ad7f1d0725796fad7de0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 24 Sep 2016 01:17:35 +0200 Subject: [PATCH 061/295] MDEV-10725 Server 10.1.17 fails to build using clang with c++11 my_offsetof() returns a difference of two pointers, it must use the appropriate return type (my_ptrdiff_t, not size_t) --- include/my_global.h | 3 +-- sql/parse_file.h | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/my_global.h b/include/my_global.h index dce38a124c125..0c15478439f38 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -857,8 +857,7 @@ typedef long long my_ptrdiff_t; and related routines are refactored. */ -#define my_offsetof(TYPE, MEMBER) \ - ((size_t)((char *)&(((TYPE *)0x10)->MEMBER) - (char*)0x10)) +#define my_offsetof(TYPE, MEMBER) PTR_BYTE_DIFF(&((TYPE *)0x10)->MEMBER, 0x10) #define NullS (char *) 0 diff --git a/sql/parse_file.h b/sql/parse_file.h index 2a0266e98b7eb..83a8eabcf5f72 100644 --- a/sql/parse_file.h +++ b/sql/parse_file.h @@ -42,9 +42,9 @@ enum file_opt_type { struct File_option { - LEX_STRING name; /**< Name of the option */ - int offset; /**< offset to base address of value */ - file_opt_type type; /**< Option type */ + LEX_STRING name; /**< Name of the option */ + my_ptrdiff_t offset; /**< offset to base address of value */ + file_opt_type type; /**< Option type */ }; From 8483659f4f017285a878ce563c1c756ee3e4d3dc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 24 Sep 2016 10:06:58 +0200 Subject: [PATCH 062/295] report correct write error on log writes --- sql/log.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/log.cc b/sql/log.cc index da45a844bb3f1..bb8f06c80f7fe 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -2863,7 +2863,7 @@ bool MYSQL_QUERY_LOG::write(THD *thd, time_t current_time, if (! write_error) { write_error= 1; - sql_print_error(ER(ER_ERROR_ON_WRITE), name, error); + sql_print_error(ER(ER_ERROR_ON_WRITE), name, tmp_errno); } } } From c91fdb66dbd26d832073e7b99075bfd0b5b9da11 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 26 Sep 2016 13:03:02 +0200 Subject: [PATCH 063/295] Windows , mtr : allow cdb to print core dumps also if --parallel > 1 --- mysql-test/lib/My/CoreDump.pm | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mysql-test/lib/My/CoreDump.pm b/mysql-test/lib/My/CoreDump.pm index 0e90967ef9545..f9f7b3d8d4b23 100644 --- a/mysql-test/lib/My/CoreDump.pm +++ b/mysql-test/lib/My/CoreDump.pm @@ -261,11 +261,7 @@ sub show { # On Windows, rely on cdb to be there... if (IS_WINDOWS) { - # Starting cdb is unsafe when used with --parallel > 1 option - if ( $parallel < 2 ) - { - _cdb($core_name); - } + _cdb($core_name); return; } From d61e5260fb9983ea8dff539b23a6d0a150c2065c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Sep 2016 17:48:08 +0200 Subject: [PATCH 064/295] MDEV-10441 Document the server_audit_loc_info variable fix PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT plugin thdvars to work. use that for server_audit_loc_info --- .../suite/plugins/r/server_audit.result | 3 - .../plugins/r/thread_pool_server_audit.result | 3 - plugin/server_audit/server_audit.c | 5 +- sql/sql_plugin.cc | 151 ++++++++---------- 4 files changed, 69 insertions(+), 93 deletions(-) diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index 2dcfa107103a3..c807107534dba 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -8,7 +8,6 @@ server_audit_file_rotate_now OFF server_audit_file_rotate_size 1000000 server_audit_file_rotations 9 server_audit_incl_users -server_audit_loc_info server_audit_logging OFF server_audit_mode 0 server_audit_output_type file @@ -72,7 +71,6 @@ server_audit_file_rotate_now OFF server_audit_file_rotate_size 1000000 server_audit_file_rotations 9 server_audit_incl_users odin, root, dva, tri -server_audit_loc_info server_audit_logging ON server_audit_mode 0 server_audit_output_type file @@ -218,7 +216,6 @@ server_audit_file_rotate_now OFF server_audit_file_rotate_size 1000000 server_audit_file_rotations 9 server_audit_incl_users odin, root, dva, tri -server_audit_loc_info server_audit_logging ON server_audit_mode 1 server_audit_output_type file diff --git a/mysql-test/suite/plugins/r/thread_pool_server_audit.result b/mysql-test/suite/plugins/r/thread_pool_server_audit.result index 2dcfa107103a3..c807107534dba 100644 --- a/mysql-test/suite/plugins/r/thread_pool_server_audit.result +++ b/mysql-test/suite/plugins/r/thread_pool_server_audit.result @@ -8,7 +8,6 @@ server_audit_file_rotate_now OFF server_audit_file_rotate_size 1000000 server_audit_file_rotations 9 server_audit_incl_users -server_audit_loc_info server_audit_logging OFF server_audit_mode 0 server_audit_output_type file @@ -72,7 +71,6 @@ server_audit_file_rotate_now OFF server_audit_file_rotate_size 1000000 server_audit_file_rotations 9 server_audit_incl_users odin, root, dva, tri -server_audit_loc_info server_audit_logging ON server_audit_mode 0 server_audit_output_type file @@ -218,7 +216,6 @@ server_audit_file_rotate_now OFF server_audit_file_rotate_size 1000000 server_audit_file_rotations 9 server_audit_incl_users odin, root, dva, tri -server_audit_loc_info server_audit_logging ON server_audit_mode 1 server_audit_output_type file diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 30b7cdb5dcb0a..95150c82f25ef 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -429,9 +429,8 @@ static MYSQL_SYSVAR_UINT(query_log_limit, query_log_limit, char locinfo_ini_value[sizeof(struct connection_info)+4]; static MYSQL_THDVAR_STR(loc_info, - PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, - "Auxiliary info.", NULL, NULL, - locinfo_ini_value); + PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT | PLUGIN_VAR_MEMALLOC, + "Internal info", NULL, NULL, locinfo_ini_value); static const char *syslog_facility_names[]= { diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index c8c8c8ba32440..2ec67a8974678 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2756,6 +2756,22 @@ static st_bookmark *find_bookmark(const char *plugin, const char *name, } +static size_t var_storage_size(int flags) +{ + switch (flags & PLUGIN_VAR_TYPEMASK) { + case PLUGIN_VAR_BOOL: return sizeof(my_bool); + case PLUGIN_VAR_INT: return sizeof(int); + case PLUGIN_VAR_LONG: return sizeof(long); + case PLUGIN_VAR_ENUM: return sizeof(long); + case PLUGIN_VAR_LONGLONG: return sizeof(ulonglong); + case PLUGIN_VAR_SET: return sizeof(ulonglong); + case PLUGIN_VAR_STR: return sizeof(char*); + case PLUGIN_VAR_DOUBLE: return sizeof(double); + default: DBUG_ASSERT(0); return 0; + } +} + + /* returns a bookmark for thd-local variables, creating if neccessary. returns null for non thd-local variables. @@ -2764,39 +2780,13 @@ static st_bookmark *find_bookmark(const char *plugin, const char *name, static st_bookmark *register_var(const char *plugin, const char *name, int flags) { - uint length= strlen(plugin) + strlen(name) + 3, size= 0, offset, new_size; + uint length= strlen(plugin) + strlen(name) + 3, size, offset, new_size; st_bookmark *result; char *varname, *p; - if (!(flags & PLUGIN_VAR_THDLOCAL)) - return NULL; - - switch (flags & PLUGIN_VAR_TYPEMASK) { - case PLUGIN_VAR_BOOL: - size= sizeof(my_bool); - break; - case PLUGIN_VAR_INT: - size= sizeof(int); - break; - case PLUGIN_VAR_LONG: - case PLUGIN_VAR_ENUM: - size= sizeof(long); - break; - case PLUGIN_VAR_LONGLONG: - case PLUGIN_VAR_SET: - size= sizeof(ulonglong); - break; - case PLUGIN_VAR_STR: - size= sizeof(char*); - break; - case PLUGIN_VAR_DOUBLE: - size= sizeof(double); - break; - default: - DBUG_ASSERT(0); - return NULL; - }; + DBUG_ASSERT(flags & PLUGIN_VAR_THDLOCAL); + size= var_storage_size(flags); varname= ((char*) my_alloca(length)); strxmov(varname + 1, plugin, "_", name, NullS); for (p= varname + 1; *p; p++) @@ -3005,25 +2995,17 @@ void sync_dynamic_session_variables(THD* thd, bool global_lock) */ for (idx= 0; idx < bookmark_hash.records; idx++) { - sys_var_pluginvar *pi; - sys_var *var; st_bookmark *v= (st_bookmark*) my_hash_element(&bookmark_hash,idx); if (v->version <= thd->variables.dynamic_variables_version) continue; /* already in thd->variables */ - if (!(var= intern_find_sys_var(v->key + 1, v->name_len)) || - !(pi= var->cast_pluginvar()) || - v->key[0] != plugin_var_bookmark_key(pi->plugin_var->flags)) - continue; - /* Here we do anything special that may be required of the data types */ - if ((pi->plugin_var->flags & PLUGIN_VAR_TYPEMASK) == PLUGIN_VAR_STR && - pi->plugin_var->flags & PLUGIN_VAR_MEMALLOC) + if ((v->key[0] & PLUGIN_VAR_TYPEMASK) == PLUGIN_VAR_STR && + v->key[0] & BOOKMARK_MEMALLOC) { - int offset= ((thdvar_str_t *)(pi->plugin_var))->offset; - char **pp= (char**) (thd->variables.dynamic_variables_ptr + offset); + char **pp= (char**) (thd->variables.dynamic_variables_ptr + v->offset); if (*pp) *pp= my_strdup(*pp, MYF(MY_WME|MY_FAE)); } @@ -3284,69 +3266,58 @@ bool sys_var_pluginvar::session_update(THD *thd, set_var *var) return false; } -bool sys_var_pluginvar::global_update(THD *thd, set_var *var) +static const void *var_def_ptr(st_mysql_sys_var *pv) { - DBUG_ASSERT(!is_readonly()); - mysql_mutex_assert_owner(&LOCK_global_system_variables); - - void *tgt= real_value_ptr(thd, var->type); - const void *src= &var->save_result; - - if (!var->value) - { - switch (plugin_var->flags & (PLUGIN_VAR_TYPEMASK | PLUGIN_VAR_THDLOCAL)) { + switch (pv->flags & (PLUGIN_VAR_TYPEMASK | PLUGIN_VAR_THDLOCAL)) { case PLUGIN_VAR_INT: - src= &((sysvar_uint_t*) plugin_var)->def_val; - break; + return &((sysvar_uint_t*) pv)->def_val; case PLUGIN_VAR_LONG: - src= &((sysvar_ulong_t*) plugin_var)->def_val; - break; + return &((sysvar_ulong_t*) pv)->def_val; case PLUGIN_VAR_LONGLONG: - src= &((sysvar_ulonglong_t*) plugin_var)->def_val; - break; + return &((sysvar_ulonglong_t*) pv)->def_val; case PLUGIN_VAR_ENUM: - src= &((sysvar_enum_t*) plugin_var)->def_val; - break; + return &((sysvar_enum_t*) pv)->def_val; case PLUGIN_VAR_SET: - src= &((sysvar_set_t*) plugin_var)->def_val; - break; + return &((sysvar_set_t*) pv)->def_val; case PLUGIN_VAR_BOOL: - src= &((sysvar_bool_t*) plugin_var)->def_val; - break; + return &((sysvar_bool_t*) pv)->def_val; case PLUGIN_VAR_STR: - src= &((sysvar_str_t*) plugin_var)->def_val; - break; + return &((sysvar_str_t*) pv)->def_val; case PLUGIN_VAR_DOUBLE: - src= &((sysvar_double_t*) plugin_var)->def_val; - break; + return &((sysvar_double_t*) pv)->def_val; case PLUGIN_VAR_INT | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_uint_t*) plugin_var)->def_val; - break; + return &((thdvar_uint_t*) pv)->def_val; case PLUGIN_VAR_LONG | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_ulong_t*) plugin_var)->def_val; - break; + return &((thdvar_ulong_t*) pv)->def_val; case PLUGIN_VAR_LONGLONG | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_ulonglong_t*) plugin_var)->def_val; - break; + return &((thdvar_ulonglong_t*) pv)->def_val; case PLUGIN_VAR_ENUM | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_enum_t*) plugin_var)->def_val; - break; + return &((thdvar_enum_t*) pv)->def_val; case PLUGIN_VAR_SET | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_set_t*) plugin_var)->def_val; - break; + return &((thdvar_set_t*) pv)->def_val; case PLUGIN_VAR_BOOL | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_bool_t*) plugin_var)->def_val; - break; + return &((thdvar_bool_t*) pv)->def_val; case PLUGIN_VAR_STR | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_str_t*) plugin_var)->def_val; - break; + return &((thdvar_str_t*) pv)->def_val; case PLUGIN_VAR_DOUBLE | PLUGIN_VAR_THDLOCAL: - src= &((thdvar_double_t*) plugin_var)->def_val; - break; + return &((thdvar_double_t*) pv)->def_val; default: DBUG_ASSERT(0); + return NULL; } - } +} + + +bool sys_var_pluginvar::global_update(THD *thd, set_var *var) +{ + DBUG_ASSERT(!is_readonly()); + mysql_mutex_assert_owner(&LOCK_global_system_variables); + + void *tgt= real_value_ptr(thd, var->type); + const void *src= &var->save_result; + + if (!var->value) + src= var_def_ptr(plugin_var); plugin_var->update(thd, plugin_var, tgt, src); return false; @@ -3713,7 +3684,18 @@ static int construct_options(MEM_ROOT *mem_root, struct st_plugin_int *tmp, *(int*)(opt + 1)= offset= v->offset; if (opt->flags & PLUGIN_VAR_NOCMDOPT) + { + char *val= global_system_variables.dynamic_variables_ptr + offset; + if (((opt->flags & PLUGIN_VAR_TYPEMASK) == PLUGIN_VAR_STR) && + (opt->flags & PLUGIN_VAR_MEMALLOC)) + { + char *def_val= *(char**)var_def_ptr(opt); + *(char**)val= def_val ? my_strdup(def_val, MYF(0)) : NULL; + } + else + memcpy(val, var_def_ptr(opt), var_storage_size(opt->flags)); continue; + } optname= (char*) memdup_root(mem_root, v->key + 1, (optnamelen= v->name_len) + 1); @@ -3912,9 +3894,10 @@ static int test_plugin_options(MEM_ROOT *tmp_root, struct st_plugin_int *tmp, *str->value= strdup_root(mem_root, *str->value); } + var= find_bookmark(plugin_name.str, o->name, o->flags); if (o->flags & PLUGIN_VAR_NOSYSVAR) continue; - if ((var= find_bookmark(plugin_name.str, o->name, o->flags))) + if (var) v= new (mem_root) sys_var_pluginvar(&chain, var->key + 1, o, tmp); else { From 4e2a0c34b02dd556c2a521555662ed993cdc66a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 26 Sep 2016 09:58:50 +0300 Subject: [PATCH 065/295] MDEV-10888: encryption.filekeys_emptyfile fails in buildbot with valgrind Problem was that length of the filekeys file was not checked and if length is less than OpenSSL_prefix_len uninitialized memory was accessed. --- mysql-test/suite/encryption/r/filekeys_tooshort.result | 10 ++++++++++ mysql-test/suite/encryption/t/filekeys-tooshort.enc | 1 + mysql-test/suite/encryption/t/filekeys_tooshort.opt | 3 +++ mysql-test/suite/encryption/t/filekeys_tooshort.test | 4 ++++ plugin/file_key_management/parser.cc | 2 +- 5 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/encryption/r/filekeys_tooshort.result create mode 100644 mysql-test/suite/encryption/t/filekeys-tooshort.enc create mode 100644 mysql-test/suite/encryption/t/filekeys_tooshort.opt create mode 100644 mysql-test/suite/encryption/t/filekeys_tooshort.test diff --git a/mysql-test/suite/encryption/r/filekeys_tooshort.result b/mysql-test/suite/encryption/r/filekeys_tooshort.result new file mode 100644 index 0000000000000..efa6609756371 --- /dev/null +++ b/mysql-test/suite/encryption/r/filekeys_tooshort.result @@ -0,0 +1,10 @@ +call mtr.add_suppression("Cannot decrypt .*tooshort.enc. Not encrypted"); +call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); +call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); +FOUND /Cannot decrypt .*tooshort.enc. Not encrypted/ in mysqld.1.err +create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") +select plugin_status from information_schema.plugins +where plugin_name = 'file_key_management'; +plugin_status +# Test checks if opening an too short filekeys does not crash the server. diff --git a/mysql-test/suite/encryption/t/filekeys-tooshort.enc b/mysql-test/suite/encryption/t/filekeys-tooshort.enc new file mode 100644 index 0000000000000..9849236c7f44c --- /dev/null +++ b/mysql-test/suite/encryption/t/filekeys-tooshort.enc @@ -0,0 +1 @@ +Salted__ \ No newline at end of file diff --git a/mysql-test/suite/encryption/t/filekeys_tooshort.opt b/mysql-test/suite/encryption/t/filekeys_tooshort.opt new file mode 100644 index 0000000000000..8999becc78df2 --- /dev/null +++ b/mysql-test/suite/encryption/t/filekeys_tooshort.opt @@ -0,0 +1,3 @@ +--loose-file-key-management-filekey=secret +--loose-file-key-management-filename=$MTR_SUITE_DIR/t/filekeys-tooshort.enc + diff --git a/mysql-test/suite/encryption/t/filekeys_tooshort.test b/mysql-test/suite/encryption/t/filekeys_tooshort.test new file mode 100644 index 0000000000000..b0e89675100e2 --- /dev/null +++ b/mysql-test/suite/encryption/t/filekeys_tooshort.test @@ -0,0 +1,4 @@ +let SEARCH_PATTERN=Cannot decrypt .*tooshort.enc. Not encrypted; +source filekeys_badtest.inc; + +--echo # Test checks if opening an too short filekeys does not crash the server. diff --git a/plugin/file_key_management/parser.cc b/plugin/file_key_management/parser.cc index 628412bc171ff..294ccc9ff790c 100644 --- a/plugin/file_key_management/parser.cc +++ b/plugin/file_key_management/parser.cc @@ -333,7 +333,7 @@ char* Parser::read_and_decrypt_file(const char *secret) // Check for file encryption uchar *decrypted; - if (is_prefix((char*)buffer, OpenSSL_prefix)) + if (file_size > OpenSSL_prefix_len && is_prefix((char*)buffer, OpenSSL_prefix)) { uchar key[OpenSSL_key_len]; uchar iv[OpenSSL_iv_len]; From 452e84952228a290b3c1fb16a8f60e2990aa8710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 26 Sep 2016 12:29:31 +0300 Subject: [PATCH 066/295] MDEV-10886: encryption.innodb-bad-key-change fails (crashes) in buildbot Problem was that NULL-pointer was accessed inside a macro when page read from tablespace is encrypted but decrypt fails because of incorrect key file. Removed unsafe macro using inlined function where used pointers are checked. --- storage/innobase/btr/btr0cur.cc | 9 +++++--- storage/innobase/btr/btr0scrub.cc | 20 +++++++++++------- storage/innobase/ibuf/ibuf0ibuf.cc | 14 +++++++++++-- storage/innobase/include/btr0btr.h | 14 ++++++++++--- storage/innobase/include/btr0btr.ic | 32 +++++++++++++++++++++++++++++ storage/xtradb/btr/btr0cur.cc | 9 +++++--- storage/xtradb/btr/btr0scrub.cc | 20 +++++++++++------- storage/xtradb/ibuf/ibuf0ibuf.cc | 14 +++++++++++-- storage/xtradb/include/btr0btr.h | 14 ++++++++++--- storage/xtradb/include/btr0btr.ic | 32 +++++++++++++++++++++++++++++ 10 files changed, 148 insertions(+), 30 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index eca232d81b425..18f193094e215 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -2138,9 +2138,12 @@ btr_cur_update_in_place( if (page_zip && !(flags & BTR_KEEP_IBUF_BITMAP) && !dict_index_is_clust(index) - && page_is_leaf(buf_block_get_frame(block))) { - /* Update the free bits in the insert buffer. */ - ibuf_update_free_bits_zip(block, mtr); + && block) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && page_is_leaf(frame)) { + /* Update the free bits in the insert buffer. */ + ibuf_update_free_bits_zip(block, mtr); + } } return(err); diff --git a/storage/innobase/btr/btr0scrub.cc b/storage/innobase/btr/btr0scrub.cc index e6acb7802f1e6..62a41d19768f6 100644 --- a/storage/innobase/btr/btr0scrub.cc +++ b/storage/innobase/btr/btr0scrub.cc @@ -368,12 +368,17 @@ btr_optimistic_scrub( /* We play safe and reset the free bits */ if (!dict_index_is_clust(index) && - page_is_leaf(buf_block_get_frame(block))) { + block != NULL) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && + page_is_leaf(frame)) { ibuf_reset_free_bits(block); + } } scrub_data->scrub_stat.page_reorganizations++; + return DB_SUCCESS; } @@ -488,9 +493,13 @@ btr_pessimistic_scrub( /* We play safe and reset the free bits * NOTE: need to call this prior to btr_page_split_and_insert */ if (!dict_index_is_clust(index) && - page_is_leaf(buf_block_get_frame(block))) { + block != NULL) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && + page_is_leaf(frame)) { - ibuf_reset_free_bits(block); + ibuf_reset_free_bits(block); + } } rec = btr_page_split_and_insert( @@ -788,11 +797,8 @@ btr_scrub_page( return BTR_SCRUB_SKIP_PAGE_AND_CLOSE_TABLE; } - buf_frame_t* frame = NULL; + buf_frame_t* frame = buf_block_get_frame(block); - if (block) { - frame = buf_block_get_frame(block); - } if (!frame || btr_page_get_index_id(frame) != scrub_data->current_index->id) { /* page has been reallocated to new index */ diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 0a2140c4a2940..fcf0388d80a89 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -915,9 +915,15 @@ ibuf_set_free_bits_low( page_t* bitmap_page; ulint space; ulint page_no; + buf_frame_t* frame; - if (!page_is_leaf(buf_block_get_frame(block))) { + if (!block) { + return; + } + + frame = buf_block_get_frame(block); + if (!frame || !page_is_leaf(frame)) { return; } @@ -1091,7 +1097,11 @@ ibuf_update_free_bits_zip( page_no = buf_block_get_page_no(block); zip_size = buf_block_get_zip_size(block); - ut_a(page_is_leaf(buf_block_get_frame(block))); + ut_a(block); + + buf_frame_t* frame = buf_block_get_frame(block); + + ut_a(frame && page_is_leaf(frame)); ut_a(zip_size); bitmap_page = ibuf_bitmap_get_map_page(space, page_no, zip_size, mtr); diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index bf3f4a7630190..a1882cdd0ad7b 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -295,9 +295,17 @@ btr_block_get_func( @param idx index tree, may be NULL if not the insert buffer tree @param mtr mini-transaction handle @return the uncompressed page frame */ -# define btr_page_get(space,zip_size,page_no,mode,idx,mtr) \ - buf_block_get_frame(btr_block_get(space,zip_size,page_no, \ - mode,idx,mtr)) +UNIV_INLINE +page_t* +btr_page_get( +/*=========*/ + ulint space, + ulint zip_size, + ulint root_page_no, + ulint mode, + dict_index_t* index, + mtr_t* mtr) + MY_ATTRIBUTE((warn_unused_result)); #endif /* !UNIV_HOTBACKUP */ /**************************************************************//** Gets the index id field of a page. diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index e94103102130f..5acbee1751edf 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -98,6 +98,38 @@ btr_page_set_index_id( mlog_write_ull(page + (PAGE_HEADER + PAGE_INDEX_ID), id, mtr); } } + +/** Gets a buffer page and declares its latching order level. +@param space tablespace identifier +@param zip_size compressed page size in bytes or 0 for uncompressed pages +@param page_no page number +@param mode latch mode +@param idx index tree, may be NULL if not the insert buffer tree +@param mtr mini-transaction handle +@return the uncompressed page frame */ +UNIV_INLINE +page_t* +btr_page_get( +/*=========*/ + ulint space, + ulint zip_size, + ulint root_page_no, + ulint mode, + dict_index_t* index, + mtr_t* mtr) +{ + buf_block_t* block=NULL; + buf_frame_t* frame=NULL; + + block = btr_block_get(space, zip_size, root_page_no, mode, index, mtr); + + if (block) { + frame = buf_block_get_frame(block); + } + + return ((page_t*)frame); +} + #endif /* !UNIV_HOTBACKUP */ /**************************************************************//** diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index 0d117e644d0af..5449397f832c1 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -2281,9 +2281,12 @@ btr_cur_update_in_place( if (page_zip && !(flags & BTR_KEEP_IBUF_BITMAP) && !dict_index_is_clust(index) - && page_is_leaf(buf_block_get_frame(block))) { - /* Update the free bits in the insert buffer. */ - ibuf_update_free_bits_zip(block, mtr); + && block) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && page_is_leaf(frame)) { + /* Update the free bits in the insert buffer. */ + ibuf_update_free_bits_zip(block, mtr); + } } return(err); diff --git a/storage/xtradb/btr/btr0scrub.cc b/storage/xtradb/btr/btr0scrub.cc index e6acb7802f1e6..62a41d19768f6 100644 --- a/storage/xtradb/btr/btr0scrub.cc +++ b/storage/xtradb/btr/btr0scrub.cc @@ -368,12 +368,17 @@ btr_optimistic_scrub( /* We play safe and reset the free bits */ if (!dict_index_is_clust(index) && - page_is_leaf(buf_block_get_frame(block))) { + block != NULL) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && + page_is_leaf(frame)) { ibuf_reset_free_bits(block); + } } scrub_data->scrub_stat.page_reorganizations++; + return DB_SUCCESS; } @@ -488,9 +493,13 @@ btr_pessimistic_scrub( /* We play safe and reset the free bits * NOTE: need to call this prior to btr_page_split_and_insert */ if (!dict_index_is_clust(index) && - page_is_leaf(buf_block_get_frame(block))) { + block != NULL) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && + page_is_leaf(frame)) { - ibuf_reset_free_bits(block); + ibuf_reset_free_bits(block); + } } rec = btr_page_split_and_insert( @@ -788,11 +797,8 @@ btr_scrub_page( return BTR_SCRUB_SKIP_PAGE_AND_CLOSE_TABLE; } - buf_frame_t* frame = NULL; + buf_frame_t* frame = buf_block_get_frame(block); - if (block) { - frame = buf_block_get_frame(block); - } if (!frame || btr_page_get_index_id(frame) != scrub_data->current_index->id) { /* page has been reallocated to new index */ diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index dc6ad49838e08..13597d384335f 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -956,9 +956,15 @@ ibuf_set_free_bits_low( page_t* bitmap_page; ulint space; ulint page_no; + buf_frame_t* frame; - if (!page_is_leaf(buf_block_get_frame(block))) { + if (!block) { + return; + } + + frame = buf_block_get_frame(block); + if (!frame || !page_is_leaf(frame)) { return; } @@ -1132,7 +1138,11 @@ ibuf_update_free_bits_zip( page_no = buf_block_get_page_no(block); zip_size = buf_block_get_zip_size(block); - ut_a(page_is_leaf(buf_block_get_frame(block))); + ut_a(block); + + buf_frame_t* frame = buf_block_get_frame(block); + + ut_a(frame && page_is_leaf(frame)); ut_a(zip_size); bitmap_page = ibuf_bitmap_get_map_page(space, page_no, zip_size, mtr); diff --git a/storage/xtradb/include/btr0btr.h b/storage/xtradb/include/btr0btr.h index 5047d1b2d4e65..9ab62f7739f36 100644 --- a/storage/xtradb/include/btr0btr.h +++ b/storage/xtradb/include/btr0btr.h @@ -298,9 +298,17 @@ btr_block_get_func( @param idx index tree, may be NULL if not the insert buffer tree @param mtr mini-transaction handle @return the uncompressed page frame */ -# define btr_page_get(space,zip_size,page_no,mode,idx,mtr) \ - buf_block_get_frame(btr_block_get(space,zip_size,page_no, \ - mode,idx,mtr)) +UNIV_INLINE +page_t* +btr_page_get( +/*=========*/ + ulint space, + ulint zip_size, + ulint root_page_no, + ulint mode, + dict_index_t* index, + mtr_t* mtr) + MY_ATTRIBUTE((warn_unused_result)); #endif /* !UNIV_HOTBACKUP */ /**************************************************************//** Gets the index id field of a page. diff --git a/storage/xtradb/include/btr0btr.ic b/storage/xtradb/include/btr0btr.ic index 34e0d36e230b2..62a24873482d8 100644 --- a/storage/xtradb/include/btr0btr.ic +++ b/storage/xtradb/include/btr0btr.ic @@ -98,6 +98,38 @@ btr_page_set_index_id( mlog_write_ull(page + (PAGE_HEADER + PAGE_INDEX_ID), id, mtr); } } + +/** Gets a buffer page and declares its latching order level. +@param space tablespace identifier +@param zip_size compressed page size in bytes or 0 for uncompressed pages +@param page_no page number +@param mode latch mode +@param idx index tree, may be NULL if not the insert buffer tree +@param mtr mini-transaction handle +@return the uncompressed page frame */ +UNIV_INLINE +page_t* +btr_page_get( +/*=========*/ + ulint space, + ulint zip_size, + ulint root_page_no, + ulint mode, + dict_index_t* index, + mtr_t* mtr) +{ + buf_block_t* block=NULL; + buf_frame_t* frame=NULL; + + block = btr_block_get(space, zip_size, root_page_no, mode, index, mtr); + + if (block) { + frame = buf_block_get_frame(block); + } + + return ((page_t*)frame); +} + #endif /* !UNIV_HOTBACKUP */ /**************************************************************//** From 661d08c36ca9181e9f2469c66160e877e284c23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 26 Sep 2016 15:16:00 +0300 Subject: [PATCH 067/295] MDEV-10887: innodb.innodb_stats_fetch_nonexistent fails in buildbot on Windows Problem was that test innodb.innodb_stats_fetch_corrupted will post a error InnoDB: Error: Table "mysql"."innodb_index_stats" not found to a error log and test ignores that message. However, following tests might see this error also and they might not ignore this error. Force a server restart after innodb.innodb_stats_fetch_corrupted to get clean log. --- mysql-test/suite/innodb/t/innodb_stats_fetch_corrupted.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql-test/suite/innodb/t/innodb_stats_fetch_corrupted.test b/mysql-test/suite/innodb/t/innodb_stats_fetch_corrupted.test index 4a3f7527c09d7..f69988175ab75 100644 --- a/mysql-test/suite/innodb/t/innodb_stats_fetch_corrupted.test +++ b/mysql-test/suite/innodb/t/innodb_stats_fetch_corrupted.test @@ -7,6 +7,8 @@ # functionality tested here is not related to the page size, so we only # test with 16k page size. -- source include/have_innodb_16k.inc +# server restart +-- source include/not_embedded.inc call mtr.add_suppression("InnoDB: Error: Table \"mysql\".\"innodb_index_stats\" not found"); call mtr.add_suppression("InnoDB: Error: Fetch of persistent statistics requested for table"); @@ -43,3 +45,6 @@ FROM information_schema.tables WHERE table_name = 'test_ps_fetch_corrupted'; ALTER TABLE mysql.innodb_index_stats_ RENAME TO mysql.innodb_index_stats; DROP TABLE test_ps_fetch_corrupted; + +# force server restart to clean up log from above error +--source include/restart_mysqld.inc From f1aefd9d758a3d464d77ede64f960ff33326eb72 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 23 Sep 2016 18:55:44 +0200 Subject: [PATCH 068/295] MDEV-10823 Certain unicode characters in hostname prevent mysqld from starting Server uses gethostname() for the default base name for pid/log files. If a character is not representable in current ANSI encoding, gethostname replaces it with question mark. Thus, generated log file name would also contain a question mark. However, Windows forbids certain characters in filenames, among them '?'. This is described in MSDN article https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx At attempts to create the file via freopen() fails, thus server would not be able to start. The fix is to verify hostname and fall back to "mysql", if invalid characters are found. --- mysys/my_access.c | 9 +++++++++ sql/mysqld.cc | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/mysys/my_access.c b/mysys/my_access.c index 68cd01d33e63e..0da5e7f0bf009 100644 --- a/mysys/my_access.c +++ b/mysys/my_access.c @@ -173,6 +173,11 @@ static my_bool does_drive_exists(char drive_letter) file names with a colon (:) are not allowed because such file names store data in Alternate Data Streams which can be used to hide the data. + Apart from colon, other characters that are not allowed in filenames + on Windows are greater/less sign, double quotes, forward slash, backslash, + pipe and star characters. + + See MSDN documentation on filename restrictions. @param name contains the file name with or without path @param length contains the length of file name @@ -181,6 +186,8 @@ static my_bool does_drive_exists(char drive_letter) @return TRUE if the file name is allowed, FALSE otherwise. */ +#define ILLEGAL_FILENAME_CHARS "<>:\"/\|?*" + my_bool is_filename_allowed(const char *name __attribute__((unused)), size_t length __attribute__((unused)), my_bool allow_current_dir __attribute__((unused))) @@ -205,6 +212,8 @@ my_bool is_filename_allowed(const char *name __attribute__((unused)), return (allow_current_dir && (ch - name == 1) && does_drive_exists(*name)); } + else if (strchr(ILLEGAL_FILENAME_CHARS, *ch)) + return FALSE; } return TRUE; } /* is_filename_allowed */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a71d6d1004204..41f6def8e0897 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4167,6 +4167,8 @@ static int init_common_variables() return 1; } + opt_log_basename= const_cast("mysql"); + if (gethostname(glob_hostname,sizeof(glob_hostname)) < 0) { /* @@ -4176,9 +4178,8 @@ static int init_common_variables() strmake(glob_hostname, STRING_WITH_LEN("localhost")); sql_print_warning("gethostname failed, using '%s' as hostname", glob_hostname); - opt_log_basename= const_cast("mysql"); } - else + else if (is_filename_allowed(glob_hostname, strlen(glob_hostname), FALSE)) opt_log_basename= glob_hostname; strmake(pidfile_name, opt_log_basename, sizeof(pidfile_name)-5); @@ -8970,9 +8971,10 @@ mysqld_get_one_option(int optid, const struct my_option *opt, char *argument) case (int) OPT_LOG_BASENAME: { if (opt_log_basename[0] == 0 || strchr(opt_log_basename, FN_EXTCHAR) || - strchr(opt_log_basename,FN_LIBCHAR)) + strchr(opt_log_basename,FN_LIBCHAR) || + !is_filename_allowed(opt_log_basename, strlen(opt_log_basename), FALSE)) { - sql_print_error("Wrong argument for --log-basename. It can't be empty or contain '.' or '" FN_DIRSEP "'"); + sql_print_error("Wrong argument for --log-basename. It can't be empty or contain '.' or '" FN_DIRSEP "'. It must be valid filename."); return 1; } if (log_error_file_ptr != disabled_my_option) From 3f5aedccca25da72e21d7859b55edeb172d45ce0 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 27 Sep 2016 11:18:24 +0000 Subject: [PATCH 069/295] MDEV-10847 Bring AWS KMS encryption plugin up-to-date with released SDK - Library path's are different now - New dependency on Linux libuuid - Add calls for SDK Initialization/shutdown - Also add request_timeout parameter, default SDK HTTPs timeout appears to be too short in my tests --- plugin/aws_key_management/CMakeLists.txt | 57 ++++++++----------- .../aws_key_management_plugin.cc | 34 ++++++++++- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/plugin/aws_key_management/CMakeLists.txt b/plugin/aws_key_management/CMakeLists.txt index 97bcfbb04dbd1..83285ee6cc9b7 100644 --- a/plugin/aws_key_management/CMakeLists.txt +++ b/plugin/aws_key_management/CMakeLists.txt @@ -3,7 +3,7 @@ # are # - OS : Windows,Linux or OSX -# - C++11 compiler : VS2013+, gcc 4.7+, clang 3.3+ +# - C++11 compiler : VS2013+, gcc 4.8+, clang 3.3+ # - libcurl development package needs to be present on Unixes # # If we build SDK outselves, we'll need require GIT to be present on the build machine @@ -13,7 +13,10 @@ # or if plugin is explicitely requested to build. Then bail out. MACRO(SKIP_AWS_PLUGIN msg) IF(VERBOSE OR "${PLUGIN_AWS_KEY_MANAGEMENT}" MATCHES "^(STATIC|DYNAMIC)$") - MESSAGE(STATUS "Skip aws_key_management - ${msg}") + MESSAGE(STATUS "Can't build aws_key_management - ${msg}") + ENDIF() + IF(TARGET aws_key_management) + MESSAGE(FATAL_ERROR "Error configuring aws_key_management - aborting") ENDIF() RETURN() ENDMACRO() @@ -27,7 +30,7 @@ ENDIF() # This plugin needs recent C++ compilers (AWS C++ SDK header files are using C++11 features) SET(CXX11_FLAGS) -SET(OLD_COMPILER_MSG "AWS SDK requires c++11 -capable compiler (minimal supported versions are g++ 4.7, clang 3.3, VS2103)") +SET(OLD_COMPILER_MSG "AWS SDK requires c++11 -capable compiler (minimal supported versions are g++ 4.8, clang 3.3, VS2103)") IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU") EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) @@ -54,26 +57,6 @@ IF (NOT(WIN32 OR APPLE OR (CMAKE_SYSTEM_NAME MATCHES "Linux"))) ENDIF() -# Figure out where AWS installs SDK libraries -# The below is defined in AWS SDK's CMakeLists.txt -# (and their handling is weird, every OS has special install directory) -IF(WIN32) - SET(SDK_INSTALL_BINARY_PREFIX "windows") -ELSEIF(APPLE) - SET(SDK_INSTALL_BINARY_PREFIX "mac") -ELSEIF(UNIX) - SET(SDK_INSTALL_BINARY_PREFIX "linux") -ENDIF() -IF(NOT APPLE) - IF(CMAKE_SIZEOF_VOID_P EQUAL 8) - SET(SDK_INSTALL_BINARY_PREFIX "${SDK_INSTALL_BINARY_PREFIX}/intel64") - ELSE() - SET(SDK_INSTALL_BINARY_PREFIX "${SDK_INSTALL_BINARY_PREFIX}/ia32") - ENDIF() -ENDIF() -IF(CMAKE_CONFIGURATION_TYPES) - SET(SDK_INSTALL_BINARY_PREFIX "${SDK_INSTALL_BINARY_PREFIX}/${CMAKE_CFG_INTDIR}") -ENDIF() FIND_LIBRARY(AWS_CPP_SDK_CORE NAMES aws-cpp-sdk-core PATH_SUFFIXES "${SDK_INSTALL_BINARY_PREFIX}") FIND_LIBRARY(AWS_CPP_SDK_KMS NAMES aws-cpp-sdk-core PATH_SUFFIXES "${SDK_INSTALL_BINARY_PREFIX}") @@ -99,26 +82,35 @@ ELSE() SKIP_AWS_PLUGIN("AWS C++ SDK requires libcurl development package") ENDIF() SET(PIC_FLAG -fPIC) + FIND_PATH(UUID_INCLUDE_DIR uuid/uuid.h) + IF(NOT UUID_INCLUDE_DIR) + SKIP_AWS_PLUGIN("AWS C++ SDK requires uuid development package") + ENDIF() + IF(NOT APPLE) + FIND_LIBRARY(UUID_LIBRARIES uuid) + IF(NOT UUID_LIBRARIES) + SKIP_AWS_PLUGIN("AWS C++ SDK requires uuid development package") + ENDIF() + ENDIF() ENDIF() IF(MSVC) - SET(EXTRA_SDK_CMAKE_FLAGS -DCMAKE_CXX_FLAGS_DEBUGOPT="" -DCMAKE_EXE_LINKER_FLAGS_DEBUGOPT="" -DCMAKE_CXX_FLAGS=/wd4592) + SET(EXTRA_SDK_CMAKE_FLAGS -DCMAKE_CXX_FLAGS_DEBUGOPT="" -DCMAKE_EXE_LINKER_FLAGS_DEBUGOPT="" "-DCMAKE_CXX_FLAGS=/wd4530 /WX-") ENDIF() IF(CMAKE_CXX_COMPILER) SET(EXTRA_SDK_CMAKE_FLAGS ${EXTRA_SDK_CMAKE_FLAGS} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) ENDIF() - # Relax AWS C++ SDK unreasonably high requirements for CMake version. Use replace utility (from MariaDB build) - # to patch their CMakeLists.txt SET(AWS_SDK_PATCH_COMMAND ) ExternalProject_Add( aws_sdk_cpp GIT_REPOSITORY "https://github.com/awslabs/aws-sdk-cpp.git" - GIT_TAG "0.9.6" # single tag + GIT_TAG "1.0.8" UPDATE_COMMAND "" - PATCH_COMMAND replace 3.1.2 2.8 -- ${CMAKE_BINARY_DIR}/aws-sdk-cpp/CMakeLists.txt SOURCE_DIR "${CMAKE_BINARY_DIR}/aws-sdk-cpp" CMAKE_ARGS - -DBUILD_ONLY=aws-cpp-sdk-kms -DSTATIC_LINKING=1 + -DBUILD_ONLY=kms + -DBUILD_SHARED_LIBS=OFF + -DFORCE_SHARED_CRT=OFF "-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} ${PIC_FLAG}" "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}" "-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} ${PIC_FLAG}" @@ -127,18 +119,18 @@ ELSE() -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/aws_sdk_cpp TEST_COMMAND "" ) - + SET_TARGET_PROPERTIES(aws_sdk_cpp PROPERTIES EXCLUDE_FROM_ALL TRUE) # We do not need to build the whole SDK , just 2 of its libs set(AWS_SDK_LIBS aws-cpp-sdk-core aws-cpp-sdk-kms) FOREACH(lib ${AWS_SDK_LIBS}) ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL) ADD_DEPENDENCIES(${lib} aws_sdk_cpp) - SET(loc "${CMAKE_BINARY_DIR}/aws_sdk_cpp/lib/${SDK_INSTALL_BINARY_PREFIX}/${CMAKE_STATIC_LIBRARY_PREFIX}${lib}${CMAKE_STATIC_LIBRARY_SUFFIX}") + SET(loc "${CMAKE_BINARY_DIR}/aws_sdk_cpp/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${lib}${CMAKE_STATIC_LIBRARY_SUFFIX}") SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LOCATION ${loc}) IF(WIN32) SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "bcrypt;winhttp;wininet;userenv") ELSE() - SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${SSL_LIBRARIES};${CURL_LIBRARIES}") + SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${SSL_LIBRARIES};${CURL_LIBRARIES};${UUID_LIBRARIES}") ENDIF() ENDFOREACH() @@ -150,5 +142,6 @@ ELSE() INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/aws_sdk_cpp/include) ENDIF() +ADD_DEFINITIONS(${SSL_DEFINES}) # Need to know whether openssl should be initialized SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS}") TARGET_LINK_LIBRARIES(aws_key_management ${AWS_SDK_LIBS}) diff --git a/plugin/aws_key_management/aws_key_management_plugin.cc b/plugin/aws_key_management/aws_key_management_plugin.cc index f4d3c7a52bc61..20a795eb437ad 100644 --- a/plugin/aws_key_management/aws_key_management_plugin.cc +++ b/plugin/aws_key_management/aws_key_management_plugin.cc @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -79,6 +80,7 @@ static char* master_key_id; static unsigned long key_spec; static unsigned long log_level; static int rotate_key; +static int request_timeout; /* AWS functionality*/ static int aws_decrypt_key(const char *path, KEY_INFO *info); @@ -138,6 +140,7 @@ class MySQLLogSystem : public Aws::Utils::Logging::FormattedLogSystem } }; +Aws::SDKOptions sdkOptions; /* Plugin initialization. @@ -148,13 +151,30 @@ class MySQLLogSystem : public Aws::Utils::Logging::FormattedLogSystem static int plugin_init(void *p) { DBUG_ENTER("plugin_init"); - client = new KMSClient(); + +#ifdef HAVE_YASSL + sdkOptions.cryptoOptions.initAndCleanupOpenSSL = true; +#else + /* Server initialized OpenSSL already, thus AWS must skip it */ + sdkOptions.cryptoOptions.initAndCleanupOpenSSL = false; +#endif + + Aws::InitAPI(sdkOptions); + InitializeAWSLogging(Aws::MakeShared("aws_key_management_plugin", (Aws::Utils::Logging::LogLevel) log_level)); + + Aws::Client::ClientConfiguration clientConfiguration; + if (request_timeout) + { + clientConfiguration.requestTimeoutMs= request_timeout; + clientConfiguration.connectTimeoutMs= request_timeout; + } + client = new KMSClient(clientConfiguration); if (!client) { sql_print_error("Can not initialize KMS client"); DBUG_RETURN(-1); } - InitializeAWSLogging(Aws::MakeShared("aws_key_management_plugin", (Aws::Utils::Logging::LogLevel) log_level)); + #ifdef HAVE_PSI_INTERFACE mysql_mutex_register("aws_key_management", &mtx_info, 1); #endif @@ -189,6 +209,8 @@ static int plugin_deinit(void *p) mysql_mutex_destroy(&mtx); delete client; ShutdownAWSLogging(); + + Aws::ShutdownAPI(sdkOptions); DBUG_RETURN(0); } @@ -557,11 +579,19 @@ static MYSQL_SYSVAR_INT(rotate_key, rotate_key, "Set this variable to key id to perform rotation of the key. Specify -1 to rotate all keys", NULL, update_rotate, 0, -1, INT_MAX, 1); + +static MYSQL_SYSVAR_INT(request_timeout, request_timeout, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Timeout in milliseconds for create HTTPS connection or execute AWS request. Specify 0 to use SDK default.", + NULL, NULL, 0, 0, INT_MAX, 1); + + static struct st_mysql_sys_var* settings[]= { MYSQL_SYSVAR(master_key_id), MYSQL_SYSVAR(key_spec), MYSQL_SYSVAR(rotate_key), MYSQL_SYSVAR(log_level), + MYSQL_SYSVAR(request_timeout), NULL }; From e226276950497c0ec92b3bf71bc6c46ac4a779d4 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 14 Sep 2016 18:15:03 +0200 Subject: [PATCH 070/295] MDEV-10777: Server crashed due to query_cache_info plugin Possible fix. Make the pluging more safe. --- plugin/qc_info/qc_info.cc | 39 ++++++++++++++++++++++++++++++++------- sql/sql_cache.cc | 3 ++- sql/sql_cache.h | 9 ++++++++- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/plugin/qc_info/qc_info.cc b/plugin/qc_info/qc_info.cc index 4ccfdc8f8c2fd..1dcef0044472c 100644 --- a/plugin/qc_info/qc_info.cc +++ b/plugin/qc_info/qc_info.cc @@ -106,6 +106,9 @@ static ST_FIELD_INFO qc_info_fields[]= {0, 0, MYSQL_TYPE_STRING, 0, 0, 0, 0} }; + +static const char unknown[]= "#UNKNOWN#"; + static int qc_info_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { @@ -146,7 +149,8 @@ static int qc_info_fill_table(THD *thd, TABLE_LIST *tables, query_cache_block_raw = my_hash_element(queries, i); query_cache_block = (Query_cache_block*)query_cache_block_raw; - if (query_cache_block->type != Query_cache_block::QUERY) + if (unlikely(!query_cache_block || + query_cache_block->type != Query_cache_block::QUERY)) continue; query_cache_query = query_cache_block->query(); @@ -169,14 +173,33 @@ static int qc_info_fill_table(THD *thd, TABLE_LIST *tables, table->field[COLUMN_GROUP_CONCAT_MAX_LENGTH]->store(flags.group_concat_max_len, 0); cs_client= get_charset(flags.character_set_client_num, MYF(MY_WME)); - table->field[COLUMN_CHARACTER_SET_CLIENT]->store(cs_client->csname, strlen(cs_client->csname), scs); + if (likely(cs_client)) + table->field[COLUMN_CHARACTER_SET_CLIENT]-> + store(cs_client->csname, strlen(cs_client->csname), scs); + else + table->field[COLUMN_CHARACTER_SET_CLIENT]-> + store(STRING_WITH_LEN(unknown), scs); + cs_result= get_charset(flags.character_set_results_num, MYF(MY_WME)); - table->field[COLUMN_CHARACTER_SET_RESULT]->store(cs_result->csname, strlen(cs_result->csname), scs); + if (likely(cs_result)) + table->field[COLUMN_CHARACTER_SET_RESULT]-> + store(cs_result->csname, strlen(cs_result->csname), scs); + else + table->field[COLUMN_CHARACTER_SET_RESULT]-> + store(STRING_WITH_LEN(unknown), scs); + collation= get_charset(flags.collation_connection_num, MYF(MY_WME)); - table->field[COLUMN_COLLATION]->store(collation->name, strlen(collation->name), scs); + if (likely(collation)) + table->field[COLUMN_COLLATION]-> + store(collation->name, strlen(collation->name), scs); + else + table->field[COLUMN_COLLATION]-> store(STRING_WITH_LEN(unknown), scs); tz= flags.time_zone->get_name(); - table->field[COLUMN_TIMEZONE]->store(tz->ptr(), tz->length(), scs); + if (likely(tz)) + table->field[COLUMN_TIMEZONE]->store(tz->ptr(), tz->length(), scs); + else + table->field[COLUMN_TIMEZONE]-> store(STRING_WITH_LEN(unknown), scs); table->field[COLUMN_DEFAULT_WEEK_FORMAT]->store(flags.default_week_format, 0); table->field[COLUMN_DIV_PRECISION_INCREMENT]->store(flags.div_precision_increment, 0); @@ -204,7 +227,8 @@ static int qc_info_fill_table(THD *thd, TABLE_LIST *tables, /* If we have result blocks, process them */ first_result_block= query_cache_query->result(); - if(first_result_block) + if(query_cache_query->is_results_ready() && + first_result_block) { /* initialize so we can loop over the result blocks*/ result_block= first_result_block; @@ -231,7 +255,8 @@ static int qc_info_fill_table(THD *thd, TABLE_LIST *tables, } table->field[COLUMN_RESULT_BLOCKS_COUNT]->store(result_blocks_count, 0); table->field[COLUMN_RESULT_BLOCKS_SIZE]->store(result_blocks_size, 0); - table->field[COLUMN_RESULT_BLOCKS_SIZE_USED]->store(result_blocks_size_used, 0); + table->field[COLUMN_RESULT_BLOCKS_SIZE_USED]-> + store(result_blocks_size_used, 0); if (schema_table_store_record(thd, table)) goto cleanup; diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc index 91dd8ad73258b..c69303c5273ee 100644 --- a/sql/sql_cache.cc +++ b/sql/sql_cache.cc @@ -961,7 +961,7 @@ inline void Query_cache_query::unlock_reading() void Query_cache_query::init_n_lock() { DBUG_ENTER("Query_cache_query::init_n_lock"); - res=0; wri = 0; len = 0; + res=0; wri = 0; len = 0; ready= 0; mysql_rwlock_init(key_rwlock_query_cache_query_lock, &lock); lock_writing(); DBUG_PRINT("qcache", ("inited & locked query for block 0x%lx", @@ -1226,6 +1226,7 @@ void Query_cache::end_of_result(THD *thd) query_cache.split_block(last_result_block,len); header->found_rows(limit_found_rows); + header->set_results_ready(); // signal for plugin header->result()->type= Query_cache_block::RESULT; /* Drop the writer. */ diff --git a/sql/sql_cache.h b/sql/sql_cache.h index 00ba9bf59d84e..657caf4a5bc90 100644 --- a/sql/sql_cache.h +++ b/sql/sql_cache.h @@ -156,8 +156,9 @@ struct Query_cache_query Query_cache_block *res; Query_cache_tls *wri; ulong len; - uint8 tbls_type; unsigned int last_pkt_nr; + uint8 tbls_type; + uint8 ready; Query_cache_query() {} /* Remove gcc warning */ inline void init_n_lock(); @@ -177,6 +178,12 @@ struct Query_cache_query { return (((uchar*)this) + ALIGN_SIZE(sizeof(Query_cache_query))); } + /** + following used to check if result ready in plugin without + locking rw_lock of the query. + */ + inline void set_results_ready() { ready= 1; } + inline bool is_results_ready() { return ready; } void lock_writing(); void lock_reading(); bool try_lock_writing(); From b38d3c3d8afea7183f2a595f0c8d8dd7efaa801f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 27 Sep 2016 12:34:15 +0000 Subject: [PATCH 071/295] MDEV-10907 MTR and server writes can interleave in the error log Ensure atomic appends to the error log by using CreateFile with FILE_APPEND_DATA flag to open error log file (both MTR and server) --- mysql-test/lib/My/Platform.pm | 49 ++++++++++++++++++++++++++++++++++- mysql-test/lib/mtr_io.pl | 9 ++++--- mysys/my_fopen.c | 6 ++--- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/mysql-test/lib/My/Platform.pm b/mysql-test/lib/My/Platform.pm index 1776f1008daa6..110cf8a20e0ed 100644 --- a/mysql-test/lib/My/Platform.pm +++ b/mysql-test/lib/My/Platform.pm @@ -24,7 +24,7 @@ use File::Path; use base qw(Exporter); our @EXPORT= qw(IS_CYGWIN IS_WINDOWS IS_WIN32PERL native_path posix_path mixed_path - check_socket_path_length process_alive); + check_socket_path_length process_alive open_for_append); BEGIN { if ($^O eq "cygwin") { @@ -161,4 +161,51 @@ sub process_alive { } + +use Symbol qw( gensym ); + +use if $^O eq 'MSWin32', 'Win32API::File', qw( CloseHandle CreateFile GetOsFHandle OsFHandleOpen OPEN_ALWAYS FILE_APPEND_DATA + FILE_SHARE_READ FILE_SHARE_WRITE FILE_SHARE_DELETE ); +use if $^O eq 'MSWin32', 'Win32::API'; + +use constant WIN32API_FILE_NULL => []; + +# Open a file for append +# On Windows we use CreateFile with FILE_APPEND_DATA +# to insure that writes are atomic, not interleaved +# with writes by another processes. +sub open_for_append +{ + my ($file) = @_; + my $fh = gensym(); + + if (IS_WIN32PERL) + { + my $handle; + if (!($handle = CreateFile( + $file, + FILE_APPEND_DATA(), + FILE_SHARE_READ()|FILE_SHARE_WRITE()|FILE_SHARE_DELETE(), + WIN32API_FILE_NULL, + OPEN_ALWAYS(),# Create if doesn't exist. + 0, + WIN32API_FILE_NULL, + ))) + { + return undef; + } + + if (!OsFHandleOpen($fh, $handle, 'wat')) + { + CloseHandle($handle); + return undef; + } + return $fh; + } + + open($fh,">>",$file) or return undef; + return $fh; +} + + 1; diff --git a/mysql-test/lib/mtr_io.pl b/mysql-test/lib/mtr_io.pl index 8c2803f042794..0de4d9612acee 100644 --- a/mysql-test/lib/mtr_io.pl +++ b/mysql-test/lib/mtr_io.pl @@ -21,6 +21,7 @@ use strict; use Carp; +use My::Platform; sub mtr_fromfile ($); sub mtr_tofile ($@); @@ -45,10 +46,10 @@ ($) sub mtr_tofile ($@) { my $file= shift; - - open(FILE,">>",$file) or mtr_error("can't open file \"$file\": $!"); - print FILE join("", @_); - close FILE; + my $fh= open_for_append $file; + mtr_error("can't open file \"$file\": $!") unless defined($fh); + print $fh join("", @_); + close $fh; } diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c index 52f61649bb39f..cc1019365ac48 100644 --- a/mysys/my_fopen.c +++ b/mysys/my_fopen.c @@ -102,6 +102,7 @@ static FILE *my_win_freopen(const char *path, const char *mode, FILE *stream) HANDLE osfh; DBUG_ASSERT(path && stream); + DBUG_ASSERT(strchr(mode, 'a')); /* We use FILE_APPEND_DATA below */ /* Services don't have stdout/stderr on Windows, so _fileno returns -1. */ if (fd < 0) @@ -112,15 +113,14 @@ static FILE *my_win_freopen(const char *path, const char *mode, FILE *stream) fd= _fileno(stream); } - if ((osfh= CreateFile(path, GENERIC_READ | GENERIC_WRITE, + if ((osfh= CreateFile(path, GENERIC_READ | FILE_APPEND_DATA, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) == INVALID_HANDLE_VALUE) return NULL; - if ((handle_fd= _open_osfhandle((intptr_t)osfh, - _O_APPEND | _O_TEXT)) == -1) + if ((handle_fd= _open_osfhandle((intptr_t)osfh, _O_TEXT)) == -1) { CloseHandle(osfh); return NULL; From 0e472236ce1e7da5f5916f712e29511fc1aade33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 27 Sep 2016 17:38:47 +0200 Subject: [PATCH 072/295] Make sure to recompile the feedback plugin for EMBEDDED On OS X with clang 7.3.0 running any test as embedded crashes. As libfeedback was not compiled with embedded, we had 2 notions of THD, with different number of members. When calling inline functions defined in sql_class.h, the functions were defined for 1 THD but called on an object that is different. This resulted in erroneous data being returned and shortly after, a crash within the alloc_query() function. Recompile the feedback plugin for the embedded server specifically to not have such symbol conflicts. --- plugin/feedback/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/feedback/CMakeLists.txt b/plugin/feedback/CMakeLists.txt index a243ba077519b..2103250e5a68c 100644 --- a/plugin/feedback/CMakeLists.txt +++ b/plugin/feedback/CMakeLists.txt @@ -19,5 +19,5 @@ ENDIF(WIN32) MYSQL_ADD_PLUGIN(FEEDBACK ${FEEDBACK_SOURCES} LINK_LIBRARIES ${SSL_LIBRARIES} - ${MAYBE_STATIC_ONLY} DEFAULT) + ${MAYBE_STATIC_ONLY} RECOMPILE_FOR_EMBEDDED DEFAULT) From 094f140c9ae672eb61a27f15c5e99c7114ed0150 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Sep 2016 17:56:00 +0200 Subject: [PATCH 073/295] 5.6.33 --- storage/innobase/fts/fts0fts.cc | 31 +++++++++++++++++++++ storage/innobase/handler/ha_innodb.cc | 9 ++++-- storage/innobase/handler/handler0alter.cc | 9 ++++-- storage/innobase/handler/i_s.cc | 2 ++ storage/innobase/include/fts0fts.h | 10 +++++++ storage/innobase/row/row0log.cc | 14 +++++----- storage/innobase/row/row0mysql.cc | 34 +++++++++++++++++++++++ 7 files changed, 97 insertions(+), 12 deletions(-) diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 25059db96b078..a0f0fab5566b0 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -108,6 +108,7 @@ UNIV_INTERN mysql_pfs_key_t fts_pll_tokenize_mutex_key; /** variable to record innodb_fts_internal_tbl_name for information schema table INNODB_FTS_INSERTED etc. */ UNIV_INTERN char* fts_internal_tbl_name = NULL; +UNIV_INTERN char* fts_internal_tbl_name2 = NULL; /** InnoDB default stopword list: There are different versions of stopwords, the stop words listed @@ -6569,6 +6570,36 @@ fts_check_corrupt_index( return(0); } +/* Get parent table name if it's a fts aux table +@param[in] aux_table_name aux table name +@param[in] aux_table_len aux table length +@return parent table name, or NULL */ +char* +fts_get_parent_table_name( + const char* aux_table_name, + ulint aux_table_len) +{ + fts_aux_table_t aux_table; + char* parent_table_name = NULL; + + if (fts_is_aux_table_name(&aux_table, aux_table_name, aux_table_len)) { + dict_table_t* parent_table; + + parent_table = dict_table_open_on_id( + aux_table.parent_id, TRUE, DICT_TABLE_OP_NORMAL); + + if (parent_table != NULL) { + parent_table_name = mem_strdupl( + parent_table->name, + strlen(parent_table->name)); + + dict_table_close(parent_table, TRUE, FALSE); + } + } + + return(parent_table_name); +} + /** Check the validity of the parent table. @param[in] aux_table auxiliary table @return true if it is a valid table or false if it is not */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 95a0e08a80609..be5e74e1617bb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14505,7 +14505,12 @@ innodb_internal_table_update( my_free(old); } - fts_internal_tbl_name = *(char**) var_ptr; + fts_internal_tbl_name2 = *(char**) var_ptr; + if (fts_internal_tbl_name2 == NULL) { + fts_internal_tbl_name = const_cast("default"); + } else { + fts_internal_tbl_name = fts_internal_tbl_name2; + } } /****************************************************************//** @@ -16253,7 +16258,7 @@ static MYSQL_SYSVAR_BOOL(disable_sort_file_cache, srv_disable_sort_file_cache, "Whether to disable OS system file cache for sort I/O", NULL, NULL, FALSE); -static MYSQL_SYSVAR_STR(ft_aux_table, fts_internal_tbl_name, +static MYSQL_SYSVAR_STR(ft_aux_table, fts_internal_tbl_name2, PLUGIN_VAR_NOCMDARG, "FTS internal auxiliary table to be checked", innodb_internal_table_validate, diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 961e0818d3987..2261754a4f5e1 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -201,7 +201,10 @@ innobase_need_rebuild( /*==================*/ const Alter_inplace_info* ha_alter_info) { - if (ha_alter_info->handler_flags + Alter_inplace_info::HA_ALTER_FLAGS alter_inplace_flags = + ha_alter_info->handler_flags & ~(INNOBASE_INPLACE_IGNORE); + + if (alter_inplace_flags == Alter_inplace_info::CHANGE_CREATE_OPTION && !(ha_alter_info->create_info->used_fields & (HA_CREATE_USED_ROW_FORMAT @@ -3760,7 +3763,7 @@ ha_innobase::prepare_inplace_alter_table( } if (!(ha_alter_info->handler_flags & INNOBASE_ALTER_DATA) - || (ha_alter_info->handler_flags + || ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) == Alter_inplace_info::CHANGE_CREATE_OPTION && !innobase_need_rebuild(ha_alter_info))) { @@ -3926,7 +3929,7 @@ ha_innobase::inplace_alter_table( DBUG_RETURN(false); } - if (ha_alter_info->handler_flags + if ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) == Alter_inplace_info::CHANGE_CREATE_OPTION && !innobase_need_rebuild(ha_alter_info)) { goto ok_exit; diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 43905f4a32bb9..a780ee5cd6243 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -4038,6 +4038,8 @@ i_s_fts_config_fill( DBUG_RETURN(0); } + DEBUG_SYNC_C("i_s_fts_config_fille_check"); + fields = table->field; /* Prevent DDL to drop fts aux tables. */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 87b5787d416be..3e2f359bbebef 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -375,6 +375,7 @@ extern bool fts_need_sync; /** Variable specifying the table that has Fulltext index to display its content through information schema table */ extern char* fts_internal_tbl_name; +extern char* fts_internal_tbl_name2; #define fts_que_graph_free(graph) \ do { \ @@ -823,6 +824,15 @@ void fts_drop_orphaned_tables(void); /*==========================*/ +/* Get parent table name if it's a fts aux table +@param[in] aux_table_name aux table name +@param[in] aux_table_len aux table length +@return parent table name, or NULL */ +char* +fts_get_parent_table_name( + const char* aux_table_name, + ulint aux_table_len); + /******************************************************************//** Since we do a horizontal split on the index table, we need to drop all the split tables. diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index a6751b208f74c..54183759e8d31 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -613,7 +613,7 @@ row_log_table_delete( &old_pk_extra_size); ut_ad(old_pk_extra_size < 0x100); - mrec_size = 4 + old_pk_size; + mrec_size = 6 + old_pk_size; /* Log enough prefix of the BLOB unless both the old and new table are in COMPACT or REDUNDANT format, @@ -643,8 +643,8 @@ row_log_table_delete( *b++ = static_cast(old_pk_extra_size); /* Log the size of external prefix we saved */ - mach_write_to_2(b, ext_size); - b += 2; + mach_write_to_4(b, ext_size); + b += 4; rec_convert_dtuple_to_temp( b + old_pk_extra_size, new_index, @@ -2268,14 +2268,14 @@ row_log_table_apply_op( break; case ROW_T_DELETE: - /* 1 (extra_size) + 2 (ext_size) + at least 1 (payload) */ - if (mrec + 4 >= mrec_end) { + /* 1 (extra_size) + 4 (ext_size) + at least 1 (payload) */ + if (mrec + 6 >= mrec_end) { return(NULL); } extra_size = *mrec++; - ext_size = mach_read_from_2(mrec); - mrec += 2; + ext_size = mach_read_from_4(mrec); + mrec += 4; ut_ad(mrec < mrec_end); /* We assume extra_size < 0x100 for the PRIMARY KEY prefix. diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index c3a7e2c280706..11bef1064d6f5 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2676,6 +2676,10 @@ row_drop_tables_for_mysql_in_background(void) return(n_tables + n_tables_dropped); } + DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", + os_thread_sleep(5000000); + ); + table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); @@ -2686,6 +2690,16 @@ row_drop_tables_for_mysql_in_background(void) goto already_dropped; } + if (!table->to_be_dropped) { + /* There is a scenario: the old table is dropped + just after it's added into drop list, and new + table with the same name is created, then we try + to drop the new table in background. */ + dict_table_close(table, FALSE, FALSE); + + goto already_dropped; + } + ut_a(!table->can_be_evicted); dict_table_close(table, FALSE, FALSE); @@ -3945,6 +3959,13 @@ row_drop_table_for_mysql( } } + + DBUG_EXECUTE_IF("row_drop_table_add_to_background", + row_add_table_to_background_drop_list(table->name); + err = DB_SUCCESS; + goto funct_exit; + ); + /* TODO: could we replace the counter n_foreign_key_checks_running with lock checks on the table? Acquire here an exclusive lock on the table, and rewrite lock0lock.cc and the lock wait in srv0srv.cc so that @@ -4561,6 +4582,19 @@ row_drop_database_for_mysql( row_mysql_lock_data_dictionary(trx); while ((table_name = dict_get_first_table_name_in_db(name))) { + /* Drop parent table if it is a fts aux table, to + avoid accessing dropped fts aux tables in information + scheam when parent table still exists. + Note: Drop parent table will drop fts aux tables. */ + char* parent_table_name; + parent_table_name = fts_get_parent_table_name( + table_name, strlen(table_name)); + + if (parent_table_name != NULL) { + mem_free(table_name); + table_name = parent_table_name; + } + ut_a(memcmp(table_name, name, namelen) == 0); table = dict_table_open_on_name( From e3124a8cd79f3639e6159d1fbf87579c3e2b3ebb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Sep 2016 17:57:28 +0200 Subject: [PATCH 074/295] 5.6.33 --- .../suite/perfschema/r/aggregate.result | 118 ----------- mysql-test/suite/perfschema/t/aggregate.test | 191 ------------------ 2 files changed, 309 deletions(-) delete mode 100644 mysql-test/suite/perfschema/r/aggregate.result delete mode 100644 mysql-test/suite/perfschema/t/aggregate.test diff --git a/mysql-test/suite/perfschema/r/aggregate.result b/mysql-test/suite/perfschema/r/aggregate.result deleted file mode 100644 index ab927f544cf0d..0000000000000 --- a/mysql-test/suite/perfschema/r/aggregate.result +++ /dev/null @@ -1,118 +0,0 @@ -"General cleanup" -drop table if exists t1; -update performance_schema.setup_instruments set enabled = 'NO'; -update performance_schema.setup_consumers set enabled = 'NO'; -truncate table performance_schema.file_summary_by_event_name; -truncate table performance_schema.file_summary_by_instance; -truncate table performance_schema.socket_summary_by_event_name; -truncate table performance_schema.socket_summary_by_instance; -truncate table performance_schema.events_waits_summary_global_by_event_name; -truncate table performance_schema.events_waits_summary_by_instance; -truncate table performance_schema.events_waits_summary_by_thread_by_event_name; -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments -set enabled = 'YES', timed = 'YES'; -create table t1 ( -id INT PRIMARY KEY, -b CHAR(100) DEFAULT 'initial value') -ENGINE=MyISAM; -insert into t1 (id) values (1), (2), (3), (4), (5), (6), (7), (8); -update performance_schema.setup_instruments SET enabled = 'NO'; -update performance_schema.setup_consumers set enabled = 'NO'; -set @dump_all=FALSE; -"Verifying file aggregate consistency" -SELECT EVENT_NAME, e.COUNT_READ, SUM(i.COUNT_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_READ <> SUM(i.COUNT_READ)) -OR @dump_all; -EVENT_NAME COUNT_READ SUM(i.COUNT_READ) -SELECT EVENT_NAME, e.COUNT_WRITE, SUM(i.COUNT_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_WRITE <> SUM(i.COUNT_WRITE)) -OR @dump_all; -EVENT_NAME COUNT_WRITE SUM(i.COUNT_WRITE) -SELECT EVENT_NAME, e.COUNT_READ, SUM(i.COUNT_READ) -FROM performance_schema.socket_summary_by_event_name AS e -JOIN performance_schema.socket_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_READ <> SUM(i.COUNT_READ)) -OR @dump_all; -EVENT_NAME COUNT_READ SUM(i.COUNT_READ) -SELECT EVENT_NAME, e.COUNT_WRITE, SUM(i.COUNT_WRITE) -FROM performance_schema.socket_summary_by_event_name AS e -JOIN performance_schema.socket_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_WRITE <> SUM(i.COUNT_WRITE)) -OR @dump_all; -EVENT_NAME COUNT_WRITE SUM(i.COUNT_WRITE) -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_READ, SUM(i.SUM_NUMBER_OF_BYTES_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_READ <> SUM(i.SUM_NUMBER_OF_BYTES_READ)) -OR @dump_all; -EVENT_NAME SUM_NUMBER_OF_BYTES_READ SUM(i.SUM_NUMBER_OF_BYTES_READ) -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_WRITE, SUM(i.SUM_NUMBER_OF_BYTES_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_WRITE <> SUM(i.SUM_NUMBER_OF_BYTES_WRITE)) -OR @dump_all; -EVENT_NAME SUM_NUMBER_OF_BYTES_WRITE SUM(i.SUM_NUMBER_OF_BYTES_WRITE) -"Verifying waits aggregate consistency (instance)" -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(i.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(i.SUM_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME SUM_TIMER_WAIT SUM(i.SUM_TIMER_WAIT) -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(i.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(i.MIN_TIMER_WAIT)) -AND (MIN(i.MIN_TIMER_WAIT) != 0) -OR @dump_all; -EVENT_NAME MIN_TIMER_WAIT MIN(i.MIN_TIMER_WAIT) -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(i.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(i.MAX_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME MAX_TIMER_WAIT MAX(i.MAX_TIMER_WAIT) -"Verifying waits aggregate consistency (thread)" -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(t.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(t.SUM_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME SUM_TIMER_WAIT SUM(t.SUM_TIMER_WAIT) -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(t.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(t.MIN_TIMER_WAIT)) -AND (MIN(t.MIN_TIMER_WAIT) != 0) -OR @dump_all; -EVENT_NAME MIN_TIMER_WAIT MIN(t.MIN_TIMER_WAIT) -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(t.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(t.MAX_TIMER_WAIT)) -OR @dump_all; -EVENT_NAME MAX_TIMER_WAIT MAX(t.MAX_TIMER_WAIT) -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments -set enabled = 'YES', timed = 'YES'; -drop table test.t1; diff --git a/mysql-test/suite/perfschema/t/aggregate.test b/mysql-test/suite/perfschema/t/aggregate.test deleted file mode 100644 index 326c0e763d9db..0000000000000 --- a/mysql-test/suite/perfschema/t/aggregate.test +++ /dev/null @@ -1,191 +0,0 @@ -# Tests for PERFORMANCE_SCHEMA -# Verify that statistics aggregated by different criteria are consistent. - ---source include/not_embedded.inc ---source include/have_perfschema.inc ---source include/have_QC_Disabled.inc - ---echo "General cleanup" - ---disable_warnings -drop table if exists t1; ---enable_warnings - -update performance_schema.setup_instruments set enabled = 'NO'; -update performance_schema.setup_consumers set enabled = 'NO'; - -# Cleanup statistics -truncate table performance_schema.file_summary_by_event_name; -truncate table performance_schema.file_summary_by_instance; -truncate table performance_schema.socket_summary_by_event_name; -truncate table performance_schema.socket_summary_by_instance; -truncate table performance_schema.events_waits_summary_global_by_event_name; -truncate table performance_schema.events_waits_summary_by_instance; -truncate table performance_schema.events_waits_summary_by_thread_by_event_name; - -# Start recording data -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments - set enabled = 'YES', timed = 'YES'; - - -create table t1 ( - id INT PRIMARY KEY, - b CHAR(100) DEFAULT 'initial value') - ENGINE=MyISAM; - -insert into t1 (id) values (1), (2), (3), (4), (5), (6), (7), (8); - -# Stop recording data, so the select below don't add noise. -update performance_schema.setup_instruments SET enabled = 'NO'; -# Disable all consumers, for long standing waits -update performance_schema.setup_consumers set enabled = 'NO'; - -# Helper to debug -set @dump_all=FALSE; - -# Note that in general: -# - COUNT/SUM/MAX(file_summary_by_event_name) >= -# COUNT/SUM/MAX(file_summary_by_instance). -# - MIN(file_summary_by_event_name) <= -# MIN(file_summary_by_instance). -# There will be equality only when file instances are not removed, -# aka when a file is not deleted from the file system, -# because doing so removes a row in file_summary_by_instance. - -# Likewise: -# - COUNT/SUM/MAX(events_waits_summary_global_by_event_name) >= -# COUNT/SUM/MAX(events_waits_summary_by_instance) -# - MIN(events_waits_summary_global_by_event_name) <= -# MIN(events_waits_summary_by_instance) -# There will be equality only when an instrument instance -# is not removed, which is next to impossible to predictably guarantee -# in the server. -# For example, a MyISAM table removed from the table cache -# will cause a mysql_mutex_destroy on myisam/MYISAM_SHARE::intern_lock. -# Another example, a thread terminating will cause a mysql_mutex_destroy -# on sql/LOCK_delete -# Both cause a row to be deleted from events_waits_summary_by_instance. - -# Likewise: -# - COUNT/SUM/MAX(events_waits_summary_global_by_event_name) >= -# COUNT/SUM/MAX(events_waits_summary_by_thread_by_event_name) -# - MIN(events_waits_summary_global_by_event_name) <= -# MIN(events_waits_summary_by_thread_by_event_name) -# There will be equality only when no thread is removed, -# that is if no thread disconnects, or no sub thread (for example insert -# delayed) ever completes. -# A thread completing will cause rows in -# events_waits_summary_by_thread_by_event_name to be removed. - ---echo "Verifying file aggregate consistency" - -# Since the code generating the load in this test does: -# - create table -# - insert -# - does not cause temporary tables to be used -# we can test for equality here for file aggregates. - -# If any of these queries returns data, the test failed. - -SELECT EVENT_NAME, e.COUNT_READ, SUM(i.COUNT_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_READ <> SUM(i.COUNT_READ)) -OR @dump_all; - -SELECT EVENT_NAME, e.COUNT_WRITE, SUM(i.COUNT_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_WRITE <> SUM(i.COUNT_WRITE)) -OR @dump_all; - -SELECT EVENT_NAME, e.COUNT_READ, SUM(i.COUNT_READ) -FROM performance_schema.socket_summary_by_event_name AS e -JOIN performance_schema.socket_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_READ <> SUM(i.COUNT_READ)) -OR @dump_all; - -SELECT EVENT_NAME, e.COUNT_WRITE, SUM(i.COUNT_WRITE) -FROM performance_schema.socket_summary_by_event_name AS e -JOIN performance_schema.socket_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.COUNT_WRITE <> SUM(i.COUNT_WRITE)) -OR @dump_all; - -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_READ, SUM(i.SUM_NUMBER_OF_BYTES_READ) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_READ <> SUM(i.SUM_NUMBER_OF_BYTES_READ)) -OR @dump_all; - -SELECT EVENT_NAME, e.SUM_NUMBER_OF_BYTES_WRITE, SUM(i.SUM_NUMBER_OF_BYTES_WRITE) -FROM performance_schema.file_summary_by_event_name AS e -JOIN performance_schema.file_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_NUMBER_OF_BYTES_WRITE <> SUM(i.SUM_NUMBER_OF_BYTES_WRITE)) -OR @dump_all; - ---echo "Verifying waits aggregate consistency (instance)" - -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(i.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(i.SUM_TIMER_WAIT)) -OR @dump_all; - -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(i.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(i.MIN_TIMER_WAIT)) -AND (MIN(i.MIN_TIMER_WAIT) != 0) -OR @dump_all; - -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(i.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_instance AS i USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(i.MAX_TIMER_WAIT)) -OR @dump_all; - ---echo "Verifying waits aggregate consistency (thread)" - -SELECT EVENT_NAME, e.SUM_TIMER_WAIT, SUM(t.SUM_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.SUM_TIMER_WAIT < SUM(t.SUM_TIMER_WAIT)) -OR @dump_all; - -SELECT EVENT_NAME, e.MIN_TIMER_WAIT, MIN(t.MIN_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MIN_TIMER_WAIT > MIN(t.MIN_TIMER_WAIT)) -AND (MIN(t.MIN_TIMER_WAIT) != 0) -OR @dump_all; - -SELECT EVENT_NAME, e.MAX_TIMER_WAIT, MAX(t.MAX_TIMER_WAIT) -FROM performance_schema.events_waits_summary_global_by_event_name AS e -JOIN performance_schema.events_waits_summary_by_thread_by_event_name AS t -USING (EVENT_NAME) -GROUP BY EVENT_NAME -HAVING (e.MAX_TIMER_WAIT < MAX(t.MAX_TIMER_WAIT)) -OR @dump_all; - - -# Cleanup - -update performance_schema.setup_consumers set enabled = 'YES'; -update performance_schema.setup_instruments - set enabled = 'YES', timed = 'YES'; - -drop table test.t1; From e312e2e636b84ba4d0d64cc5a7bb368d3286c5ed Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Sep 2016 17:59:58 +0200 Subject: [PATCH 075/295] 5.6.32-78.1 --- storage/tokudb/CMakeLists.txt | 2 +- .../tokudb/PerconaFT/buildheader/make_tdb.cc | 4 +- .../cmake_modules/TokuSetupCompiler.cmake | 1 + storage/tokudb/PerconaFT/ft/CMakeLists.txt | 2 +- storage/tokudb/PerconaFT/ft/ft-flusher.cc | 4 +- storage/tokudb/PerconaFT/ft/ft-ops.cc | 215 +++-- .../tokudb/PerconaFT/ft/ft-recount-rows.cc | 29 +- storage/tokudb/PerconaFT/ft/ft.cc | 3 + .../PerconaFT/ft/loader/loader-internal.h | 2 +- storage/tokudb/PerconaFT/ft/loader/loader.cc | 2 +- storage/tokudb/PerconaFT/ft/node.cc | 125 +-- storage/tokudb/PerconaFT/ft/node.h | 1 + .../PerconaFT/ft/serialize/block_allocator.cc | 473 +++------- .../PerconaFT/ft/serialize/block_allocator.h | 162 ++-- .../ft/serialize/block_allocator_strategy.cc | 224 ----- .../PerconaFT/ft/serialize/block_table.cc | 632 ++++++++----- .../PerconaFT/ft/serialize/block_table.h | 143 ++- .../tokudb/PerconaFT/ft/serialize/compress.cc | 2 +- .../PerconaFT/ft/serialize/ft-serialize.cc | 350 ++++---- .../ft/serialize/ft_node-serialize.cc | 69 +- .../PerconaFT/ft/serialize/rbtree_mhs.cc | 833 ++++++++++++++++++ .../PerconaFT/ft/serialize/rbtree_mhs.h | 351 ++++++++ .../ft/tests/block_allocator_strategy_test.cc | 126 --- .../ft/tests/block_allocator_test.cc | 380 ++++---- .../PerconaFT/ft/tests/cachetable-5978.cc | 2 +- .../ft/tests/cachetable-simple-clone2.cc | 2 +- .../tokudb/PerconaFT/ft/tests/ft-bfe-query.cc | 403 +++++---- .../PerconaFT/ft/tests/ft-clock-test.cc | 281 +++--- .../ft/tests/ft-serialize-benchmark.cc | 231 +++-- .../PerconaFT/ft/tests/ft-serialize-test.cc | 831 ++++++++++------- storage/tokudb/PerconaFT/ft/tests/ft-test.cc | 11 +- .../tokudb/PerconaFT/ft/tests/pqueue-test.cc | 4 +- .../ft/tests/test-leafentry-nested.cc | 2 +- .../tests/test-oldest-referenced-xid-flush.cc | 6 +- .../test-rbtree-insert-remove-with-mhs.cc} | 86 +- .../test-rbtree-insert-remove-without-mhs.cc | 102 +++ storage/tokudb/PerconaFT/ft/txn/roll.cc | 2 +- .../tokudb/PerconaFT/ft/txn/rollback-apply.cc | 2 +- .../PerconaFT/ft/txn/rollback-ct-callbacks.cc | 23 +- storage/tokudb/PerconaFT/ft/ule.cc | 4 +- .../portability/tests/test-max-data.cc | 2 +- .../PerconaFT/portability/toku_config.h.in | 1 - .../tokudb/PerconaFT/portability/toku_time.h | 8 + .../tokudb/PerconaFT/src/indexer-internal.h | 2 +- .../tokudb/PerconaFT/src/indexer-undo-do.cc | 4 +- .../hotindexer-undo-do-tests/commit.i0.test | 2 +- .../PerconaFT/src/tests/loader-dup-test.cc | 2 +- .../src/tests/recovery_fileops_unit.cc | 2 +- .../src/tests/stat64-root-changes.cc | 4 +- .../src/tests/test_insert_many_gc.cc | 2 +- .../PerconaFT/src/tests/test_stress0.cc | 2 +- .../PerconaFT/src/tests/test_txn_abort5a.cc | 3 +- storage/tokudb/PerconaFT/src/ydb-internal.h | 2 +- .../xz-4.999.9beta/build-aux/config.guess | 449 +++++----- storage/tokudb/PerconaFT/tools/CMakeLists.txt | 3 +- storage/tokudb/PerconaFT/tools/ba_replay.cc | 629 ------------- storage/tokudb/PerconaFT/tools/ftverify.cc | 2 +- storage/tokudb/PerconaFT/tools/tokuftdump.cc | 1 + .../tokudb/PerconaFT/util/tests/x1764-test.cc | 2 +- storage/tokudb/ha_tokudb.cc | 26 +- storage/tokudb/ha_tokudb_admin.cc | 278 +++--- storage/tokudb/hatoku_defines.h | 7 +- .../tokudb/r/background_job_manager.result | 2 +- .../mysql-test/tokudb_bugs/t/frm_store.test | 26 +- .../mysql-test/tokudb_bugs/t/frm_store2.test | 26 +- .../mysql-test/tokudb_bugs/t/frm_store3.test | 26 +- .../t/tokudb_drop_part_table_668.test | 41 +- .../t/tokudb_drop_simple_table_668.test | 41 +- .../r/rpl_foreign_key_tokudb.result | 58 -- .../tokudb_rpl/t/rpl_foreign_key_tokudb.test | 4 - storage/tokudb/tokudb_background.cc | 27 +- storage/tokudb/tokudb_background.h | 49 +- storage/tokudb/tokudb_information_schema.cc | 47 +- 73 files changed, 4192 insertions(+), 3715 deletions(-) delete mode 100644 storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.cc create mode 100644 storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.cc create mode 100644 storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h delete mode 100644 storage/tokudb/PerconaFT/ft/tests/block_allocator_strategy_test.cc rename storage/tokudb/PerconaFT/ft/{serialize/block_allocator_strategy.h => tests/test-rbtree-insert-remove-with-mhs.cc} (55%) create mode 100644 storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc delete mode 100644 storage/tokudb/PerconaFT/tools/ba_replay.cc delete mode 100644 storage/tokudb/mysql-test/tokudb_rpl/r/rpl_foreign_key_tokudb.result delete mode 100644 storage/tokudb/mysql-test/tokudb_rpl/t/rpl_foreign_key_tokudb.test diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 4ec539f7d0b8a..fbb02582f4da6 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(TOKUDB_VERSION 5.6.31-77.0) +SET(TOKUDB_VERSION 5.6.32-78.1) # PerconaFT only supports x86-64 and cmake-2.8.9+ IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT CMAKE_VERSION VERSION_LESS "2.8.9") diff --git a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc index 4b62703480f16..576f902f6aee8 100644 --- a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc +++ b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc @@ -367,8 +367,8 @@ static void print_db_env_struct (void) { "int (*checkpointing_get_period) (DB_ENV*, uint32_t*) /* Retrieve the delay between automatic checkpoints. 0 means disabled. */", "int (*cleaner_set_period) (DB_ENV*, uint32_t) /* Change the delay between automatic cleaner attempts. 0 means disabled. */", "int (*cleaner_get_period) (DB_ENV*, uint32_t*) /* Retrieve the delay between automatic cleaner attempts. 0 means disabled. */", - "int (*cleaner_set_iterations) (DB_ENV*, uint32_t) /* Change the number of attempts on each cleaner invokation. 0 means disabled. */", - "int (*cleaner_get_iterations) (DB_ENV*, uint32_t*) /* Retrieve the number of attempts on each cleaner invokation. 0 means disabled. */", + "int (*cleaner_set_iterations) (DB_ENV*, uint32_t) /* Change the number of attempts on each cleaner invocation. 0 means disabled. */", + "int (*cleaner_get_iterations) (DB_ENV*, uint32_t*) /* Retrieve the number of attempts on each cleaner invocation. 0 means disabled. */", "int (*evictor_set_enable_partial_eviction) (DB_ENV*, bool) /* Enables or disabled partial eviction of nodes from cachetable. */", "int (*evictor_get_enable_partial_eviction) (DB_ENV*, bool*) /* Retrieve the status of partial eviction of nodes from cachetable. */", "int (*checkpointing_postpone) (DB_ENV*) /* Use for 'rename table' or any other operation that must be disjoint from a checkpoint */", diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake index 5f2c9ef2c2aa6..ea5c7e2220924 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake @@ -101,6 +101,7 @@ set_cflags_if_supported( -Wno-pointer-bool-conversion -fno-rtti -fno-exceptions + -Wno-error=nonnull-compare ) ## set_cflags_if_supported_named("-Weffc++" -Weffcpp) diff --git a/storage/tokudb/PerconaFT/ft/CMakeLists.txt b/storage/tokudb/PerconaFT/ft/CMakeLists.txt index 11091073ac2a1..6696c26ecc00c 100644 --- a/storage/tokudb/PerconaFT/ft/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/ft/CMakeLists.txt @@ -55,8 +55,8 @@ set(FT_SOURCES msg_buffer node pivotkeys + serialize/rbtree_mhs serialize/block_allocator - serialize/block_allocator_strategy serialize/block_table serialize/compress serialize/ft_node-serialize diff --git a/storage/tokudb/PerconaFT/ft/ft-flusher.cc b/storage/tokudb/PerconaFT/ft/ft-flusher.cc index fb456ea6a18e3..e6452f60cfcc6 100644 --- a/storage/tokudb/PerconaFT/ft/ft-flusher.cc +++ b/storage/tokudb/PerconaFT/ft/ft-flusher.cc @@ -496,7 +496,7 @@ handle_split_of_child( // We never set the rightmost blocknum to be the root. // Instead, we wait for the root to split and let promotion initialize the rightmost - // blocknum to be the first non-root leaf node on the right extreme to recieve an insert. + // blocknum to be the first non-root leaf node on the right extreme to receive an insert. BLOCKNUM rightmost_blocknum = toku_unsafe_fetch(&ft->rightmost_blocknum); invariant(ft->h->root_blocknum.b != rightmost_blocknum.b); if (childa->blocknum.b == rightmost_blocknum.b) { @@ -1470,7 +1470,7 @@ void toku_ft_flush_some_child(FT ft, FTNODE parent, struct flusher_advice *fa) // It is possible after reading in the entire child, // that we now know that the child is not reactive // if so, we can unpin parent right now - // we wont be splitting/merging child + // we won't be splitting/merging child // and we have already replaced the bnc // for the root with a fresh one enum reactivity child_re = toku_ftnode_get_reactivity(ft, child); diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index 8f61bc67339fa..f131668889e9e 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -598,15 +598,12 @@ void toku_ftnode_checkpoint_complete_callback(void *value_data) { } } -void toku_ftnode_clone_callback( - void* value_data, - void** cloned_value_data, - long* clone_size, - PAIR_ATTR* new_attr, - bool for_checkpoint, - void* write_extraargs - ) -{ +void toku_ftnode_clone_callback(void *value_data, + void **cloned_value_data, + long *clone_size, + PAIR_ATTR *new_attr, + bool for_checkpoint, + void *write_extraargs) { FTNODE node = static_cast(value_data); toku_ftnode_assert_fully_in_memory(node); FT ft = static_cast(write_extraargs); @@ -618,13 +615,16 @@ void toku_ftnode_clone_callback( toku_ftnode_leaf_rebalance(node, ft->h->basementnodesize); } - cloned_node->oldest_referenced_xid_known = node->oldest_referenced_xid_known; - cloned_node->max_msn_applied_to_node_on_disk = node->max_msn_applied_to_node_on_disk; + cloned_node->oldest_referenced_xid_known = + node->oldest_referenced_xid_known; + cloned_node->max_msn_applied_to_node_on_disk = + node->max_msn_applied_to_node_on_disk; cloned_node->flags = node->flags; cloned_node->blocknum = node->blocknum; cloned_node->layout_version = node->layout_version; cloned_node->layout_version_original = node->layout_version_original; - cloned_node->layout_version_read_from_disk = node->layout_version_read_from_disk; + cloned_node->layout_version_read_from_disk = + node->layout_version_read_from_disk; cloned_node->build_id = node->build_id; cloned_node->height = node->height; cloned_node->dirty = node->dirty; @@ -649,38 +649,39 @@ void toku_ftnode_clone_callback( // set new pair attr if necessary if (node->height == 0) { *new_attr = make_ftnode_pair_attr(node); - } - else { + for (int i = 0; i < node->n_children; i++) { + BLB(node, i)->logical_rows_delta = 0; + BLB(cloned_node, i)->logical_rows_delta = 0; + } + } else { new_attr->is_valid = false; } *clone_size = ftnode_memory_size(cloned_node); *cloned_value_data = cloned_node; } -void toku_ftnode_flush_callback( - CACHEFILE UU(cachefile), - int fd, - BLOCKNUM blocknum, - void *ftnode_v, - void** disk_data, - void *extraargs, - PAIR_ATTR size __attribute__((unused)), - PAIR_ATTR* new_size, - bool write_me, - bool keep_me, - bool for_checkpoint, - bool is_clone - ) -{ - FT ft = (FT) extraargs; - FTNODE ftnode = (FTNODE) ftnode_v; - FTNODE_DISK_DATA* ndd = (FTNODE_DISK_DATA*)disk_data; +void toku_ftnode_flush_callback(CACHEFILE UU(cachefile), + int fd, + BLOCKNUM blocknum, + void *ftnode_v, + void **disk_data, + void *extraargs, + PAIR_ATTR size __attribute__((unused)), + PAIR_ATTR *new_size, + bool write_me, + bool keep_me, + bool for_checkpoint, + bool is_clone) { + FT ft = (FT)extraargs; + FTNODE ftnode = (FTNODE)ftnode_v; + FTNODE_DISK_DATA *ndd = (FTNODE_DISK_DATA *)disk_data; assert(ftnode->blocknum.b == blocknum.b); int height = ftnode->height; if (write_me) { toku_ftnode_assert_fully_in_memory(ftnode); if (height > 0 && !is_clone) { - // cloned nodes already had their stale messages moved, see toku_ftnode_clone_callback() + // cloned nodes already had their stale messages moved, see + // toku_ftnode_clone_callback() toku_move_ftnode_messages_to_stale(ft, ftnode); } else if (height == 0) { toku_ftnode_leaf_run_gc(ft, ftnode); @@ -688,7 +689,8 @@ void toku_ftnode_flush_callback( toku_ftnode_update_disk_stats(ftnode, ft, for_checkpoint); } } - int r = toku_serialize_ftnode_to(fd, ftnode->blocknum, ftnode, ndd, !is_clone, ft, for_checkpoint); + int r = toku_serialize_ftnode_to( + fd, ftnode->blocknum, ftnode, ndd, !is_clone, ft, for_checkpoint); assert_zero(r); ftnode->layout_version_read_from_disk = FT_LAYOUT_VERSION; } @@ -703,20 +705,22 @@ void toku_ftnode_flush_callback( FT_STATUS_INC(FT_FULL_EVICTIONS_NONLEAF_BYTES, node_size); } toku_free(*disk_data); - } - else { + } else { if (ftnode->height == 0) { for (int i = 0; i < ftnode->n_children; i++) { - if (BP_STATE(ftnode,i) == PT_AVAIL) { + if (BP_STATE(ftnode, i) == PT_AVAIL) { BASEMENTNODE bn = BLB(ftnode, i); - toku_ft_decrease_stats(&ft->in_memory_stats, bn->stat64_delta); + toku_ft_decrease_stats(&ft->in_memory_stats, + bn->stat64_delta); + if (!ftnode->dirty) + toku_ft_adjust_logical_row_count( + ft, -bn->logical_rows_delta); } } } } toku_ftnode_free(&ftnode); - } - else { + } else { *new_size = make_ftnode_pair_attr(ftnode); } } @@ -845,10 +849,13 @@ static void compress_internal_node_partition(FTNODE node, int i, enum toku_compr } // callback for partially evicting a node -int toku_ftnode_pe_callback(void *ftnode_pv, PAIR_ATTR old_attr, void *write_extraargs, - void (*finalize)(PAIR_ATTR new_attr, void *extra), void *finalize_extra) { - FTNODE node = (FTNODE) ftnode_pv; - FT ft = (FT) write_extraargs; +int toku_ftnode_pe_callback(void *ftnode_pv, + PAIR_ATTR old_attr, + void *write_extraargs, + void (*finalize)(PAIR_ATTR new_attr, void *extra), + void *finalize_extra) { + FTNODE node = (FTNODE)ftnode_pv; + FT ft = (FT)write_extraargs; int num_partial_evictions = 0; // Hold things we intend to destroy here. @@ -866,7 +873,8 @@ int toku_ftnode_pe_callback(void *ftnode_pv, PAIR_ATTR old_attr, void *write_ext } // Don't partially evict nodes whose partitions can't be read back // from disk individually - if (node->layout_version_read_from_disk < FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES) { + if (node->layout_version_read_from_disk < + FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES) { goto exit; } // @@ -874,77 +882,77 @@ int toku_ftnode_pe_callback(void *ftnode_pv, PAIR_ATTR old_attr, void *write_ext // if (node->height > 0) { for (int i = 0; i < node->n_children; i++) { - if (BP_STATE(node,i) == PT_AVAIL) { - if (BP_SHOULD_EVICT(node,i)) { + if (BP_STATE(node, i) == PT_AVAIL) { + if (BP_SHOULD_EVICT(node, i)) { NONLEAF_CHILDINFO bnc = BNC(node, i); if (ft_compress_buffers_before_eviction && - // We may not serialize and compress a partition in memory if its - // in memory layout version is different than what's on disk (and - // therefore requires upgrade). + // We may not serialize and compress a partition in + // memory if its in memory layout version is different + // than what's on disk (and therefore requires upgrade). // - // Auto-upgrade code assumes that if a node's layout version read - // from disk is not current, it MUST require upgrade. Breaking - // this rule would cause upgrade code to upgrade this partition - // again after we serialize it as the current version, which is bad. - node->layout_version == node->layout_version_read_from_disk) { + // Auto-upgrade code assumes that if a node's layout + // version read from disk is not current, it MUST + // require upgrade. + // Breaking this rule would cause upgrade code to + // upgrade this partition again after we serialize it as + // the current version, which is bad. + node->layout_version == + node->layout_version_read_from_disk) { toku_ft_bnc_move_messages_to_stale(ft, bnc); compress_internal_node_partition( node, i, // Always compress with quicklz - TOKU_QUICKLZ_METHOD - ); + TOKU_QUICKLZ_METHOD); } else { // We're not compressing buffers before eviction. Simply - // detach the buffer and set the child's state to on-disk. + // detach the buffer and set the child's state to + // on-disk. set_BNULL(node, i); BP_STATE(node, i) = PT_ON_DISK; } buffers_to_destroy[num_buffers_to_destroy++] = bnc; num_partial_evictions++; + } else { + BP_SWEEP_CLOCK(node, i); } - else { - BP_SWEEP_CLOCK(node,i); - } - } - else { + } else { continue; } } - } - // - // partial eviction strategy for basement nodes: - // if the bn is compressed, evict it - // else: check if it requires eviction, if it does, evict it, if not, sweep the clock count - // - else { + } else { + // + // partial eviction strategy for basement nodes: + // if the bn is compressed, evict it + // else: check if it requires eviction, if it does, evict it, if not, + // sweep the clock count + // for (int i = 0; i < node->n_children; i++) { // Get rid of compressed stuff no matter what. - if (BP_STATE(node,i) == PT_COMPRESSED) { + if (BP_STATE(node, i) == PT_COMPRESSED) { SUB_BLOCK sb = BSB(node, i); pointers_to_free[num_pointers_to_free++] = sb->compressed_ptr; pointers_to_free[num_pointers_to_free++] = sb; set_BNULL(node, i); - BP_STATE(node,i) = PT_ON_DISK; + BP_STATE(node, i) = PT_ON_DISK; num_partial_evictions++; - } - else if (BP_STATE(node,i) == PT_AVAIL) { - if (BP_SHOULD_EVICT(node,i)) { + } else if (BP_STATE(node, i) == PT_AVAIL) { + if (BP_SHOULD_EVICT(node, i)) { BASEMENTNODE bn = BLB(node, i); basements_to_destroy[num_basements_to_destroy++] = bn; - toku_ft_decrease_stats(&ft->in_memory_stats, bn->stat64_delta); + toku_ft_decrease_stats(&ft->in_memory_stats, + bn->stat64_delta); + toku_ft_adjust_logical_row_count(ft, + -bn->logical_rows_delta); set_BNULL(node, i); BP_STATE(node, i) = PT_ON_DISK; num_partial_evictions++; + } else { + BP_SWEEP_CLOCK(node, i); } - else { - BP_SWEEP_CLOCK(node,i); - } - } - else if (BP_STATE(node,i) == PT_ON_DISK) { + } else if (BP_STATE(node, i) == PT_ON_DISK) { continue; - } - else { + } else { abort(); } } @@ -2378,12 +2386,16 @@ ft_send_update_msg(FT_HANDLE ft_h, const ft_msg &msg, TOKUTXN txn) { toku_ft_root_put_msg(ft_h->ft, msg, &gc_info); } -void toku_ft_maybe_update(FT_HANDLE ft_h, const DBT *key, const DBT *update_function_extra, - TOKUTXN txn, bool oplsn_valid, LSN oplsn, - bool do_logging) { +void toku_ft_maybe_update(FT_HANDLE ft_h, + const DBT *key, + const DBT *update_function_extra, + TOKUTXN txn, + bool oplsn_valid, + LSN oplsn, + bool do_logging) { TXNID_PAIR xid = toku_txn_get_txnid(txn); if (txn) { - BYTESTRING keybs = { key->size, (char *) key->data }; + BYTESTRING keybs = {key->size, (char *)key->data}; toku_logger_save_rollback_cmdupdate( txn, toku_cachefile_filenum(ft_h->ft->cf), &keybs); toku_txn_maybe_note_ft(txn, ft_h->ft); @@ -2392,22 +2404,33 @@ void toku_ft_maybe_update(FT_HANDLE ft_h, const DBT *key, const DBT *update_func TOKULOGGER logger; logger = toku_txn_logger(txn); if (do_logging && logger) { - BYTESTRING keybs = {.len=key->size, .data=(char *) key->data}; - BYTESTRING extrabs = {.len=update_function_extra->size, - .data = (char *) update_function_extra->data}; - toku_log_enq_update(logger, NULL, 0, txn, - toku_cachefile_filenum(ft_h->ft->cf), - xid, keybs, extrabs); + BYTESTRING keybs = {.len = key->size, .data = (char *)key->data}; + BYTESTRING extrabs = {.len = update_function_extra->size, + .data = (char *)update_function_extra->data}; + toku_log_enq_update(logger, + NULL, + 0, + txn, + toku_cachefile_filenum(ft_h->ft->cf), + xid, + keybs, + extrabs); } LSN treelsn; - if (oplsn_valid && oplsn.lsn <= (treelsn = toku_ft_checkpoint_lsn(ft_h->ft)).lsn) { + if (oplsn_valid && + oplsn.lsn <= (treelsn = toku_ft_checkpoint_lsn(ft_h->ft)).lsn) { // do nothing } else { - XIDS message_xids = txn ? toku_txn_get_xids(txn) : toku_xids_get_root_xids(); - ft_msg msg(key, update_function_extra, FT_UPDATE, ZERO_MSN, message_xids); + XIDS message_xids = + txn ? toku_txn_get_xids(txn) : toku_xids_get_root_xids(); + ft_msg msg( + key, update_function_extra, FT_UPDATE, ZERO_MSN, message_xids); ft_send_update_msg(ft_h, msg, txn); } + // updates get converted to insert messages, which should do a -1 on the + // logical row count when the messages are permanently applied + toku_ft_adjust_logical_row_count(ft_h->ft, 1); } void toku_ft_maybe_update_broadcast(FT_HANDLE ft_h, const DBT *update_function_extra, diff --git a/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc b/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc index adac96f488213..e31d80772d56e 100644 --- a/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc +++ b/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc @@ -73,30 +73,20 @@ static bool recount_rows_interrupt(void* extra, uint64_t deleted_rows) { return rre->_cancelled = rre->_progress_callback(rre->_keys, deleted_rows, rre->_progress_extra); } -int toku_ft_recount_rows( - FT_HANDLE ft, - int (*progress_callback)( - uint64_t count, - uint64_t deleted, - void* progress_extra), - void* progress_extra) { - +int toku_ft_recount_rows(FT_HANDLE ft, + int (*progress_callback)(uint64_t count, + uint64_t deleted, + void* progress_extra), + void* progress_extra) { int ret = 0; - recount_rows_extra_t rre = { - progress_callback, - progress_extra, - 0, - false - }; + recount_rows_extra_t rre = {progress_callback, progress_extra, 0, false}; ft_cursor c; ret = toku_ft_cursor_create(ft, &c, nullptr, C_READ_ANY, false, false); - if (ret) return ret; + if (ret) + return ret; - toku_ft_cursor_set_check_interrupt_cb( - &c, - recount_rows_interrupt, - &rre); + toku_ft_cursor_set_check_interrupt_cb(&c, recount_rows_interrupt, &rre); ret = toku_ft_cursor_first(&c, recount_rows_found, &rre); while (FT_LIKELY(ret == 0)) { @@ -108,6 +98,7 @@ int toku_ft_recount_rows( if (rre._cancelled == false) { // update ft count toku_unsafe_set(&ft->ft->in_memory_logical_rows, rre._keys); + ft->ft->h->dirty = 1; ret = 0; } diff --git a/storage/tokudb/PerconaFT/ft/ft.cc b/storage/tokudb/PerconaFT/ft/ft.cc index 93d21233bf73d..699fcc5760375 100644 --- a/storage/tokudb/PerconaFT/ft/ft.cc +++ b/storage/tokudb/PerconaFT/ft/ft.cc @@ -903,6 +903,9 @@ void toku_ft_adjust_logical_row_count(FT ft, int64_t delta) { // must be returned in toku_ft_stat64. if (delta != 0 && ft->in_memory_logical_rows != (uint64_t)-1) { toku_sync_fetch_and_add(&(ft->in_memory_logical_rows), delta); + if (ft->in_memory_logical_rows == (uint64_t)-1) { + toku_sync_fetch_and_add(&(ft->in_memory_logical_rows), 1); + } } } diff --git a/storage/tokudb/PerconaFT/ft/loader/loader-internal.h b/storage/tokudb/PerconaFT/ft/loader/loader-internal.h index dd070373e26d8..1aa2c20383150 100644 --- a/storage/tokudb/PerconaFT/ft/loader/loader-internal.h +++ b/storage/tokudb/PerconaFT/ft/loader/loader-internal.h @@ -301,7 +301,7 @@ int toku_ft_loader_internal_init (/* out */ FTLOADER *blp, void toku_ft_loader_internal_destroy (FTLOADER bl, bool is_error); -// For test purposes only. (In production, the rowset size is determined by negotation with the cachetable for some memory. See #2613.) +// For test purposes only. (In production, the rowset size is determined by negotiation with the cachetable for some memory. See #2613.) uint64_t toku_ft_loader_get_rowset_budget_for_testing (void); int toku_ft_loader_finish_extractor(FTLOADER bl); diff --git a/storage/tokudb/PerconaFT/ft/loader/loader.cc b/storage/tokudb/PerconaFT/ft/loader/loader.cc index 20f9363da1efa..528c86a8f79bf 100644 --- a/storage/tokudb/PerconaFT/ft/loader/loader.cc +++ b/storage/tokudb/PerconaFT/ft/loader/loader.cc @@ -91,7 +91,7 @@ toku_ft_loader_set_size_factor(uint32_t factor) { uint64_t toku_ft_loader_get_rowset_budget_for_testing (void) -// For test purposes only. In production, the rowset size is determined by negotation with the cachetable for some memory. (See #2613). +// For test purposes only. In production, the rowset size is determined by negotiation with the cachetable for some memory. (See #2613). { return 16ULL*size_factor*1024ULL; } diff --git a/storage/tokudb/PerconaFT/ft/node.cc b/storage/tokudb/PerconaFT/ft/node.cc index 58ba675eb7c24..12e5fda226e95 100644 --- a/storage/tokudb/PerconaFT/ft/node.cc +++ b/storage/tokudb/PerconaFT/ft/node.cc @@ -373,52 +373,48 @@ find_bounds_within_message_tree( } } -/** - * For each message in the ancestor's buffer (determined by childnum) that - * is key-wise between lower_bound_exclusive and upper_bound_inclusive, - * apply the message to the basement node. We treat the bounds as minus - * or plus infinity respectively if they are NULL. Do not mark the node - * as dirty (preserve previous state of 'dirty' bit). - */ +// For each message in the ancestor's buffer (determined by childnum) that +// is key-wise between lower_bound_exclusive and upper_bound_inclusive, +// apply the message to the basement node. We treat the bounds as minus +// or plus infinity respectively if they are NULL. Do not mark the node +// as dirty (preserve previous state of 'dirty' bit). static void bnc_apply_messages_to_basement_node( - FT_HANDLE t, // used for comparison function - BASEMENTNODE bn, // where to apply messages + FT_HANDLE t, // used for comparison function + BASEMENTNODE bn, // where to apply messages FTNODE ancestor, // the ancestor node where we can find messages to apply - int childnum, // which child buffer of ancestor contains messages we want - const pivot_bounds &bounds, // contains pivot key bounds of this basement node - txn_gc_info* gc_info, - bool* msgs_applied) { - + int childnum, // which child buffer of ancestor contains messages we want + const pivot_bounds & + bounds, // contains pivot key bounds of this basement node + txn_gc_info *gc_info, + bool *msgs_applied) { int r; NONLEAF_CHILDINFO bnc = BNC(ancestor, childnum); // Determine the offsets in the message trees between which we need to // apply messages from this buffer - STAT64INFO_S stats_delta = {0,0}; + STAT64INFO_S stats_delta = {0, 0}; uint64_t workdone_this_ancestor = 0; int64_t logical_rows_delta = 0; uint32_t stale_lbi, stale_ube; if (!bn->stale_ancestor_messages_applied) { - find_bounds_within_message_tree( - t->ft->cmp, - bnc->stale_message_tree, - &bnc->msg_buffer, - bounds, - &stale_lbi, - &stale_ube); + find_bounds_within_message_tree(t->ft->cmp, + bnc->stale_message_tree, + &bnc->msg_buffer, + bounds, + &stale_lbi, + &stale_ube); } else { stale_lbi = 0; stale_ube = 0; } uint32_t fresh_lbi, fresh_ube; - find_bounds_within_message_tree( - t->ft->cmp, - bnc->fresh_message_tree, - &bnc->msg_buffer, - bounds, - &fresh_lbi, - &fresh_ube); + find_bounds_within_message_tree(t->ft->cmp, + bnc->fresh_message_tree, + &bnc->msg_buffer, + bounds, + &fresh_lbi, + &fresh_ube); // We now know where all the messages we must apply are, so one of the // following 4 cases will do the application, depending on which of @@ -432,44 +428,53 @@ static void bnc_apply_messages_to_basement_node( // We have messages in multiple trees, so we grab all // the relevant messages' offsets and sort them by MSN, then apply // them in MSN order. - const int buffer_size = ((stale_ube - stale_lbi) + - (fresh_ube - fresh_lbi) + - bnc->broadcast_list.size()); + const int buffer_size = + ((stale_ube - stale_lbi) + (fresh_ube - fresh_lbi) + + bnc->broadcast_list.size()); toku::scoped_malloc offsets_buf(buffer_size * sizeof(int32_t)); int32_t *offsets = reinterpret_cast(offsets_buf.get()); - struct store_msg_buffer_offset_extra sfo_extra = { .offsets = offsets, .i = 0 }; + struct store_msg_buffer_offset_extra sfo_extra = {.offsets = offsets, + .i = 0}; // Populate offsets array with offsets to stale messages - r = bnc->stale_message_tree.iterate_on_range(stale_lbi, stale_ube, &sfo_extra); + r = bnc->stale_message_tree + .iterate_on_range( + stale_lbi, stale_ube, &sfo_extra); assert_zero(r); // Then store fresh offsets, and mark them to be moved to stale later. - r = bnc->fresh_message_tree.iterate_and_mark_range(fresh_lbi, fresh_ube, &sfo_extra); + r = bnc->fresh_message_tree + .iterate_and_mark_range( + fresh_lbi, fresh_ube, &sfo_extra); assert_zero(r); // Store offsets of all broadcast messages. - r = bnc->broadcast_list.iterate(&sfo_extra); + r = bnc->broadcast_list.iterate(&sfo_extra); assert_zero(r); invariant(sfo_extra.i == buffer_size); // Sort by MSN. - toku::sort::mergesort_r(offsets, buffer_size, bnc->msg_buffer); + toku::sort:: + mergesort_r(offsets, buffer_size, bnc->msg_buffer); // Apply the messages in MSN order. for (int i = 0; i < buffer_size; ++i) { *msgs_applied = true; - do_bn_apply_msg( - t, - bn, - &bnc->msg_buffer, - offsets[i], - gc_info, - &workdone_this_ancestor, - &stats_delta, - &logical_rows_delta); + do_bn_apply_msg(t, + bn, + &bnc->msg_buffer, + offsets[i], + gc_info, + &workdone_this_ancestor, + &stats_delta, + &logical_rows_delta); } } else if (stale_lbi == stale_ube) { - // No stale messages to apply, we just apply fresh messages, and mark them to be moved to stale later. + // No stale messages to apply, we just apply fresh messages, and mark + // them to be moved to stale later. struct iterate_do_bn_apply_msg_extra iter_extra = { .t = t, .bn = bn, @@ -477,16 +482,20 @@ static void bnc_apply_messages_to_basement_node( .gc_info = gc_info, .workdone = &workdone_this_ancestor, .stats_to_update = &stats_delta, - .logical_rows_delta = &logical_rows_delta - }; - if (fresh_ube - fresh_lbi > 0) *msgs_applied = true; - r = bnc->fresh_message_tree.iterate_and_mark_range(fresh_lbi, fresh_ube, &iter_extra); + .logical_rows_delta = &logical_rows_delta}; + if (fresh_ube - fresh_lbi > 0) + *msgs_applied = true; + r = bnc->fresh_message_tree + .iterate_and_mark_range( + fresh_lbi, fresh_ube, &iter_extra); assert_zero(r); } else { invariant(fresh_lbi == fresh_ube); // No fresh messages to apply, we just apply stale messages. - if (stale_ube - stale_lbi > 0) *msgs_applied = true; + if (stale_ube - stale_lbi > 0) + *msgs_applied = true; struct iterate_do_bn_apply_msg_extra iter_extra = { .t = t, .bn = bn, @@ -494,22 +503,26 @@ static void bnc_apply_messages_to_basement_node( .gc_info = gc_info, .workdone = &workdone_this_ancestor, .stats_to_update = &stats_delta, - .logical_rows_delta = &logical_rows_delta - }; + .logical_rows_delta = &logical_rows_delta}; - r = bnc->stale_message_tree.iterate_on_range(stale_lbi, stale_ube, &iter_extra); + r = bnc->stale_message_tree + .iterate_on_range( + stale_lbi, stale_ube, &iter_extra); assert_zero(r); } // // update stats // if (workdone_this_ancestor > 0) { - (void) toku_sync_fetch_and_add(&BP_WORKDONE(ancestor, childnum), workdone_this_ancestor); + (void)toku_sync_fetch_and_add(&BP_WORKDONE(ancestor, childnum), + workdone_this_ancestor); } if (stats_delta.numbytes || stats_delta.numrows) { toku_ft_update_stats(&t->ft->in_memory_stats, stats_delta); } toku_ft_adjust_logical_row_count(t->ft, logical_rows_delta); + bn->logical_rows_delta += logical_rows_delta; } static void diff --git a/storage/tokudb/PerconaFT/ft/node.h b/storage/tokudb/PerconaFT/ft/node.h index ad0298e81c5de..52eefec0936ae 100644 --- a/storage/tokudb/PerconaFT/ft/node.h +++ b/storage/tokudb/PerconaFT/ft/node.h @@ -199,6 +199,7 @@ struct ftnode_leaf_basement_node { MSN max_msn_applied; // max message sequence number applied bool stale_ancestor_messages_applied; STAT64INFO_S stat64_delta; // change in stat64 counters since basement was last written to disk + int64_t logical_rows_delta; }; typedef struct ftnode_leaf_basement_node *BASEMENTNODE; diff --git a/storage/tokudb/PerconaFT/ft/serialize/block_allocator.cc b/storage/tokudb/PerconaFT/ft/serialize/block_allocator.cc index 1355f3739ee1b..19811373d1636 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/block_allocator.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/block_allocator.cc @@ -46,415 +46,214 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "portability/toku_stdlib.h" #include "ft/serialize/block_allocator.h" -#include "ft/serialize/block_allocator_strategy.h" +#include "ft/serialize/rbtree_mhs.h" #if TOKU_DEBUG_PARANOID -#define VALIDATE() validate() +#define VALIDATE() Validate() #else #define VALIDATE() #endif -static FILE *ba_trace_file = nullptr; - -void block_allocator::maybe_initialize_trace(void) { - const char *ba_trace_path = getenv("TOKU_BA_TRACE_PATH"); - if (ba_trace_path != nullptr) { - ba_trace_file = toku_os_fopen(ba_trace_path, "w"); - if (ba_trace_file == nullptr) { - fprintf(stderr, "tokuft: error: block allocator trace path found in environment (%s), " - "but it could not be opened for writing (errno %d)\n", - ba_trace_path, get_maybe_error_errno()); - } else { - fprintf(stderr, "tokuft: block allocator tracing enabled, path: %s\n", ba_trace_path); - } - } -} - -void block_allocator::maybe_close_trace() { - if (ba_trace_file != nullptr) { - int r = toku_os_fclose(ba_trace_file); - if (r != 0) { - fprintf(stderr, "tokuft: error: block allocator trace file did not close properly (r %d, errno %d)\n", - r, get_maybe_error_errno()); - } else { - fprintf(stderr, "tokuft: block allocator tracing finished, file closed successfully\n"); - } - } -} - -void block_allocator::_create_internal(uint64_t reserve_at_beginning, uint64_t alignment) { - // the alignment must be at least 512 and aligned with 512 to work with direct I/O - assert(alignment >= 512 && (alignment % 512) == 0); +void BlockAllocator::CreateInternal(uint64_t reserve_at_beginning, + uint64_t alignment) { + // the alignment must be at least 512 and aligned with 512 to work with + // direct I/O + invariant(alignment >= 512 && (alignment % 512) == 0); _reserve_at_beginning = reserve_at_beginning; _alignment = alignment; _n_blocks = 0; - _blocks_array_size = 1; - XMALLOC_N(_blocks_array_size, _blocks_array); _n_bytes_in_use = reserve_at_beginning; - _strategy = BA_STRATEGY_FIRST_FIT; - - memset(&_trace_lock, 0, sizeof(toku_mutex_t)); - toku_mutex_init(&_trace_lock, nullptr); + _tree = new MhsRbTree::Tree(alignment); +} +void BlockAllocator::Create(uint64_t reserve_at_beginning, uint64_t alignment) { + CreateInternal(reserve_at_beginning, alignment); + _tree->Insert({reserve_at_beginning, MAX_BYTE}); VALIDATE(); } -void block_allocator::create(uint64_t reserve_at_beginning, uint64_t alignment) { - _create_internal(reserve_at_beginning, alignment); - _trace_create(); +void BlockAllocator::Destroy() { + delete _tree; } -void block_allocator::destroy() { - toku_free(_blocks_array); - _trace_destroy(); - toku_mutex_destroy(&_trace_lock); -} +void BlockAllocator::CreateFromBlockPairs(uint64_t reserve_at_beginning, + uint64_t alignment, + struct BlockPair *translation_pairs, + uint64_t n_blocks) { + CreateInternal(reserve_at_beginning, alignment); + _n_blocks = n_blocks; -void block_allocator::set_strategy(enum allocation_strategy strategy) { - _strategy = strategy; -} + struct BlockPair *XMALLOC_N(n_blocks, pairs); + memcpy(pairs, translation_pairs, n_blocks * sizeof(struct BlockPair)); + std::sort(pairs, pairs + n_blocks); -void block_allocator::grow_blocks_array_by(uint64_t n_to_add) { - if (_n_blocks + n_to_add > _blocks_array_size) { - uint64_t new_size = _n_blocks + n_to_add; - uint64_t at_least = _blocks_array_size * 2; - if (at_least > new_size) { - new_size = at_least; - } - _blocks_array_size = new_size; - XREALLOC_N(_blocks_array_size, _blocks_array); + if (pairs[0]._offset > reserve_at_beginning) { + _tree->Insert( + {reserve_at_beginning, pairs[0]._offset - reserve_at_beginning}); } -} - -void block_allocator::grow_blocks_array() { - grow_blocks_array_by(1); -} - -void block_allocator::create_from_blockpairs(uint64_t reserve_at_beginning, uint64_t alignment, - struct blockpair *pairs, uint64_t n_blocks) { - _create_internal(reserve_at_beginning, alignment); - - _n_blocks = n_blocks; - grow_blocks_array_by(_n_blocks); - memcpy(_blocks_array, pairs, _n_blocks * sizeof(struct blockpair)); - std::sort(_blocks_array, _blocks_array + _n_blocks); for (uint64_t i = 0; i < _n_blocks; i++) { - // Allocator does not support size 0 blocks. See block_allocator_free_block. - invariant(_blocks_array[i].size > 0); - invariant(_blocks_array[i].offset >= _reserve_at_beginning); - invariant(_blocks_array[i].offset % _alignment == 0); - - _n_bytes_in_use += _blocks_array[i].size; + // Allocator does not support size 0 blocks. See + // block_allocator_free_block. + invariant(pairs[i]._size > 0); + invariant(pairs[i]._offset >= _reserve_at_beginning); + invariant(pairs[i]._offset % _alignment == 0); + + _n_bytes_in_use += pairs[i]._size; + + MhsRbTree::OUUInt64 free_size(MAX_BYTE); + MhsRbTree::OUUInt64 free_offset(pairs[i]._offset + pairs[i]._size); + if (i < n_blocks - 1) { + MhsRbTree::OUUInt64 next_offset(pairs[i + 1]._offset); + invariant(next_offset >= free_offset); + free_size = next_offset - free_offset; + if (free_size == 0) + continue; + } + _tree->Insert({free_offset, free_size}); } - + toku_free(pairs); VALIDATE(); - - _trace_create_from_blockpairs(); } // Effect: align a value by rounding up. -static inline uint64_t align(uint64_t value, uint64_t ba_alignment) { +static inline uint64_t Align(uint64_t value, uint64_t ba_alignment) { return ((value + ba_alignment - 1) / ba_alignment) * ba_alignment; } -struct block_allocator::blockpair * -block_allocator::choose_block_to_alloc_after(size_t size, uint64_t heat) { - switch (_strategy) { - case BA_STRATEGY_FIRST_FIT: - return block_allocator_strategy::first_fit(_blocks_array, _n_blocks, size, _alignment); - case BA_STRATEGY_BEST_FIT: - return block_allocator_strategy::best_fit(_blocks_array, _n_blocks, size, _alignment); - case BA_STRATEGY_HEAT_ZONE: - return block_allocator_strategy::heat_zone(_blocks_array, _n_blocks, size, _alignment, heat); - case BA_STRATEGY_PADDED_FIT: - return block_allocator_strategy::padded_fit(_blocks_array, _n_blocks, size, _alignment); - default: - abort(); - } -} - -// Effect: Allocate a block. The resulting block must be aligned on the ba->alignment (which to make direct_io happy must be a positive multiple of 512). -void block_allocator::alloc_block(uint64_t size, uint64_t heat, uint64_t *offset) { - struct blockpair *bp; - +// Effect: Allocate a block. The resulting block must be aligned on the +// ba->alignment (which to make direct_io happy must be a positive multiple of +// 512). +void BlockAllocator::AllocBlock(uint64_t size, + uint64_t *offset) { // Allocator does not support size 0 blocks. See block_allocator_free_block. invariant(size > 0); - grow_blocks_array(); _n_bytes_in_use += size; + *offset = _tree->Remove(size); - uint64_t end_of_reserve = align(_reserve_at_beginning, _alignment); - - if (_n_blocks == 0) { - // First and only block - assert(_n_bytes_in_use == _reserve_at_beginning + size); // we know exactly how many are in use - _blocks_array[0].offset = align(_reserve_at_beginning, _alignment); - _blocks_array[0].size = size; - *offset = _blocks_array[0].offset; - goto done; - } else if (end_of_reserve + size <= _blocks_array[0].offset ) { - // Check to see if the space immediately after the reserve is big enough to hold the new block. - bp = &_blocks_array[0]; - memmove(bp + 1, bp, _n_blocks * sizeof(*bp)); - bp[0].offset = end_of_reserve; - bp[0].size = size; - *offset = end_of_reserve; - goto done; - } - - bp = choose_block_to_alloc_after(size, heat); - if (bp != nullptr) { - // our allocation strategy chose the space after `bp' to fit the new block - uint64_t answer_offset = align(bp->offset + bp->size, _alignment); - uint64_t blocknum = bp - _blocks_array; - invariant(&_blocks_array[blocknum] == bp); - invariant(blocknum < _n_blocks); - memmove(bp + 2, bp + 1, (_n_blocks - blocknum - 1) * sizeof(*bp)); - bp[1].offset = answer_offset; - bp[1].size = size; - *offset = answer_offset; - } else { - // It didn't fit anywhere, so fit it on the end. - assert(_n_blocks < _blocks_array_size); - bp = &_blocks_array[_n_blocks]; - uint64_t answer_offset = align(bp[-1].offset + bp[-1].size, _alignment); - bp->offset = answer_offset; - bp->size = size; - *offset = answer_offset; - } - -done: _n_blocks++; VALIDATE(); - - _trace_alloc(size, heat, *offset); -} - -// Find the index in the blocks array that has a particular offset. Requires that the block exist. -// Use binary search so it runs fast. -int64_t block_allocator::find_block(uint64_t offset) { - VALIDATE(); - if (_n_blocks == 1) { - assert(_blocks_array[0].offset == offset); - return 0; - } - - uint64_t lo = 0; - uint64_t hi = _n_blocks; - while (1) { - assert(lo < hi); // otherwise no such block exists. - uint64_t mid = (lo + hi) / 2; - uint64_t thisoff = _blocks_array[mid].offset; - if (thisoff < offset) { - lo = mid + 1; - } else if (thisoff > offset) { - hi = mid; - } else { - return mid; - } - } } -// To support 0-sized blocks, we need to include size as an input to this function. +// To support 0-sized blocks, we need to include size as an input to this +// function. // All 0-sized blocks at the same offset can be considered identical, but // a 0-sized block can share offset with a non-zero sized block. -// The non-zero sized block is not exchangable with a zero sized block (or vice versa), -// so inserting 0-sized blocks can cause corruption here. -void block_allocator::free_block(uint64_t offset) { +// The non-zero sized block is not exchangable with a zero sized block (or vice +// versa), so inserting 0-sized blocks can cause corruption here. +void BlockAllocator::FreeBlock(uint64_t offset, uint64_t size) { VALIDATE(); - int64_t bn = find_block(offset); - assert(bn >= 0); // we require that there is a block with that offset. - _n_bytes_in_use -= _blocks_array[bn].size; - memmove(&_blocks_array[bn], &_blocks_array[bn + 1], - (_n_blocks - bn - 1) * sizeof(struct blockpair)); + _n_bytes_in_use -= size; + _tree->Insert({offset, size}); _n_blocks--; VALIDATE(); - - _trace_free(offset); -} - -uint64_t block_allocator::block_size(uint64_t offset) { - int64_t bn = find_block(offset); - assert(bn >=0); // we require that there is a block with that offset. - return _blocks_array[bn].size; } -uint64_t block_allocator::allocated_limit() const { - if (_n_blocks == 0) { - return _reserve_at_beginning; - } else { - struct blockpair *last = &_blocks_array[_n_blocks - 1]; - return last->offset + last->size; - } +uint64_t BlockAllocator::AllocatedLimit() const { + MhsRbTree::Node *max_node = _tree->MaxNode(); + return rbn_offset(max_node).ToInt(); } -// Effect: Consider the blocks in sorted order. The reserved block at the beginning is number 0. The next one is number 1 and so forth. +// Effect: Consider the blocks in sorted order. The reserved block at the +// beginning is number 0. The next one is number 1 and so forth. // Return the offset and size of the block with that number. // Return 0 if there is a block that big, return nonzero if b is too big. -int block_allocator::get_nth_block_in_layout_order(uint64_t b, uint64_t *offset, uint64_t *size) { - if (b ==0 ) { +int BlockAllocator::NthBlockInLayoutOrder(uint64_t b, + uint64_t *offset, + uint64_t *size) { + MhsRbTree::Node *x, *y; + if (b == 0) { *offset = 0; *size = _reserve_at_beginning; - return 0; + return 0; } else if (b > _n_blocks) { return -1; } else { - *offset =_blocks_array[b - 1].offset; - *size =_blocks_array[b - 1].size; + x = _tree->MinNode(); + for (uint64_t i = 1; i <= b; i++) { + y = x; + x = _tree->Successor(x); + } + *size = (rbn_offset(x) - (rbn_offset(y) + rbn_size(y))).ToInt(); + *offset = (rbn_offset(y) + rbn_size(y)).ToInt(); return 0; } } +struct VisUnusedExtra { + TOKU_DB_FRAGMENTATION _report; + uint64_t _align; +}; + +static void VisUnusedCollector(void *extra, + MhsRbTree::Node *node, + uint64_t UU(depth)) { + struct VisUnusedExtra *v_e = (struct VisUnusedExtra *)extra; + TOKU_DB_FRAGMENTATION report = v_e->_report; + uint64_t alignm = v_e->_align; + + MhsRbTree::OUUInt64 offset = rbn_offset(node); + MhsRbTree::OUUInt64 size = rbn_size(node); + MhsRbTree::OUUInt64 answer_offset(Align(offset.ToInt(), alignm)); + uint64_t free_space = (offset + size - answer_offset).ToInt(); + if (free_space > 0) { + report->unused_bytes += free_space; + report->unused_blocks++; + if (free_space > report->largest_unused_block) { + report->largest_unused_block = free_space; + } + } +} // Requires: report->file_size_bytes is filled in // Requires: report->data_bytes is filled in // Requires: report->checkpoint_bytes_additional is filled in -void block_allocator::get_unused_statistics(TOKU_DB_FRAGMENTATION report) { - assert(_n_bytes_in_use == report->data_bytes + report->checkpoint_bytes_additional); +void BlockAllocator::UnusedStatistics(TOKU_DB_FRAGMENTATION report) { + invariant(_n_bytes_in_use == + report->data_bytes + report->checkpoint_bytes_additional); report->unused_bytes = 0; report->unused_blocks = 0; report->largest_unused_block = 0; - if (_n_blocks > 0) { - //Deal with space before block 0 and after reserve: - { - struct blockpair *bp = &_blocks_array[0]; - assert(bp->offset >= align(_reserve_at_beginning, _alignment)); - uint64_t free_space = bp->offset - align(_reserve_at_beginning, _alignment); - if (free_space > 0) { - report->unused_bytes += free_space; - report->unused_blocks++; - if (free_space > report->largest_unused_block) { - report->largest_unused_block = free_space; - } - } - } - - //Deal with space between blocks: - for (uint64_t blocknum = 0; blocknum +1 < _n_blocks; blocknum ++) { - // Consider the space after blocknum - struct blockpair *bp = &_blocks_array[blocknum]; - uint64_t this_offset = bp[0].offset; - uint64_t this_size = bp[0].size; - uint64_t end_of_this_block = align(this_offset+this_size, _alignment); - uint64_t next_offset = bp[1].offset; - uint64_t free_space = next_offset - end_of_this_block; - if (free_space > 0) { - report->unused_bytes += free_space; - report->unused_blocks++; - if (free_space > report->largest_unused_block) { - report->largest_unused_block = free_space; - } - } - } - - //Deal with space after last block - { - struct blockpair *bp = &_blocks_array[_n_blocks-1]; - uint64_t this_offset = bp[0].offset; - uint64_t this_size = bp[0].size; - uint64_t end_of_this_block = align(this_offset+this_size, _alignment); - if (end_of_this_block < report->file_size_bytes) { - uint64_t free_space = report->file_size_bytes - end_of_this_block; - assert(free_space > 0); - report->unused_bytes += free_space; - report->unused_blocks++; - if (free_space > report->largest_unused_block) { - report->largest_unused_block = free_space; - } - } - } - } else { - // No blocks. Just the reserve. - uint64_t end_of_this_block = align(_reserve_at_beginning, _alignment); - if (end_of_this_block < report->file_size_bytes) { - uint64_t free_space = report->file_size_bytes - end_of_this_block; - assert(free_space > 0); - report->unused_bytes += free_space; - report->unused_blocks++; - if (free_space > report->largest_unused_block) { - report->largest_unused_block = free_space; - } - } - } + struct VisUnusedExtra extra = {report, _alignment}; + _tree->InOrderVisitor(VisUnusedCollector, &extra); } -void block_allocator::get_statistics(TOKU_DB_FRAGMENTATION report) { - report->data_bytes = _n_bytes_in_use; - report->data_blocks = _n_blocks; +void BlockAllocator::Statistics(TOKU_DB_FRAGMENTATION report) { + report->data_bytes = _n_bytes_in_use; + report->data_blocks = _n_blocks; report->file_size_bytes = 0; report->checkpoint_bytes_additional = 0; - get_unused_statistics(report); + UnusedStatistics(report); } -void block_allocator::validate() const { - uint64_t n_bytes_in_use = _reserve_at_beginning; - for (uint64_t i = 0; i < _n_blocks; i++) { - n_bytes_in_use += _blocks_array[i].size; - if (i > 0) { - assert(_blocks_array[i].offset > _blocks_array[i - 1].offset); - assert(_blocks_array[i].offset >= _blocks_array[i - 1].offset + _blocks_array[i - 1].size ); - } - } - assert(n_bytes_in_use == _n_bytes_in_use); -} - -// Tracing - -void block_allocator::_trace_create(void) { - if (ba_trace_file != nullptr) { - toku_mutex_lock(&_trace_lock); - fprintf(ba_trace_file, "ba_trace_create %p %" PRIu64 " %" PRIu64 "\n", - this, _reserve_at_beginning, _alignment); - toku_mutex_unlock(&_trace_lock); - - fflush(ba_trace_file); - } -} - -void block_allocator::_trace_create_from_blockpairs(void) { - if (ba_trace_file != nullptr) { - toku_mutex_lock(&_trace_lock); - fprintf(ba_trace_file, "ba_trace_create_from_blockpairs %p %" PRIu64 " %" PRIu64 " ", - this, _reserve_at_beginning, _alignment); - for (uint64_t i = 0; i < _n_blocks; i++) { - fprintf(ba_trace_file, "[%" PRIu64 " %" PRIu64 "] ", - _blocks_array[i].offset, _blocks_array[i].size); - } - fprintf(ba_trace_file, "\n"); - toku_mutex_unlock(&_trace_lock); - - fflush(ba_trace_file); - } -} - -void block_allocator::_trace_destroy(void) { - if (ba_trace_file != nullptr) { - toku_mutex_lock(&_trace_lock); - fprintf(ba_trace_file, "ba_trace_destroy %p\n", this); - toku_mutex_unlock(&_trace_lock); - - fflush(ba_trace_file); - } -} - -void block_allocator::_trace_alloc(uint64_t size, uint64_t heat, uint64_t offset) { - if (ba_trace_file != nullptr) { - toku_mutex_lock(&_trace_lock); - fprintf(ba_trace_file, "ba_trace_alloc %p %" PRIu64 " %" PRIu64 " %" PRIu64 "\n", - this, size, heat, offset); - toku_mutex_unlock(&_trace_lock); - - fflush(ba_trace_file); +struct ValidateExtra { + uint64_t _bytes; + MhsRbTree::Node *_pre_node; +}; +static void VisUsedBlocksInOrder(void *extra, + MhsRbTree::Node *cur_node, + uint64_t UU(depth)) { + struct ValidateExtra *v_e = (struct ValidateExtra *)extra; + MhsRbTree::Node *pre_node = v_e->_pre_node; + // verify no overlaps + if (pre_node) { + invariant(rbn_size(pre_node) > 0); + invariant(rbn_offset(cur_node) > + rbn_offset(pre_node) + rbn_size(pre_node)); + MhsRbTree::OUUInt64 used_space = + rbn_offset(cur_node) - (rbn_offset(pre_node) + rbn_size(pre_node)); + v_e->_bytes += used_space.ToInt(); + } else { + v_e->_bytes += rbn_offset(cur_node).ToInt(); } + v_e->_pre_node = cur_node; } -void block_allocator::_trace_free(uint64_t offset) { - if (ba_trace_file != nullptr) { - toku_mutex_lock(&_trace_lock); - fprintf(ba_trace_file, "ba_trace_free %p %" PRIu64 "\n", this, offset); - toku_mutex_unlock(&_trace_lock); - - fflush(ba_trace_file); - } +void BlockAllocator::Validate() const { + _tree->ValidateBalance(); + _tree->ValidateMhs(); + struct ValidateExtra extra = {0, nullptr}; + _tree->InOrderVisitor(VisUsedBlocksInOrder, &extra); + invariant(extra._bytes == _n_bytes_in_use); } diff --git a/storage/tokudb/PerconaFT/ft/serialize/block_allocator.h b/storage/tokudb/PerconaFT/ft/serialize/block_allocator.h index 9b2c1553e7f93..648ea9a9ef2c8 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/block_allocator.h +++ b/storage/tokudb/PerconaFT/ft/serialize/block_allocator.h @@ -43,6 +43,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "portability/toku_pthread.h" #include "portability/toku_stdint.h" #include "portability/toku_stdlib.h" +#include "ft/serialize/rbtree_mhs.h" // Block allocator. // @@ -51,151 +52,128 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. // The allocation of block numbers is handled elsewhere. // // When creating a block allocator we also specify a certain-sized -// block at the beginning that is preallocated (and cannot be allocated or freed) +// block at the beginning that is preallocated (and cannot be allocated or +// freed) // // We can allocate blocks of a particular size at a particular location. -// We can allocate blocks of a particular size at a location chosen by the allocator. // We can free blocks. // We can determine the size of a block. - -class block_allocator { -public: +#define MAX_BYTE 0xffffffffffffffff +class BlockAllocator { + public: static const size_t BLOCK_ALLOCATOR_ALIGNMENT = 4096; // How much must be reserved at the beginning for the block? - // The actual header is 8+4+4+8+8_4+8+ the length of the db names + 1 pointer for each root. + // The actual header is 8+4+4+8+8_4+8+ the length of the db names + 1 + // pointer for each root. // So 4096 should be enough. static const size_t BLOCK_ALLOCATOR_HEADER_RESERVE = 4096; - - static_assert(BLOCK_ALLOCATOR_HEADER_RESERVE % BLOCK_ALLOCATOR_ALIGNMENT == 0, + + static_assert(BLOCK_ALLOCATOR_HEADER_RESERVE % BLOCK_ALLOCATOR_ALIGNMENT == + 0, "block allocator header must have proper alignment"); - static const size_t BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE = BLOCK_ALLOCATOR_HEADER_RESERVE * 2; + static const size_t BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE = + BLOCK_ALLOCATOR_HEADER_RESERVE * 2; - enum allocation_strategy { - BA_STRATEGY_FIRST_FIT = 1, - BA_STRATEGY_BEST_FIT, - BA_STRATEGY_PADDED_FIT, - BA_STRATEGY_HEAT_ZONE - }; - - struct blockpair { - uint64_t offset; - uint64_t size; - blockpair(uint64_t o, uint64_t s) : - offset(o), size(s) { - } - int operator<(const struct blockpair &rhs) const { - return offset < rhs.offset; - } - int operator<(const uint64_t &o) const { - return offset < o; + struct BlockPair { + uint64_t _offset; + uint64_t _size; + BlockPair(uint64_t o, uint64_t s) : _offset(o), _size(s) {} + int operator<(const struct BlockPair &rhs) const { + return _offset < rhs._offset; } + int operator<(const uint64_t &o) const { return _offset < o; } }; - // Effect: Create a block allocator, in which the first RESERVE_AT_BEGINNING bytes are not put into a block. - // The default allocation strategy is first fit (BA_STRATEGY_FIRST_FIT) + // Effect: Create a block allocator, in which the first RESERVE_AT_BEGINNING + // bytes are not put into a block. + // The default allocation strategy is first fit + // (BA_STRATEGY_FIRST_FIT) // All blocks be start on a multiple of ALIGNMENT. // Aborts if we run out of memory. // Parameters - // reserve_at_beginning (IN) Size of reserved block at beginning. This size does not have to be aligned. + // reserve_at_beginning (IN) Size of reserved block at beginning. + // This size does not have to be aligned. // alignment (IN) Block alignment. - void create(uint64_t reserve_at_beginning, uint64_t alignment); + void Create(uint64_t reserve_at_beginning, uint64_t alignment); - // Effect: Create a block allocator, in which the first RESERVE_AT_BEGINNING bytes are not put into a block. - // The default allocation strategy is first fit (BA_STRATEGY_FIRST_FIT) - // The allocator is initialized to contain `n_blocks' of blockpairs, taken from `pairs' + // Effect: Create a block allocator, in which the first RESERVE_AT_BEGINNING + // bytes are not put into a block. + // The allocator is initialized to contain `n_blocks' of BlockPairs, + // taken from `pairs' // All blocks be start on a multiple of ALIGNMENT. // Aborts if we run out of memory. // Parameters // pairs, unowned array of pairs to copy // n_blocks, Size of pairs array - // reserve_at_beginning (IN) Size of reserved block at beginning. This size does not have to be aligned. + // reserve_at_beginning (IN) Size of reserved block at beginning. + // This size does not have to be aligned. // alignment (IN) Block alignment. - void create_from_blockpairs(uint64_t reserve_at_beginning, uint64_t alignment, - struct blockpair *pairs, uint64_t n_blocks); + void CreateFromBlockPairs(uint64_t reserve_at_beginning, + uint64_t alignment, + struct BlockPair *pairs, + uint64_t n_blocks); // Effect: Destroy this block allocator - void destroy(); - - // Effect: Set the allocation strategy that the allocator should use - // Requires: No other threads are operating on this block allocator - void set_strategy(enum allocation_strategy strategy); + void Destroy(); - // Effect: Allocate a block of the specified size at an address chosen by the allocator. + // Effect: Allocate a block of the specified size at an address chosen by + // the allocator. // Aborts if anything goes wrong. // The block address will be a multiple of the alignment. // Parameters: - // size (IN): The size of the block. (The size does not have to be aligned.) + // size (IN): The size of the block. (The size does not have to be + // aligned.) // offset (OUT): The location of the block. - // heat (IN): A higher heat means we should be prepared to free this block soon (perhaps in the next checkpoint) - // Heat values are lexiographically ordered (like integers), but their specific values are arbitrary - void alloc_block(uint64_t size, uint64_t heat, uint64_t *offset); + // block soon (perhaps in the next checkpoint) + // Heat values are lexiographically ordered (like integers), + // but their specific values are arbitrary + void AllocBlock(uint64_t size, uint64_t *offset); // Effect: Free the block at offset. // Requires: There must be a block currently allocated at that offset. // Parameters: // offset (IN): The offset of the block. - void free_block(uint64_t offset); + void FreeBlock(uint64_t offset, uint64_t size); - // Effect: Return the size of the block that starts at offset. - // Requires: There must be a block currently allocated at that offset. - // Parameters: - // offset (IN): The offset of the block. - uint64_t block_size(uint64_t offset); - - // Effect: Check to see if the block allocator is OK. This may take a long time. + // Effect: Check to see if the block allocator is OK. This may take a long + // time. // Usage Hints: Probably only use this for unit tests. // TODO: Private? - void validate() const; + void Validate() const; // Effect: Return the unallocated block address of "infinite" size. - // That is, return the smallest address that is above all the allocated blocks. - uint64_t allocated_limit() const; + // That is, return the smallest address that is above all the allocated + // blocks. + uint64_t AllocatedLimit() const; - // Effect: Consider the blocks in sorted order. The reserved block at the beginning is number 0. The next one is number 1 and so forth. + // Effect: Consider the blocks in sorted order. The reserved block at the + // beginning is number 0. The next one is number 1 and so forth. // Return the offset and size of the block with that number. // Return 0 if there is a block that big, return nonzero if b is too big. // Rationale: This is probably useful only for tests. - int get_nth_block_in_layout_order(uint64_t b, uint64_t *offset, uint64_t *size); + int NthBlockInLayoutOrder(uint64_t b, uint64_t *offset, uint64_t *size); // Effect: Fill in report to indicate how the file is used. - // Requires: + // Requires: // report->file_size_bytes is filled in // report->data_bytes is filled in // report->checkpoint_bytes_additional is filled in - void get_unused_statistics(TOKU_DB_FRAGMENTATION report); + void UnusedStatistics(TOKU_DB_FRAGMENTATION report); // Effect: Fill in report->data_bytes with the number of bytes in use - // Fill in report->data_blocks with the number of blockpairs in use + // Fill in report->data_blocks with the number of BlockPairs in use // Fill in unused statistics using this->get_unused_statistics() // Requires: // report->file_size is ignored on return // report->checkpoint_bytes_additional is ignored on return - void get_statistics(TOKU_DB_FRAGMENTATION report); - - // Block allocator tracing. - // - Enabled by setting TOKU_BA_TRACE_PATH to the file that the trace file - // should be written to. - // - Trace may be replayed by ba_trace_replay tool in tools/ directory - // eg: "cat mytracefile | ba_trace_replay" - static void maybe_initialize_trace(); - static void maybe_close_trace(); - -private: - void _create_internal(uint64_t reserve_at_beginning, uint64_t alignment); - void grow_blocks_array_by(uint64_t n_to_add); - void grow_blocks_array(); - int64_t find_block(uint64_t offset); - struct blockpair *choose_block_to_alloc_after(size_t size, uint64_t heat); - - // Tracing - toku_mutex_t _trace_lock; - void _trace_create(void); - void _trace_create_from_blockpairs(void); - void _trace_destroy(void); - void _trace_alloc(uint64_t size, uint64_t heat, uint64_t offset); - void _trace_free(uint64_t offset); + void Statistics(TOKU_DB_FRAGMENTATION report); + + virtual ~BlockAllocator(){}; + + private: + void CreateInternal(uint64_t reserve_at_beginning, uint64_t alignment); // How much to reserve at the beginning uint64_t _reserve_at_beginning; @@ -203,12 +181,8 @@ class block_allocator { uint64_t _alignment; // How many blocks uint64_t _n_blocks; - // How big is the blocks_array. Must be >= n_blocks. - uint64_t _blocks_array_size; - // These blocks are sorted by address. - struct blockpair *_blocks_array; - // Including the reserve_at_beginning uint64_t _n_bytes_in_use; - // The allocation strategy are we using - enum allocation_strategy _strategy; + + // These blocks are sorted by address. + MhsRbTree::Tree *_tree; }; diff --git a/storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.cc b/storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.cc deleted file mode 100644 index 62bb8fc4a87bc..0000000000000 --- a/storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.cc +++ /dev/null @@ -1,224 +0,0 @@ -/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ -// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: -#ident "$Id$" -/*====== -This file is part of PerconaFT. - - -Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - - PerconaFT is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License, version 2, - as published by the Free Software Foundation. - - PerconaFT is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with PerconaFT. If not, see . - ----------------------------------------- - - PerconaFT is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License, version 3, - as published by the Free Software Foundation. - - PerconaFT is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with PerconaFT. If not, see . -======= */ - -#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." - -#include - -#include - -#include "portability/toku_assert.h" - -#include "ft/serialize/block_allocator_strategy.h" - -static uint64_t _align(uint64_t value, uint64_t ba_alignment) { - return ((value + ba_alignment - 1) / ba_alignment) * ba_alignment; -} - -static uint64_t _roundup_to_power_of_two(uint64_t value) { - uint64_t r = 4096; - while (r < value) { - r *= 2; - invariant(r > 0); - } - return r; -} - -// First fit block allocation -static struct block_allocator::blockpair * -_first_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment, - uint64_t max_padding) { - if (n_blocks == 1) { - // won't enter loop, can't underflow the direction < 0 case - return nullptr; - } - - struct block_allocator::blockpair *bp = &blocks_array[0]; - for (uint64_t n_spaces_to_check = n_blocks - 1; n_spaces_to_check > 0; - n_spaces_to_check--, bp++) { - // Consider the space after bp - uint64_t padded_alignment = max_padding != 0 ? _align(max_padding, alignment) : alignment; - uint64_t possible_offset = _align(bp->offset + bp->size, padded_alignment); - if (possible_offset + size <= bp[1].offset) { // bp[1] is always valid since bp < &blocks_array[n_blocks-1] - invariant(bp - blocks_array < (int64_t) n_blocks); - return bp; - } - } - return nullptr; -} - -static struct block_allocator::blockpair * -_first_fit_bw(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment, - uint64_t max_padding, struct block_allocator::blockpair *blocks_array_limit) { - if (n_blocks == 1) { - // won't enter loop, can't underflow the direction < 0 case - return nullptr; - } - - struct block_allocator::blockpair *bp = &blocks_array[-1]; - for (uint64_t n_spaces_to_check = n_blocks - 1; n_spaces_to_check > 0; - n_spaces_to_check--, bp--) { - // Consider the space after bp - uint64_t padded_alignment = max_padding != 0 ? _align(max_padding, alignment) : alignment; - uint64_t possible_offset = _align(bp->offset + bp->size, padded_alignment); - if (&bp[1] < blocks_array_limit && possible_offset + size <= bp[1].offset) { - invariant(blocks_array - bp < (int64_t) n_blocks); - return bp; - } - } - return nullptr; -} - -struct block_allocator::blockpair * -block_allocator_strategy::first_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment) { - return _first_fit(blocks_array, n_blocks, size, alignment, 0); -} - -// Best fit block allocation -struct block_allocator::blockpair * -block_allocator_strategy::best_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment) { - struct block_allocator::blockpair *best_bp = nullptr; - uint64_t best_hole_size = 0; - for (uint64_t blocknum = 0; blocknum + 1 < n_blocks; blocknum++) { - // Consider the space after blocknum - struct block_allocator::blockpair *bp = &blocks_array[blocknum]; - uint64_t possible_offset = _align(bp->offset + bp->size, alignment); - uint64_t possible_end_offset = possible_offset + size; - if (possible_end_offset <= bp[1].offset) { - // It fits here. Is it the best fit? - uint64_t hole_size = bp[1].offset - possible_end_offset; - if (best_bp == nullptr || hole_size < best_hole_size) { - best_hole_size = hole_size; - best_bp = bp; - } - } - } - return best_bp; -} - -static uint64_t padded_fit_alignment = 4096; - -// TODO: These compiler specific directives should be abstracted in a portability header -// portability/toku_compiler.h? -__attribute__((__constructor__)) -static void determine_padded_fit_alignment_from_env(void) { - // TODO: Should be in portability as 'toku_os_getenv()?' - const char *s = getenv("TOKU_BA_PADDED_FIT_ALIGNMENT"); - if (s != nullptr && strlen(s) > 0) { - const int64_t alignment = strtoll(s, nullptr, 10); - if (alignment <= 0) { - fprintf(stderr, "tokuft: error: block allocator padded fit alignment found in environment (%s), " - "but it's out of range (should be an integer > 0). defaulting to %" PRIu64 "\n", - s, padded_fit_alignment); - } else { - padded_fit_alignment = _roundup_to_power_of_two(alignment); - fprintf(stderr, "tokuft: setting block allocator padded fit alignment to %" PRIu64 "\n", - padded_fit_alignment); - } - } -} - -// First fit into a block that is oversized by up to max_padding. -// The hope is that if we purposefully waste a bit of space at allocation -// time we'll be more likely to reuse this block later. -struct block_allocator::blockpair * -block_allocator_strategy::padded_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment) { - return _first_fit(blocks_array, n_blocks, size, alignment, padded_fit_alignment); -} - -static double hot_zone_threshold = 0.85; - -// TODO: These compiler specific directives should be abstracted in a portability header -// portability/toku_compiler.h? -__attribute__((__constructor__)) -static void determine_hot_zone_threshold_from_env(void) { - // TODO: Should be in portability as 'toku_os_getenv()?' - const char *s = getenv("TOKU_BA_HOT_ZONE_THRESHOLD"); - if (s != nullptr && strlen(s) > 0) { - const double hot_zone = strtod(s, nullptr); - if (hot_zone < 1 || hot_zone > 99) { - fprintf(stderr, "tokuft: error: block allocator hot zone threshold found in environment (%s), " - "but it's out of range (should be an integer 1 through 99). defaulting to 85\n", s); - hot_zone_threshold = 85 / 100; - } else { - fprintf(stderr, "tokuft: setting block allocator hot zone threshold to %s\n", s); - hot_zone_threshold = hot_zone / 100; - } - } -} - -struct block_allocator::blockpair * -block_allocator_strategy::heat_zone(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment, - uint64_t heat) { - if (heat > 0) { - struct block_allocator::blockpair *bp, *boundary_bp; - - // Hot allocation. Find the beginning of the hot zone. - boundary_bp = &blocks_array[n_blocks - 1]; - uint64_t highest_offset = _align(boundary_bp->offset + boundary_bp->size, alignment); - uint64_t hot_zone_offset = static_cast(hot_zone_threshold * highest_offset); - - boundary_bp = std::lower_bound(blocks_array, blocks_array + n_blocks, hot_zone_offset); - uint64_t blocks_in_zone = (blocks_array + n_blocks) - boundary_bp; - uint64_t blocks_outside_zone = boundary_bp - blocks_array; - invariant(blocks_in_zone + blocks_outside_zone == n_blocks); - - if (blocks_in_zone > 0) { - // Find the first fit in the hot zone, going forward. - bp = _first_fit(boundary_bp, blocks_in_zone, size, alignment, 0); - if (bp != nullptr) { - return bp; - } - } - if (blocks_outside_zone > 0) { - // Find the first fit in the cold zone, going backwards. - bp = _first_fit_bw(boundary_bp, blocks_outside_zone, size, alignment, 0, &blocks_array[n_blocks]); - if (bp != nullptr) { - return bp; - } - } - } else { - // Cold allocations are simply first-fit from the beginning. - return _first_fit(blocks_array, n_blocks, size, alignment, 0); - } - return nullptr; -} diff --git a/storage/tokudb/PerconaFT/ft/serialize/block_table.cc b/storage/tokudb/PerconaFT/ft/serialize/block_table.cc index 7101ba9f58c17..d2532134d96cf 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/block_table.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/block_table.cc @@ -46,31 +46,27 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "ft/ft-internal.h" // TODO: reorganize this dependency (FT-303) -#include "ft/ft-ops.h" // for toku_maybe_truncate_file +#include "ft/ft-ops.h" // for toku_maybe_truncate_file #include "ft/serialize/block_table.h" #include "ft/serialize/rbuf.h" #include "ft/serialize/wbuf.h" #include "ft/serialize/block_allocator.h" - #include "util/nb_mutex.h" #include "util/scoped_malloc.h" // indicates the end of a freelist -static const BLOCKNUM freelist_null = { -1 }; +static const BLOCKNUM freelist_null = {-1}; // value of block_translation_pair.size if blocknum is unused -static const DISKOFF size_is_free = (DISKOFF) -1; +static const DISKOFF size_is_free = (DISKOFF)-1; -// value of block_translation_pair.u.diskoff if blocknum is used but does not yet have a diskblock -static const DISKOFF diskoff_unused = (DISKOFF) -2; +// value of block_translation_pair.u.diskoff if blocknum is used but does not +// yet have a diskblock +static const DISKOFF diskoff_unused = (DISKOFF)-2; -void block_table::_mutex_lock() { - toku_mutex_lock(&_mutex); -} +void block_table::_mutex_lock() { toku_mutex_lock(&_mutex); } -void block_table::_mutex_unlock() { - toku_mutex_unlock(&_mutex); -} +void block_table::_mutex_unlock() { toku_mutex_unlock(&_mutex); } // TODO: Move lock to FT void toku_ft_lock(FT ft) { @@ -85,13 +81,16 @@ void toku_ft_unlock(FT ft) { bt->_mutex_unlock(); } -// There are two headers: the reserve must fit them both and be suitably aligned. -static_assert(block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE % - block_allocator::BLOCK_ALLOCATOR_ALIGNMENT == 0, +// There are two headers: the reserve must fit them both and be suitably +// aligned. +static_assert(BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE % + BlockAllocator::BLOCK_ALLOCATOR_ALIGNMENT == + 0, "Block allocator's header reserve must be suitibly aligned"); -static_assert(block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE * 2 == - block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, - "Block allocator's total header reserve must exactly fit two headers"); +static_assert( + BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE * 2 == + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, + "Block allocator's total header reserve must exactly fit two headers"); // does NOT initialize the block allocator: the caller is responsible void block_table::_create_internal() { @@ -99,25 +98,30 @@ void block_table::_create_internal() { memset(&_inprogress, 0, sizeof(struct translation)); memset(&_checkpointed, 0, sizeof(struct translation)); memset(&_mutex, 0, sizeof(_mutex)); + _bt_block_allocator = new BlockAllocator(); toku_mutex_init(&_mutex, nullptr); nb_mutex_init(&_safe_file_size_lock); } -// Fill in the checkpointed translation from buffer, and copy checkpointed to current. -// The one read from disk is the last known checkpointed one, so we are keeping it in -// place and then setting current (which is never stored on disk) for current use. -// The translation_buffer has translation only, we create the rest of the block_table. -int block_table::create_from_buffer(int fd, - DISKOFF location_on_disk, //Location of translation_buffer - DISKOFF size_on_disk, - unsigned char *translation_buffer) { +// Fill in the checkpointed translation from buffer, and copy checkpointed to +// current. +// The one read from disk is the last known checkpointed one, so we are keeping +// it in +// place and then setting current (which is never stored on disk) for current +// use. +// The translation_buffer has translation only, we create the rest of the +// block_table. +int block_table::create_from_buffer( + int fd, + DISKOFF location_on_disk, // Location of translation_buffer + DISKOFF size_on_disk, + unsigned char *translation_buffer) { // Does not initialize the block allocator _create_internal(); // Deserialize the translation and copy it to current - int r = _translation_deserialize_from_buffer(&_checkpointed, - location_on_disk, size_on_disk, - translation_buffer); + int r = _translation_deserialize_from_buffer( + &_checkpointed, location_on_disk, size_on_disk, translation_buffer); if (r != 0) { return r; } @@ -130,22 +134,26 @@ int block_table::create_from_buffer(int fd, invariant(file_size >= 0); _safe_file_size = file_size; - // Gather the non-empty translations and use them to create the block allocator + // Gather the non-empty translations and use them to create the block + // allocator toku::scoped_malloc pairs_buf(_checkpointed.smallest_never_used_blocknum.b * - sizeof(struct block_allocator::blockpair)); - struct block_allocator::blockpair *CAST_FROM_VOIDP(pairs, pairs_buf.get()); + sizeof(struct BlockAllocator::BlockPair)); + struct BlockAllocator::BlockPair *CAST_FROM_VOIDP(pairs, pairs_buf.get()); uint64_t n_pairs = 0; for (int64_t i = 0; i < _checkpointed.smallest_never_used_blocknum.b; i++) { struct block_translation_pair pair = _checkpointed.block_translation[i]; if (pair.size > 0) { invariant(pair.u.diskoff != diskoff_unused); - pairs[n_pairs++] = block_allocator::blockpair(pair.u.diskoff, pair.size); + pairs[n_pairs++] = + BlockAllocator::BlockPair(pair.u.diskoff, pair.size); } } - _bt_block_allocator.create_from_blockpairs(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, - block_allocator::BLOCK_ALLOCATOR_ALIGNMENT, - pairs, n_pairs); + _bt_block_allocator->CreateFromBlockPairs( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, + BlockAllocator::BLOCK_ALLOCATOR_ALIGNMENT, + pairs, + n_pairs); return 0; } @@ -155,8 +163,10 @@ void block_table::create() { _create_internal(); _checkpointed.type = TRANSLATION_CHECKPOINTED; - _checkpointed.smallest_never_used_blocknum = make_blocknum(RESERVED_BLOCKNUMS); - _checkpointed.length_of_array = _checkpointed.smallest_never_used_blocknum.b; + _checkpointed.smallest_never_used_blocknum = + make_blocknum(RESERVED_BLOCKNUMS); + _checkpointed.length_of_array = + _checkpointed.smallest_never_used_blocknum.b; _checkpointed.blocknum_freelist_head = freelist_null; XMALLOC_N(_checkpointed.length_of_array, _checkpointed.block_translation); for (int64_t i = 0; i < _checkpointed.length_of_array; i++) { @@ -164,12 +174,13 @@ void block_table::create() { _checkpointed.block_translation[i].u.diskoff = diskoff_unused; } - // we just created a default checkpointed, now copy it to current. + // we just created a default checkpointed, now copy it to current. _copy_translation(&_current, &_checkpointed, TRANSLATION_CURRENT); // Create an empty block allocator. - _bt_block_allocator.create(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, - block_allocator::BLOCK_ALLOCATOR_ALIGNMENT); + _bt_block_allocator->Create( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, + BlockAllocator::BLOCK_ALLOCATOR_ALIGNMENT); } // TODO: Refactor with FT-303 @@ -185,20 +196,24 @@ static void ft_set_dirty(FT ft, bool for_checkpoint) { void block_table::_maybe_truncate_file(int fd, uint64_t size_needed_before) { toku_mutex_assert_locked(&_mutex); - uint64_t new_size_needed = _bt_block_allocator.allocated_limit(); - //Save a call to toku_os_get_file_size (kernel call) if unlikely to be useful. - if (new_size_needed < size_needed_before && new_size_needed < _safe_file_size) { + uint64_t new_size_needed = _bt_block_allocator->AllocatedLimit(); + // Save a call to toku_os_get_file_size (kernel call) if unlikely to be + // useful. + if (new_size_needed < size_needed_before && + new_size_needed < _safe_file_size) { nb_mutex_lock(&_safe_file_size_lock, &_mutex); // Must hold _safe_file_size_lock to change _safe_file_size. if (new_size_needed < _safe_file_size) { int64_t safe_file_size_before = _safe_file_size; - // Not safe to use the 'to-be-truncated' portion until truncate is done. + // Not safe to use the 'to-be-truncated' portion until truncate is + // done. _safe_file_size = new_size_needed; _mutex_unlock(); uint64_t size_after; - toku_maybe_truncate_file(fd, new_size_needed, safe_file_size_before, &size_after); + toku_maybe_truncate_file( + fd, new_size_needed, safe_file_size_before, &size_after); _mutex_lock(); _safe_file_size = size_after; @@ -213,26 +228,35 @@ void block_table::maybe_truncate_file_on_open(int fd) { _mutex_unlock(); } -void block_table::_copy_translation(struct translation *dst, struct translation *src, enum translation_type newtype) { - // We intend to malloc a fresh block, so the incoming translation should be empty +void block_table::_copy_translation(struct translation *dst, + struct translation *src, + enum translation_type newtype) { + // We intend to malloc a fresh block, so the incoming translation should be + // empty invariant_null(dst->block_translation); invariant(src->length_of_array >= src->smallest_never_used_blocknum.b); invariant(newtype == TRANSLATION_DEBUG || - (src->type == TRANSLATION_CURRENT && newtype == TRANSLATION_INPROGRESS) || - (src->type == TRANSLATION_CHECKPOINTED && newtype == TRANSLATION_CURRENT)); + (src->type == TRANSLATION_CURRENT && + newtype == TRANSLATION_INPROGRESS) || + (src->type == TRANSLATION_CHECKPOINTED && + newtype == TRANSLATION_CURRENT)); dst->type = newtype; dst->smallest_never_used_blocknum = src->smallest_never_used_blocknum; - dst->blocknum_freelist_head = src->blocknum_freelist_head; + dst->blocknum_freelist_head = src->blocknum_freelist_head; - // destination btt is of fixed size. Allocate + memcpy the exact length necessary. + // destination btt is of fixed size. Allocate + memcpy the exact length + // necessary. dst->length_of_array = dst->smallest_never_used_blocknum.b; XMALLOC_N(dst->length_of_array, dst->block_translation); - memcpy(dst->block_translation, src->block_translation, dst->length_of_array * sizeof(*dst->block_translation)); + memcpy(dst->block_translation, + src->block_translation, + dst->length_of_array * sizeof(*dst->block_translation)); // New version of btt is not yet stored on disk. dst->block_translation[RESERVED_BLOCKNUM_TRANSLATION].size = 0; - dst->block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff = diskoff_unused; + dst->block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff = + diskoff_unused; } int64_t block_table::get_blocks_in_use_unlocked() { @@ -240,8 +264,9 @@ int64_t block_table::get_blocks_in_use_unlocked() { struct translation *t = &_current; int64_t num_blocks = 0; { - //Reserved blocknums do not get upgraded; They are part of the header. - for (b.b = RESERVED_BLOCKNUMS; b.b < t->smallest_never_used_blocknum.b; b.b++) { + // Reserved blocknums do not get upgraded; They are part of the header. + for (b.b = RESERVED_BLOCKNUMS; b.b < t->smallest_never_used_blocknum.b; + b.b++) { if (t->block_translation[b.b].size != size_is_free) { num_blocks++; } @@ -251,38 +276,43 @@ int64_t block_table::get_blocks_in_use_unlocked() { } void block_table::_maybe_optimize_translation(struct translation *t) { - //Reduce 'smallest_never_used_blocknum.b' (completely free blocknums instead of just - //on a free list. Doing so requires us to regenerate the free list. - //This is O(n) work, so do it only if you're already doing that. + // Reduce 'smallest_never_used_blocknum.b' (completely free blocknums + // instead of just + // on a free list. Doing so requires us to regenerate the free list. + // This is O(n) work, so do it only if you're already doing that. BLOCKNUM b; paranoid_invariant(t->smallest_never_used_blocknum.b >= RESERVED_BLOCKNUMS); - //Calculate how large the free suffix is. + // Calculate how large the free suffix is. int64_t freed; { - for (b.b = t->smallest_never_used_blocknum.b; b.b > RESERVED_BLOCKNUMS; b.b--) { - if (t->block_translation[b.b-1].size != size_is_free) { + for (b.b = t->smallest_never_used_blocknum.b; b.b > RESERVED_BLOCKNUMS; + b.b--) { + if (t->block_translation[b.b - 1].size != size_is_free) { break; } } freed = t->smallest_never_used_blocknum.b - b.b; } - if (freed>0) { + if (freed > 0) { t->smallest_never_used_blocknum.b = b.b; - if (t->length_of_array/4 > t->smallest_never_used_blocknum.b) { - //We're using more memory than necessary to represent this now. Reduce. + if (t->length_of_array / 4 > t->smallest_never_used_blocknum.b) { + // We're using more memory than necessary to represent this now. + // Reduce. uint64_t new_length = t->smallest_never_used_blocknum.b * 2; XREALLOC_N(new_length, t->block_translation); t->length_of_array = new_length; - //No need to zero anything out. + // No need to zero anything out. } - //Regenerate free list. + // Regenerate free list. t->blocknum_freelist_head.b = freelist_null.b; - for (b.b = RESERVED_BLOCKNUMS; b.b < t->smallest_never_used_blocknum.b; b.b++) { + for (b.b = RESERVED_BLOCKNUMS; b.b < t->smallest_never_used_blocknum.b; + b.b++) { if (t->block_translation[b.b].size == size_is_free) { - t->block_translation[b.b].u.next_free_blocknum = t->blocknum_freelist_head; - t->blocknum_freelist_head = b; + t->block_translation[b.b].u.next_free_blocknum = + t->blocknum_freelist_head; + t->blocknum_freelist_head = b; } } } @@ -303,14 +333,16 @@ void block_table::note_start_checkpoint_unlocked() { } void block_table::note_skipped_checkpoint() { - //Purpose, alert block translation that the checkpoint was skipped, e.x. for a non-dirty header + // Purpose, alert block translation that the checkpoint was skipped, e.x. + // for a non-dirty header _mutex_lock(); paranoid_invariant_notnull(_inprogress.block_translation); _checkpoint_skipped = true; _mutex_unlock(); } -// Purpose: free any disk space used by previous checkpoint that isn't in use by either +// Purpose: free any disk space used by previous checkpoint that isn't in use by +// either // - current state // - in-progress checkpoint // capture inprogress as new checkpointed. @@ -323,7 +355,7 @@ void block_table::note_skipped_checkpoint() { void block_table::note_end_checkpoint(int fd) { // Free unused blocks _mutex_lock(); - uint64_t allocated_limit_at_start = _bt_block_allocator.allocated_limit(); + uint64_t allocated_limit_at_start = _bt_block_allocator->AllocatedLimit(); paranoid_invariant_notnull(_inprogress.block_translation); if (_checkpoint_skipped) { toku_free(_inprogress.block_translation); @@ -331,17 +363,23 @@ void block_table::note_end_checkpoint(int fd) { goto end; } - //Make certain inprogress was allocated space on disk - assert(_inprogress.block_translation[RESERVED_BLOCKNUM_TRANSLATION].size > 0); - assert(_inprogress.block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff > 0); + // Make certain inprogress was allocated space on disk + invariant( + _inprogress.block_translation[RESERVED_BLOCKNUM_TRANSLATION].size > 0); + invariant( + _inprogress.block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff > + 0); { struct translation *t = &_checkpointed; for (int64_t i = 0; i < t->length_of_array; i++) { struct block_translation_pair *pair = &t->block_translation[i]; - if (pair->size > 0 && !_translation_prevents_freeing(&_inprogress, make_blocknum(i), pair)) { - assert(!_translation_prevents_freeing(&_current, make_blocknum(i), pair)); - _bt_block_allocator.free_block(pair->u.diskoff); + if (pair->size > 0 && + !_translation_prevents_freeing( + &_inprogress, make_blocknum(i), pair)) { + invariant(!_translation_prevents_freeing( + &_current, make_blocknum(i), pair)); + _bt_block_allocator->FreeBlock(pair->u.diskoff, pair->size); } } toku_free(_checkpointed.block_translation); @@ -359,53 +397,65 @@ bool block_table::_is_valid_blocknum(struct translation *t, BLOCKNUM b) { return b.b >= 0 && b.b < t->smallest_never_used_blocknum.b; } -void block_table::_verify_valid_blocknum(struct translation *UU(t), BLOCKNUM UU(b)) { +void block_table::_verify_valid_blocknum(struct translation *UU(t), + BLOCKNUM UU(b)) { invariant(_is_valid_blocknum(t, b)); } -bool block_table::_is_valid_freeable_blocknum(struct translation *t, BLOCKNUM b) { +bool block_table::_is_valid_freeable_blocknum(struct translation *t, + BLOCKNUM b) { invariant(t->length_of_array >= t->smallest_never_used_blocknum.b); return b.b >= RESERVED_BLOCKNUMS && b.b < t->smallest_never_used_blocknum.b; } // should be freeable -void block_table::_verify_valid_freeable_blocknum(struct translation *UU(t), BLOCKNUM UU(b)) { +void block_table::_verify_valid_freeable_blocknum(struct translation *UU(t), + BLOCKNUM UU(b)) { invariant(_is_valid_freeable_blocknum(t, b)); } // Also used only in ft-serialize-test. -void block_table::block_free(uint64_t offset) { +void block_table::block_free(uint64_t offset, uint64_t size) { _mutex_lock(); - _bt_block_allocator.free_block(offset); + _bt_block_allocator->FreeBlock(offset, size); _mutex_unlock(); } int64_t block_table::_calculate_size_on_disk(struct translation *t) { - return 8 + // smallest_never_used_blocknum - 8 + // blocknum_freelist_head - t->smallest_never_used_blocknum.b * 16 + // Array - 4; // 4 for checksum + return 8 + // smallest_never_used_blocknum + 8 + // blocknum_freelist_head + t->smallest_never_used_blocknum.b * 16 + // Array + 4; // 4 for checksum } -// We cannot free the disk space allocated to this blocknum if it is still in use by the given translation table. -bool block_table::_translation_prevents_freeing(struct translation *t, BLOCKNUM b, struct block_translation_pair *old_pair) { - return t->block_translation && - b.b < t->smallest_never_used_blocknum.b && +// We cannot free the disk space allocated to this blocknum if it is still in +// use by the given translation table. +bool block_table::_translation_prevents_freeing( + struct translation *t, + BLOCKNUM b, + struct block_translation_pair *old_pair) { + return t->block_translation && b.b < t->smallest_never_used_blocknum.b && old_pair->u.diskoff == t->block_translation[b.b].u.diskoff; } -void block_table::_realloc_on_disk_internal(BLOCKNUM b, DISKOFF size, DISKOFF *offset, FT ft, bool for_checkpoint, uint64_t heat) { +void block_table::_realloc_on_disk_internal(BLOCKNUM b, + DISKOFF size, + DISKOFF *offset, + FT ft, + bool for_checkpoint) { toku_mutex_assert_locked(&_mutex); ft_set_dirty(ft, for_checkpoint); struct translation *t = &_current; struct block_translation_pair old_pair = t->block_translation[b.b]; - //Free the old block if it is not still in use by the checkpoint in progress or the previous checkpoint - bool cannot_free = (bool) - ((!for_checkpoint && _translation_prevents_freeing(&_inprogress, b, &old_pair)) || - _translation_prevents_freeing(&_checkpointed, b, &old_pair)); - if (!cannot_free && old_pair.u.diskoff!=diskoff_unused) { - _bt_block_allocator.free_block(old_pair.u.diskoff); + // Free the old block if it is not still in use by the checkpoint in + // progress or the previous checkpoint + bool cannot_free = + (!for_checkpoint && + _translation_prevents_freeing(&_inprogress, b, &old_pair)) || + _translation_prevents_freeing(&_checkpointed, b, &old_pair); + if (!cannot_free && old_pair.u.diskoff != diskoff_unused) { + _bt_block_allocator->FreeBlock(old_pair.u.diskoff, old_pair.size); } uint64_t allocator_offset = diskoff_unused; @@ -413,19 +463,22 @@ void block_table::_realloc_on_disk_internal(BLOCKNUM b, DISKOFF size, DISKOFF *o if (size > 0) { // Allocate a new block if the size is greater than 0, // if the size is just 0, offset will be set to diskoff_unused - _bt_block_allocator.alloc_block(size, heat, &allocator_offset); + _bt_block_allocator->AllocBlock(size, &allocator_offset); } t->block_translation[b.b].u.diskoff = allocator_offset; *offset = allocator_offset; - //Update inprogress btt if appropriate (if called because Pending bit is set). + // Update inprogress btt if appropriate (if called because Pending bit is + // set). if (for_checkpoint) { paranoid_invariant(b.b < _inprogress.length_of_array); _inprogress.block_translation[b.b] = t->block_translation[b.b]; } } -void block_table::_ensure_safe_write_unlocked(int fd, DISKOFF block_size, DISKOFF block_offset) { +void block_table::_ensure_safe_write_unlocked(int fd, + DISKOFF block_size, + DISKOFF block_offset) { // Requires: holding _mutex uint64_t size_needed = block_size + block_offset; if (size_needed > _safe_file_size) { @@ -435,7 +488,8 @@ void block_table::_ensure_safe_write_unlocked(int fd, DISKOFF block_size, DISKOF _mutex_unlock(); int64_t size_after; - toku_maybe_preallocate_in_file(fd, size_needed, _safe_file_size, &size_after); + toku_maybe_preallocate_in_file( + fd, size_needed, _safe_file_size, &size_after); _mutex_lock(); _safe_file_size = size_after; @@ -444,11 +498,16 @@ void block_table::_ensure_safe_write_unlocked(int fd, DISKOFF block_size, DISKOF } } -void block_table::realloc_on_disk(BLOCKNUM b, DISKOFF size, DISKOFF *offset, FT ft, int fd, bool for_checkpoint, uint64_t heat) { +void block_table::realloc_on_disk(BLOCKNUM b, + DISKOFF size, + DISKOFF *offset, + FT ft, + int fd, + bool for_checkpoint) { _mutex_lock(); struct translation *t = &_current; _verify_valid_freeable_blocknum(t, b); - _realloc_on_disk_internal(b, size, offset, ft, for_checkpoint, heat); + _realloc_on_disk_internal(b, size, offset, ft, for_checkpoint); _ensure_safe_write_unlocked(fd, size, *offset); _mutex_unlock(); @@ -458,70 +517,97 @@ bool block_table::_pair_is_unallocated(struct block_translation_pair *pair) { return pair->size == 0 && pair->u.diskoff == diskoff_unused; } -// Effect: figure out where to put the inprogress btt on disk, allocate space for it there. -// The space must be 512-byte aligned (both the starting address and the size). -// As a result, the allcoated space may be a little bit bigger (up to the next 512-byte boundary) than the actual btt. +// Effect: figure out where to put the inprogress btt on disk, allocate space +// for it there. +// The space must be 512-byte aligned (both the starting address and the +// size). +// As a result, the allcoated space may be a little bit bigger (up to the next +// 512-byte boundary) than the actual btt. void block_table::_alloc_inprogress_translation_on_disk_unlocked() { toku_mutex_assert_locked(&_mutex); struct translation *t = &_inprogress; paranoid_invariant_notnull(t->block_translation); BLOCKNUM b = make_blocknum(RESERVED_BLOCKNUM_TRANSLATION); - //Each inprogress is allocated only once + // Each inprogress is allocated only once paranoid_invariant(_pair_is_unallocated(&t->block_translation[b.b])); - //Allocate a new block + // Allocate a new block int64_t size = _calculate_size_on_disk(t); uint64_t offset; - _bt_block_allocator.alloc_block(size, 0, &offset); + _bt_block_allocator->AllocBlock(size, &offset); t->block_translation[b.b].u.diskoff = offset; - t->block_translation[b.b].size = size; + t->block_translation[b.b].size = size; } // Effect: Serializes the blocktable to a wbuf (which starts uninitialized) -// A clean shutdown runs checkpoint start so that current and inprogress are copies. -// The resulting wbuf buffer is guaranteed to be be 512-byte aligned and the total length is a multiple of 512 (so we pad with zeros at the end if needd) -// The address is guaranteed to be 512-byte aligned, but the size is not guaranteed. -// It *is* guaranteed that we can read up to the next 512-byte boundary, however -void block_table::serialize_translation_to_wbuf(int fd, struct wbuf *w, - int64_t *address, int64_t *size) { +// A clean shutdown runs checkpoint start so that current and inprogress are +// copies. +// The resulting wbuf buffer is guaranteed to be be 512-byte aligned and the +// total length is a multiple of 512 (so we pad with zeros at the end if +// needd) +// The address is guaranteed to be 512-byte aligned, but the size is not +// guaranteed. +// It *is* guaranteed that we can read up to the next 512-byte boundary, +// however +void block_table::serialize_translation_to_wbuf(int fd, + struct wbuf *w, + int64_t *address, + int64_t *size) { _mutex_lock(); struct translation *t = &_inprogress; BLOCKNUM b = make_blocknum(RESERVED_BLOCKNUM_TRANSLATION); - _alloc_inprogress_translation_on_disk_unlocked(); // The allocated block must be 512-byte aligned to make O_DIRECT happy. + _alloc_inprogress_translation_on_disk_unlocked(); // The allocated block + // must be 512-byte + // aligned to make + // O_DIRECT happy. uint64_t size_translation = _calculate_size_on_disk(t); - uint64_t size_aligned = roundup_to_multiple(512, size_translation); - assert((int64_t)size_translation==t->block_translation[b.b].size); + uint64_t size_aligned = roundup_to_multiple(512, size_translation); + invariant((int64_t)size_translation == t->block_translation[b.b].size); { - //Init wbuf + // Init wbuf if (0) - printf("%s:%d writing translation table of size_translation %" PRIu64 " at %" PRId64 "\n", __FILE__, __LINE__, size_translation, t->block_translation[b.b].u.diskoff); + printf( + "%s:%d writing translation table of size_translation %" PRIu64 + " at %" PRId64 "\n", + __FILE__, + __LINE__, + size_translation, + t->block_translation[b.b].u.diskoff); char *XMALLOC_N_ALIGNED(512, size_aligned, buf); - for (uint64_t i=size_translation; ismallest_never_used_blocknum); - wbuf_BLOCKNUM(w, t->blocknum_freelist_head); + wbuf_BLOCKNUM(w, t->smallest_never_used_blocknum); + wbuf_BLOCKNUM(w, t->blocknum_freelist_head); int64_t i; - for (i=0; ismallest_never_used_blocknum.b; i++) { + for (i = 0; i < t->smallest_never_used_blocknum.b; i++) { if (0) - printf("%s:%d %" PRId64 ",%" PRId64 "\n", __FILE__, __LINE__, t->block_translation[i].u.diskoff, t->block_translation[i].size); + printf("%s:%d %" PRId64 ",%" PRId64 "\n", + __FILE__, + __LINE__, + t->block_translation[i].u.diskoff, + t->block_translation[i].size); wbuf_DISKOFF(w, t->block_translation[i].u.diskoff); wbuf_DISKOFF(w, t->block_translation[i].size); } uint32_t checksum = toku_x1764_finish(&w->checksum); wbuf_int(w, checksum); *address = t->block_translation[b.b].u.diskoff; - *size = size_translation; - assert((*address)%512 == 0); + *size = size_translation; + invariant((*address) % 512 == 0); _ensure_safe_write_unlocked(fd, size_aligned, *address); _mutex_unlock(); } -// Perhaps rename: purpose is get disk address of a block, given its blocknum (blockid?) -void block_table::_translate_blocknum_to_offset_size_unlocked(BLOCKNUM b, DISKOFF *offset, DISKOFF *size) { +// Perhaps rename: purpose is get disk address of a block, given its blocknum +// (blockid?) +void block_table::_translate_blocknum_to_offset_size_unlocked(BLOCKNUM b, + DISKOFF *offset, + DISKOFF *size) { struct translation *t = &_current; _verify_valid_blocknum(t, b); if (offset) { @@ -532,8 +618,11 @@ void block_table::_translate_blocknum_to_offset_size_unlocked(BLOCKNUM b, DISKOF } } -// Perhaps rename: purpose is get disk address of a block, given its blocknum (blockid?) -void block_table::translate_blocknum_to_offset_size(BLOCKNUM b, DISKOFF *offset, DISKOFF *size) { +// Perhaps rename: purpose is get disk address of a block, given its blocknum +// (blockid?) +void block_table::translate_blocknum_to_offset_size(BLOCKNUM b, + DISKOFF *offset, + DISKOFF *size) { _mutex_lock(); _translate_blocknum_to_offset_size_unlocked(b, offset, size); _mutex_unlock(); @@ -544,13 +633,13 @@ void block_table::translate_blocknum_to_offset_size(BLOCKNUM b, DISKOFF *offset, // given that one more never-used blocknum will soon be used. void block_table::_maybe_expand_translation(struct translation *t) { if (t->length_of_array <= t->smallest_never_used_blocknum.b) { - //expansion is necessary + // expansion is necessary uint64_t new_length = t->smallest_never_used_blocknum.b * 2; XREALLOC_N(new_length, t->block_translation); uint64_t i; for (i = t->length_of_array; i < new_length; i++) { t->block_translation[i].u.next_free_blocknum = freelist_null; - t->block_translation[i].size = size_is_free; + t->block_translation[i].size = size_is_free; } t->length_of_array = new_length; } @@ -563,7 +652,8 @@ void block_table::_allocate_blocknum_unlocked(BLOCKNUM *res, FT ft) { if (t->blocknum_freelist_head.b == freelist_null.b) { // no previously used blocknums are available // use a never used blocknum - _maybe_expand_translation(t); //Ensure a never used blocknums is available + _maybe_expand_translation( + t); // Ensure a never used blocknums is available result = t->smallest_never_used_blocknum; t->smallest_never_used_blocknum.b++; } else { // reuse a previously used blocknum @@ -571,11 +661,11 @@ void block_table::_allocate_blocknum_unlocked(BLOCKNUM *res, FT ft) { BLOCKNUM next = t->block_translation[result.b].u.next_free_blocknum; t->blocknum_freelist_head = next; } - //Verify the blocknum is free + // Verify the blocknum is free paranoid_invariant(t->block_translation[result.b].size == size_is_free); - //blocknum is not free anymore + // blocknum is not free anymore t->block_translation[result.b].u.diskoff = diskoff_unused; - t->block_translation[result.b].size = 0; + t->block_translation[result.b].size = 0; _verify_valid_freeable_blocknum(t, result); *res = result; ft_set_dirty(ft, false); @@ -587,42 +677,46 @@ void block_table::allocate_blocknum(BLOCKNUM *res, FT ft) { _mutex_unlock(); } -void block_table::_free_blocknum_in_translation(struct translation *t, BLOCKNUM b) { +void block_table::_free_blocknum_in_translation(struct translation *t, + BLOCKNUM b) { _verify_valid_freeable_blocknum(t, b); paranoid_invariant(t->block_translation[b.b].size != size_is_free); - t->block_translation[b.b].size = size_is_free; + t->block_translation[b.b].size = size_is_free; t->block_translation[b.b].u.next_free_blocknum = t->blocknum_freelist_head; - t->blocknum_freelist_head = b; + t->blocknum_freelist_head = b; } // Effect: Free a blocknum. // If the blocknum holds the only reference to a block on disk, free that block -void block_table::_free_blocknum_unlocked(BLOCKNUM *bp, FT ft, bool for_checkpoint) { +void block_table::_free_blocknum_unlocked(BLOCKNUM *bp, + FT ft, + bool for_checkpoint) { toku_mutex_assert_locked(&_mutex); BLOCKNUM b = *bp; - bp->b = 0; //Remove caller's reference. + bp->b = 0; // Remove caller's reference. struct block_translation_pair old_pair = _current.block_translation[b.b]; _free_blocknum_in_translation(&_current, b); if (for_checkpoint) { - paranoid_invariant(ft->checkpoint_header->type == FT_CHECKPOINT_INPROGRESS); + paranoid_invariant(ft->checkpoint_header->type == + FT_CHECKPOINT_INPROGRESS); _free_blocknum_in_translation(&_inprogress, b); } - //If the size is 0, no disk block has ever been assigned to this blocknum. + // If the size is 0, no disk block has ever been assigned to this blocknum. if (old_pair.size > 0) { - //Free the old block if it is not still in use by the checkpoint in progress or the previous checkpoint - bool cannot_free = (bool) - (_translation_prevents_freeing(&_inprogress, b, &old_pair) || - _translation_prevents_freeing(&_checkpointed, b, &old_pair)); + // Free the old block if it is not still in use by the checkpoint in + // progress or the previous checkpoint + bool cannot_free = + _translation_prevents_freeing(&_inprogress, b, &old_pair) || + _translation_prevents_freeing(&_checkpointed, b, &old_pair); if (!cannot_free) { - _bt_block_allocator.free_block(old_pair.u.diskoff); + _bt_block_allocator->FreeBlock(old_pair.u.diskoff, old_pair.size); } - } - else { - paranoid_invariant(old_pair.size==0); + } else { + paranoid_invariant(old_pair.size == 0); paranoid_invariant(old_pair.u.diskoff == diskoff_unused); } ft_set_dirty(ft, for_checkpoint); @@ -644,13 +738,14 @@ void block_table::verify_no_free_blocknums() { void block_table::free_unused_blocknums(BLOCKNUM root) { _mutex_lock(); int64_t smallest = _current.smallest_never_used_blocknum.b; - for (int64_t i=RESERVED_BLOCKNUMS; i < smallest; i++) { + for (int64_t i = RESERVED_BLOCKNUMS; i < smallest; i++) { if (i == root.b) { continue; } BLOCKNUM b = make_blocknum(i); if (_current.block_translation[b.b].size == 0) { - invariant(_current.block_translation[b.b].u.diskoff == diskoff_unused); + invariant(_current.block_translation[b.b].u.diskoff == + diskoff_unused); _free_blocknum_in_translation(&_current, b); } } @@ -675,13 +770,14 @@ bool block_table::_no_data_blocks_except_root(BLOCKNUM root) { goto cleanup; } } - cleanup: +cleanup: _mutex_unlock(); return ok; } // Verify there are no data blocks except root. -// TODO(leif): This actually takes a lock, but I don't want to fix all the callers right now. +// TODO(leif): This actually takes a lock, but I don't want to fix all the +// callers right now. void block_table::verify_no_data_blocks_except_root(BLOCKNUM UU(root)) { paranoid_invariant(_no_data_blocks_except_root(root)); } @@ -705,13 +801,24 @@ void block_table::_dump_translation_internal(FILE *f, struct translation *t) { if (t->block_translation) { BLOCKNUM b = make_blocknum(RESERVED_BLOCKNUM_TRANSLATION); fprintf(f, " length_of_array[%" PRId64 "]", t->length_of_array); - fprintf(f, " smallest_never_used_blocknum[%" PRId64 "]", t->smallest_never_used_blocknum.b); - fprintf(f, " blocknum_free_list_head[%" PRId64 "]", t->blocknum_freelist_head.b); - fprintf(f, " size_on_disk[%" PRId64 "]", t->block_translation[b.b].size); - fprintf(f, " location_on_disk[%" PRId64 "]\n", t->block_translation[b.b].u.diskoff); + fprintf(f, + " smallest_never_used_blocknum[%" PRId64 "]", + t->smallest_never_used_blocknum.b); + fprintf(f, + " blocknum_free_list_head[%" PRId64 "]", + t->blocknum_freelist_head.b); + fprintf( + f, " size_on_disk[%" PRId64 "]", t->block_translation[b.b].size); + fprintf(f, + " location_on_disk[%" PRId64 "]\n", + t->block_translation[b.b].u.diskoff); int64_t i; - for (i=0; ilength_of_array; i++) { - fprintf(f, " %" PRId64 ": %" PRId64 " %" PRId64 "\n", i, t->block_translation[i].u.diskoff, t->block_translation[i].size); + for (i = 0; i < t->length_of_array; i++) { + fprintf(f, + " %" PRId64 ": %" PRId64 " %" PRId64 "\n", + i, + t->block_translation[i].u.diskoff, + t->block_translation[i].size); } fprintf(f, "\n"); } else { @@ -724,9 +831,13 @@ void block_table::_dump_translation_internal(FILE *f, struct translation *t) { void block_table::dump_translation_table_pretty(FILE *f) { _mutex_lock(); struct translation *t = &_checkpointed; - assert(t->block_translation != nullptr); + invariant(t->block_translation != nullptr); for (int64_t i = 0; i < t->length_of_array; ++i) { - fprintf(f, "%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", i, t->block_translation[i].u.diskoff, t->block_translation[i].size); + fprintf(f, + "%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", + i, + t->block_translation[i].u.diskoff, + t->block_translation[i].size); } _mutex_unlock(); } @@ -750,7 +861,10 @@ void block_table::blocknum_dump_translation(BLOCKNUM b) { struct translation *t = &_current; if (b.b < t->length_of_array) { struct block_translation_pair *bx = &t->block_translation[b.b]; - printf("%" PRId64 ": %" PRId64 " %" PRId64 "\n", b.b, bx->u.diskoff, bx->size); + printf("%" PRId64 ": %" PRId64 " %" PRId64 "\n", + b.b, + bx->u.diskoff, + bx->size); } _mutex_unlock(); } @@ -763,26 +877,31 @@ void block_table::destroy(void) { toku_free(_inprogress.block_translation); toku_free(_checkpointed.block_translation); - _bt_block_allocator.destroy(); + _bt_block_allocator->Destroy(); + delete _bt_block_allocator; toku_mutex_destroy(&_mutex); nb_mutex_destroy(&_safe_file_size_lock); } -int block_table::_translation_deserialize_from_buffer(struct translation *t, - DISKOFF location_on_disk, - uint64_t size_on_disk, - // out: buffer with serialized translation - unsigned char *translation_buffer) { +int block_table::_translation_deserialize_from_buffer( + struct translation *t, + DISKOFF location_on_disk, + uint64_t size_on_disk, + // out: buffer with serialized translation + unsigned char *translation_buffer) { int r = 0; - assert(location_on_disk != 0); + invariant(location_on_disk != 0); t->type = TRANSLATION_CHECKPOINTED; // check the checksum uint32_t x1764 = toku_x1764_memory(translation_buffer, size_on_disk - 4); uint64_t offset = size_on_disk - 4; - uint32_t stored_x1764 = toku_dtoh32(*(int*)(translation_buffer + offset)); + uint32_t stored_x1764 = toku_dtoh32(*(int *)(translation_buffer + offset)); if (x1764 != stored_x1764) { - fprintf(stderr, "Translation table checksum failure: calc=0x%08x read=0x%08x\n", x1764, stored_x1764); + fprintf(stderr, + "Translation table checksum failure: calc=0x%08x read=0x%08x\n", + x1764, + stored_x1764); r = TOKUDB_BAD_CHECKSUM; goto exit; } @@ -790,42 +909,47 @@ int block_table::_translation_deserialize_from_buffer(struct translation *t, struct rbuf rb; rb.buf = translation_buffer; rb.ndone = 0; - rb.size = size_on_disk-4;//4==checksum + rb.size = size_on_disk - 4; // 4==checksum - t->smallest_never_used_blocknum = rbuf_blocknum(&rb); + t->smallest_never_used_blocknum = rbuf_blocknum(&rb); t->length_of_array = t->smallest_never_used_blocknum.b; invariant(t->smallest_never_used_blocknum.b >= RESERVED_BLOCKNUMS); - t->blocknum_freelist_head = rbuf_blocknum(&rb); + t->blocknum_freelist_head = rbuf_blocknum(&rb); XMALLOC_N(t->length_of_array, t->block_translation); for (int64_t i = 0; i < t->length_of_array; i++) { t->block_translation[i].u.diskoff = rbuf_DISKOFF(&rb); t->block_translation[i].size = rbuf_DISKOFF(&rb); } - invariant(_calculate_size_on_disk(t) == (int64_t) size_on_disk); - invariant(t->block_translation[RESERVED_BLOCKNUM_TRANSLATION].size == (int64_t) size_on_disk); - invariant(t->block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff == location_on_disk); + invariant(_calculate_size_on_disk(t) == (int64_t)size_on_disk); + invariant(t->block_translation[RESERVED_BLOCKNUM_TRANSLATION].size == + (int64_t)size_on_disk); + invariant(t->block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff == + location_on_disk); exit: return r; } int block_table::iterate(enum translation_type type, - BLOCKTABLE_CALLBACK f, void *extra, bool data_only, bool used_only) { + BLOCKTABLE_CALLBACK f, + void *extra, + bool data_only, + bool used_only) { struct translation *src; - + int r = 0; switch (type) { - case TRANSLATION_CURRENT: - src = &_current; - break; - case TRANSLATION_INPROGRESS: - src = &_inprogress; - break; - case TRANSLATION_CHECKPOINTED: - src = &_checkpointed; - break; - default: - r = EINVAL; + case TRANSLATION_CURRENT: + src = &_current; + break; + case TRANSLATION_INPROGRESS: + src = &_inprogress; + break; + case TRANSLATION_CHECKPOINTED: + src = &_checkpointed; + break; + default: + r = EINVAL; } struct translation fakecurrent; @@ -839,12 +963,15 @@ int block_table::iterate(enum translation_type type, src->block_translation[RESERVED_BLOCKNUM_TRANSLATION]; _mutex_unlock(); int64_t i; - for (i=0; ismallest_never_used_blocknum.b; i++) { + for (i = 0; i < t->smallest_never_used_blocknum.b; i++) { struct block_translation_pair pair = t->block_translation[i]; - if (data_only && i< RESERVED_BLOCKNUMS) continue; - if (used_only && pair.size <= 0) continue; + if (data_only && i < RESERVED_BLOCKNUMS) + continue; + if (used_only && pair.size <= 0) + continue; r = f(make_blocknum(i), pair.size, pair.u.diskoff, extra); - if (r!=0) break; + if (r != 0) + break; } toku_free(t->block_translation); } @@ -856,8 +983,11 @@ typedef struct { int64_t total_space; } frag_extra; -static int frag_helper(BLOCKNUM UU(b), int64_t size, int64_t address, void *extra) { - frag_extra *info = (frag_extra *) extra; +static int frag_helper(BLOCKNUM UU(b), + int64_t size, + int64_t address, + void *extra) { + frag_extra *info = (frag_extra *)extra; if (size + address > info->total_space) info->total_space = size + address; @@ -865,22 +995,30 @@ static int frag_helper(BLOCKNUM UU(b), int64_t size, int64_t address, void *extr return 0; } -void block_table::internal_fragmentation(int64_t *total_sizep, int64_t *used_sizep) { - frag_extra info = { 0, 0 }; +void block_table::internal_fragmentation(int64_t *total_sizep, + int64_t *used_sizep) { + frag_extra info = {0, 0}; int r = iterate(TRANSLATION_CHECKPOINTED, frag_helper, &info, false, true); - assert_zero(r); + invariant_zero(r); - if (total_sizep) *total_sizep = info.total_space; - if (used_sizep) *used_sizep = info.used_space; + if (total_sizep) + *total_sizep = info.total_space; + if (used_sizep) + *used_sizep = info.used_space; } -void block_table::_realloc_descriptor_on_disk_unlocked(DISKOFF size, DISKOFF *offset, FT ft) { +void block_table::_realloc_descriptor_on_disk_unlocked(DISKOFF size, + DISKOFF *offset, + FT ft) { toku_mutex_assert_locked(&_mutex); BLOCKNUM b = make_blocknum(RESERVED_BLOCKNUM_DESCRIPTOR); - _realloc_on_disk_internal(b, size, offset, ft, false, 0); + _realloc_on_disk_internal(b, size, offset, ft, false); } -void block_table::realloc_descriptor_on_disk(DISKOFF size, DISKOFF *offset, FT ft, int fd) { +void block_table::realloc_descriptor_on_disk(DISKOFF size, + DISKOFF *offset, + FT ft, + int fd) { _mutex_lock(); _realloc_descriptor_on_disk_unlocked(size, offset, ft); _ensure_safe_write_unlocked(fd, size, *offset); @@ -897,11 +1035,12 @@ void block_table::get_descriptor_offset_size(DISKOFF *offset, DISKOFF *size) { void block_table::get_fragmentation_unlocked(TOKU_DB_FRAGMENTATION report) { // Requires: blocktable lock is held. // Requires: report->file_size_bytes is already filled in. - + // Count the headers. - report->data_bytes = block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE; + report->data_bytes = BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE; report->data_blocks = 1; - report->checkpoint_bytes_additional = block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE; + report->checkpoint_bytes_additional = + BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE; report->checkpoint_blocks_additional = 1; struct translation *current = &_current; @@ -915,30 +1054,34 @@ void block_table::get_fragmentation_unlocked(TOKU_DB_FRAGMENTATION report) { struct translation *checkpointed = &_checkpointed; for (int64_t i = 0; i < checkpointed->length_of_array; i++) { - struct block_translation_pair *pair = &checkpointed->block_translation[i]; - if (pair->size > 0 && !(i < current->length_of_array && - current->block_translation[i].size > 0 && - current->block_translation[i].u.diskoff == pair->u.diskoff)) { - report->checkpoint_bytes_additional += pair->size; - report->checkpoint_blocks_additional++; + struct block_translation_pair *pair = + &checkpointed->block_translation[i]; + if (pair->size > 0 && + !(i < current->length_of_array && + current->block_translation[i].size > 0 && + current->block_translation[i].u.diskoff == pair->u.diskoff)) { + report->checkpoint_bytes_additional += pair->size; + report->checkpoint_blocks_additional++; } } struct translation *inprogress = &_inprogress; for (int64_t i = 0; i < inprogress->length_of_array; i++) { struct block_translation_pair *pair = &inprogress->block_translation[i]; - if (pair->size > 0 && !(i < current->length_of_array && - current->block_translation[i].size > 0 && - current->block_translation[i].u.diskoff == pair->u.diskoff) && - !(i < checkpointed->length_of_array && - checkpointed->block_translation[i].size > 0 && - checkpointed->block_translation[i].u.diskoff == pair->u.diskoff)) { + if (pair->size > 0 && + !(i < current->length_of_array && + current->block_translation[i].size > 0 && + current->block_translation[i].u.diskoff == pair->u.diskoff) && + !(i < checkpointed->length_of_array && + checkpointed->block_translation[i].size > 0 && + checkpointed->block_translation[i].u.diskoff == + pair->u.diskoff)) { report->checkpoint_bytes_additional += pair->size; report->checkpoint_blocks_additional++; } } - _bt_block_allocator.get_unused_statistics(report); + _bt_block_allocator->UnusedStatistics(report); } void block_table::get_info64(struct ftinfo64 *s) { @@ -967,25 +1110,38 @@ void block_table::get_info64(struct ftinfo64 *s) { _mutex_unlock(); } -int block_table::iterate_translation_tables(uint64_t checkpoint_count, - int (*iter)(uint64_t checkpoint_count, - int64_t total_num_rows, - int64_t blocknum, - int64_t diskoff, - int64_t size, - void *extra), - void *iter_extra) { +int block_table::iterate_translation_tables( + uint64_t checkpoint_count, + int (*iter)(uint64_t checkpoint_count, + int64_t total_num_rows, + int64_t blocknum, + int64_t diskoff, + int64_t size, + void *extra), + void *iter_extra) { int error = 0; _mutex_lock(); - int64_t total_num_rows = _current.length_of_array + _checkpointed.length_of_array; + int64_t total_num_rows = + _current.length_of_array + _checkpointed.length_of_array; for (int64_t i = 0; error == 0 && i < _current.length_of_array; ++i) { struct block_translation_pair *block = &_current.block_translation[i]; - error = iter(checkpoint_count, total_num_rows, i, block->u.diskoff, block->size, iter_extra); + error = iter(checkpoint_count, + total_num_rows, + i, + block->u.diskoff, + block->size, + iter_extra); } for (int64_t i = 0; error == 0 && i < _checkpointed.length_of_array; ++i) { - struct block_translation_pair *block = &_checkpointed.block_translation[i]; - error = iter(checkpoint_count - 1, total_num_rows, i, block->u.diskoff, block->size, iter_extra); + struct block_translation_pair *block = + &_checkpointed.block_translation[i]; + error = iter(checkpoint_count - 1, + total_num_rows, + i, + block->u.diskoff, + block->size, + iter_extra); } _mutex_unlock(); diff --git a/storage/tokudb/PerconaFT/ft/serialize/block_table.h b/storage/tokudb/PerconaFT/ft/serialize/block_table.h index 8d39167454094..dd732d4f3726c 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/block_table.h +++ b/storage/tokudb/PerconaFT/ft/serialize/block_table.h @@ -62,13 +62,16 @@ enum { RESERVED_BLOCKNUMS }; -typedef int (*BLOCKTABLE_CALLBACK)(BLOCKNUM b, int64_t size, int64_t address, void *extra); +typedef int (*BLOCKTABLE_CALLBACK)(BLOCKNUM b, + int64_t size, + int64_t address, + void *extra); static inline BLOCKNUM make_blocknum(int64_t b) { - BLOCKNUM result = { .b = b }; + BLOCKNUM result = {.b = b}; return result; } -static const BLOCKNUM ROLLBACK_NONE = { .b = 0 }; +static const BLOCKNUM ROLLBACK_NONE = {.b = 0}; /** * There are three copies of the translation table (btt) in the block table: @@ -80,18 +83,20 @@ static const BLOCKNUM ROLLBACK_NONE = { .b = 0 }; * * inprogress Is only filled by copying from current, * and is the only version ever serialized to disk. - * (It is serialized to disk on checkpoint and clean shutdown.) + * (It is serialized to disk on checkpoint and clean + *shutdown.) * At end of checkpoint it replaces 'checkpointed'. * During a checkpoint, any 'pending' dirty writes will update * inprogress. * * current Is initialized by copying from checkpointed, - * is the only version ever modified while the database is in use, + * is the only version ever modified while the database is in + *use, * and is the only version ever copied to inprogress. * It is never stored on disk. */ class block_table { -public: + public: enum translation_type { TRANSLATION_NONE = 0, TRANSLATION_CURRENT, @@ -102,7 +107,10 @@ class block_table { void create(); - int create_from_buffer(int fd, DISKOFF location_on_disk, DISKOFF size_on_disk, unsigned char *translation_buffer); + int create_from_buffer(int fd, + DISKOFF location_on_disk, + DISKOFF size_on_disk, + unsigned char *translation_buffer); void destroy(); @@ -114,11 +122,21 @@ class block_table { // Blocknums void allocate_blocknum(BLOCKNUM *res, struct ft *ft); - void realloc_on_disk(BLOCKNUM b, DISKOFF size, DISKOFF *offset, struct ft *ft, int fd, bool for_checkpoint, uint64_t heat); + void realloc_on_disk(BLOCKNUM b, + DISKOFF size, + DISKOFF *offset, + struct ft *ft, + int fd, + bool for_checkpoint); void free_blocknum(BLOCKNUM *b, struct ft *ft, bool for_checkpoint); - void translate_blocknum_to_offset_size(BLOCKNUM b, DISKOFF *offset, DISKOFF *size); + void translate_blocknum_to_offset_size(BLOCKNUM b, + DISKOFF *offset, + DISKOFF *size); void free_unused_blocknums(BLOCKNUM root); - void realloc_descriptor_on_disk(DISKOFF size, DISKOFF *offset, struct ft *ft, int fd); + void realloc_descriptor_on_disk(DISKOFF size, + DISKOFF *offset, + struct ft *ft, + int fd); void get_descriptor_offset_size(DISKOFF *offset, DISKOFF *size); // External verfication @@ -127,15 +145,22 @@ class block_table { void verify_no_free_blocknums(); // Serialization - void serialize_translation_to_wbuf(int fd, struct wbuf *w, int64_t *address, int64_t *size); + void serialize_translation_to_wbuf(int fd, + struct wbuf *w, + int64_t *address, + int64_t *size); // DEBUG ONLY (ftdump included), tests included void blocknum_dump_translation(BLOCKNUM b); void dump_translation_table_pretty(FILE *f); void dump_translation_table(FILE *f); - void block_free(uint64_t offset); + void block_free(uint64_t offset, uint64_t size); - int iterate(enum translation_type type, BLOCKTABLE_CALLBACK f, void *extra, bool data_only, bool used_only); + int iterate(enum translation_type type, + BLOCKTABLE_CALLBACK f, + void *extra, + bool data_only, + bool used_only); void internal_fragmentation(int64_t *total_sizep, int64_t *used_sizep); // Requires: blocktable lock is held. @@ -146,13 +171,16 @@ class block_table { void get_info64(struct ftinfo64 *); - int iterate_translation_tables(uint64_t, int (*)(uint64_t, int64_t, int64_t, int64_t, int64_t, void *), void *); + int iterate_translation_tables( + uint64_t, + int (*)(uint64_t, int64_t, int64_t, int64_t, int64_t, void *), + void *); -private: + private: struct block_translation_pair { // If in the freelist, use next_free_blocknum, otherwise diskoff. union { - DISKOFF diskoff; + DISKOFF diskoff; BLOCKNUM next_free_blocknum; } u; @@ -173,7 +201,8 @@ class block_table { struct translation { enum translation_type type; - // Number of elements in array (block_translation). always >= smallest_never_used_blocknum + // Number of elements in array (block_translation). always >= + // smallest_never_used_blocknum int64_t length_of_array; BLOCKNUM smallest_never_used_blocknum; @@ -181,20 +210,28 @@ class block_table { BLOCKNUM blocknum_freelist_head; struct block_translation_pair *block_translation; - // size_on_disk is stored in block_translation[RESERVED_BLOCKNUM_TRANSLATION].size - // location_on is stored in block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff + // size_on_disk is stored in + // block_translation[RESERVED_BLOCKNUM_TRANSLATION].size + // location_on is stored in + // block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff }; void _create_internal(); - int _translation_deserialize_from_buffer(struct translation *t, // destination into which to deserialize - DISKOFF location_on_disk, // location of translation_buffer - uint64_t size_on_disk, - unsigned char * translation_buffer); // buffer with serialized translation - - void _copy_translation(struct translation *dst, struct translation *src, enum translation_type newtype); + int _translation_deserialize_from_buffer( + struct translation *t, // destination into which to deserialize + DISKOFF location_on_disk, // location of translation_buffer + uint64_t size_on_disk, + unsigned char * + translation_buffer); // buffer with serialized translation + + void _copy_translation(struct translation *dst, + struct translation *src, + enum translation_type newtype); void _maybe_optimize_translation(struct translation *t); void _maybe_expand_translation(struct translation *t); - bool _translation_prevents_freeing(struct translation *t, BLOCKNUM b, struct block_translation_pair *old_pair); + bool _translation_prevents_freeing(struct translation *t, + BLOCKNUM b, + struct block_translation_pair *old_pair); void _free_blocknum_in_translation(struct translation *t, BLOCKNUM b); int64_t _calculate_size_on_disk(struct translation *t); bool _pair_is_unallocated(struct block_translation_pair *pair); @@ -203,14 +240,26 @@ class block_table { // Blocknum management void _allocate_blocknum_unlocked(BLOCKNUM *res, struct ft *ft); - void _free_blocknum_unlocked(BLOCKNUM *bp, struct ft *ft, bool for_checkpoint); - void _realloc_descriptor_on_disk_unlocked(DISKOFF size, DISKOFF *offset, struct ft *ft); - void _realloc_on_disk_internal(BLOCKNUM b, DISKOFF size, DISKOFF *offset, struct ft *ft, bool for_checkpoint, uint64_t heat); - void _translate_blocknum_to_offset_size_unlocked(BLOCKNUM b, DISKOFF *offset, DISKOFF *size); + void _free_blocknum_unlocked(BLOCKNUM *bp, + struct ft *ft, + bool for_checkpoint); + void _realloc_descriptor_on_disk_unlocked(DISKOFF size, + DISKOFF *offset, + struct ft *ft); + void _realloc_on_disk_internal(BLOCKNUM b, + DISKOFF size, + DISKOFF *offset, + struct ft *ft, + bool for_checkpoint); + void _translate_blocknum_to_offset_size_unlocked(BLOCKNUM b, + DISKOFF *offset, + DISKOFF *size); // File management void _maybe_truncate_file(int fd, uint64_t size_needed_before); - void _ensure_safe_write_unlocked(int fd, DISKOFF block_size, DISKOFF block_offset); + void _ensure_safe_write_unlocked(int fd, + DISKOFF block_size, + DISKOFF block_offset); // Verification bool _is_valid_blocknum(struct translation *t, BLOCKNUM b); @@ -220,29 +269,33 @@ class block_table { bool _no_data_blocks_except_root(BLOCKNUM root); bool _blocknum_allocated(BLOCKNUM b); - // Locking + // Locking // // TODO: Move the lock to the FT void _mutex_lock(); void _mutex_unlock(); - // The current translation is the one used by client threads. + // The current translation is the one used by client threads. // It is not represented on disk. struct translation _current; - // The translation used by the checkpoint currently in progress. - // If the checkpoint thread allocates a block, it must also update the current translation. + // The translation used by the checkpoint currently in progress. + // If the checkpoint thread allocates a block, it must also update the + // current translation. struct translation _inprogress; - // The translation for the data that shall remain inviolate on disk until the next checkpoint finishes, + // The translation for the data that shall remain inviolate on disk until + // the next checkpoint finishes, // after which any blocks used only in this translation can be freed. struct translation _checkpointed; - // The in-memory data structure for block allocation. + // The in-memory data structure for block allocation. // There is no on-disk data structure for block allocation. - // Note: This is *allocation* not *translation* - the block allocator is unaware of which - // blocks are used for which translation, but simply allocates and deallocates blocks. - block_allocator _bt_block_allocator; + // Note: This is *allocation* not *translation* - the block allocator is + // unaware of which + // blocks are used for which translation, but simply allocates and + // deallocates blocks. + BlockAllocator *_bt_block_allocator; toku_mutex_t _mutex; struct nb_mutex _safe_file_size_lock; bool _checkpoint_skipped; @@ -257,16 +310,16 @@ class block_table { #include "ft/serialize/wbuf.h" -static inline void wbuf_BLOCKNUM (struct wbuf *w, BLOCKNUM b) { +static inline void wbuf_BLOCKNUM(struct wbuf *w, BLOCKNUM b) { wbuf_ulonglong(w, b.b); } -static inline void wbuf_nocrc_BLOCKNUM (struct wbuf *w, BLOCKNUM b) { +static inline void wbuf_nocrc_BLOCKNUM(struct wbuf *w, BLOCKNUM b) { wbuf_nocrc_ulonglong(w, b.b); } static inline void wbuf_DISKOFF(struct wbuf *wb, DISKOFF off) { - wbuf_ulonglong(wb, (uint64_t) off); + wbuf_ulonglong(wb, (uint64_t)off); } #include "ft/serialize/rbuf.h" @@ -280,6 +333,8 @@ static inline BLOCKNUM rbuf_blocknum(struct rbuf *rb) { return result; } -static inline void rbuf_ma_BLOCKNUM(struct rbuf *rb, memarena *UU(ma), BLOCKNUM *blocknum) { +static inline void rbuf_ma_BLOCKNUM(struct rbuf *rb, + memarena *UU(ma), + BLOCKNUM *blocknum) { *blocknum = rbuf_blocknum(rb); } diff --git a/storage/tokudb/PerconaFT/ft/serialize/compress.cc b/storage/tokudb/PerconaFT/ft/serialize/compress.cc index 1719b6b7cb51d..c2f815c6cf22c 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/compress.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/compress.cc @@ -235,7 +235,7 @@ void toku_decompress (Bytef *dest, uLongf destLen, strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; - char windowBits = source[1]; + int8_t windowBits = source[1]; int r = inflateInit2(&strm, windowBits); lazy_assert(r == Z_OK); strm.next_out = dest; diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc index 49d4368a3ab83..8fcb529341205 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc @@ -217,8 +217,8 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version) // translation table itself won't fit in main memory. ssize_t readsz = toku_os_pread(fd, tbuf, size_to_read, translation_address_on_disk); - assert(readsz >= translation_size_on_disk); - assert(readsz <= (ssize_t)size_to_read); + invariant(readsz >= translation_size_on_disk); + invariant(readsz <= (ssize_t)size_to_read); } // Create table and read in data. r = ft->blocktable.create_from_buffer(fd, @@ -411,73 +411,90 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version) return r; } -static size_t -serialize_ft_min_size (uint32_t version) { +static size_t serialize_ft_min_size(uint32_t version) { size_t size = 0; - switch(version) { - case FT_LAYOUT_VERSION_29: - size += sizeof(uint64_t); // logrows in ft - case FT_LAYOUT_VERSION_28: - size += sizeof(uint32_t); // fanout in ft - case FT_LAYOUT_VERSION_27: - case FT_LAYOUT_VERSION_26: - case FT_LAYOUT_VERSION_25: - case FT_LAYOUT_VERSION_24: - case FT_LAYOUT_VERSION_23: - case FT_LAYOUT_VERSION_22: - case FT_LAYOUT_VERSION_21: - size += sizeof(MSN); // max_msn_in_ft - case FT_LAYOUT_VERSION_20: - case FT_LAYOUT_VERSION_19: - size += 1; // compression method - size += sizeof(MSN); // highest_unused_msn_for_upgrade - case FT_LAYOUT_VERSION_18: - size += sizeof(uint64_t); // time_of_last_optimize_begin - size += sizeof(uint64_t); // time_of_last_optimize_end - size += sizeof(uint32_t); // count_of_optimize_in_progress - size += sizeof(MSN); // msn_at_start_of_last_completed_optimize - size -= 8; // removed num_blocks_to_upgrade_14 - size -= 8; // removed num_blocks_to_upgrade_13 - case FT_LAYOUT_VERSION_17: - size += 16; - invariant(sizeof(STAT64INFO_S) == 16); - case FT_LAYOUT_VERSION_16: - case FT_LAYOUT_VERSION_15: - size += 4; // basement node size - size += 8; // num_blocks_to_upgrade_14 (previously num_blocks_to_upgrade, now one int each for upgrade from 13, 14 - size += 8; // time of last verification - case FT_LAYOUT_VERSION_14: - size += 8; //TXNID that created - case FT_LAYOUT_VERSION_13: - size += ( 4 // build_id - +4 // build_id_original - +8 // time_of_creation - +8 // time_of_last_modification - ); + switch (version) { + case FT_LAYOUT_VERSION_29: + size += sizeof(uint64_t); // logrows in ft + case FT_LAYOUT_VERSION_28: + size += sizeof(uint32_t); // fanout in ft + case FT_LAYOUT_VERSION_27: + case FT_LAYOUT_VERSION_26: + case FT_LAYOUT_VERSION_25: + case FT_LAYOUT_VERSION_24: + case FT_LAYOUT_VERSION_23: + case FT_LAYOUT_VERSION_22: + case FT_LAYOUT_VERSION_21: + size += sizeof(MSN); // max_msn_in_ft + case FT_LAYOUT_VERSION_20: + case FT_LAYOUT_VERSION_19: + size += 1; // compression method + size += sizeof(MSN); // highest_unused_msn_for_upgrade + case FT_LAYOUT_VERSION_18: + size += sizeof(uint64_t); // time_of_last_optimize_begin + size += sizeof(uint64_t); // time_of_last_optimize_end + size += sizeof(uint32_t); // count_of_optimize_in_progress + size += sizeof(MSN); // msn_at_start_of_last_completed_optimize + size -= 8; // removed num_blocks_to_upgrade_14 + size -= 8; // removed num_blocks_to_upgrade_13 + case FT_LAYOUT_VERSION_17: + size += 16; + invariant(sizeof(STAT64INFO_S) == 16); + case FT_LAYOUT_VERSION_16: + case FT_LAYOUT_VERSION_15: + size += 4; // basement node size + size += 8; // num_blocks_to_upgrade_14 (previously + // num_blocks_to_upgrade, now one int each for upgrade + // from 13, 14 + size += 8; // time of last verification + case FT_LAYOUT_VERSION_14: + size += 8; // TXNID that created + case FT_LAYOUT_VERSION_13: + size += (4 // build_id + + + 4 // build_id_original + + + 8 // time_of_creation + + + 8 // time_of_last_modification + ); // fall through - case FT_LAYOUT_VERSION_12: - size += (+8 // "tokudata" - +4 // version - +4 // original_version - +4 // size - +8 // byte order verification - +8 // checkpoint_count - +8 // checkpoint_lsn - +4 // tree's nodesize - +8 // translation_size_on_disk - +8 // translation_address_on_disk - +4 // checksum - +8 // Number of blocks in old version. - +8 // diskoff - +4 // flags - ); - break; - default: - abort(); - } - - lazy_assert(size <= block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE); + case FT_LAYOUT_VERSION_12: + size += (+8 // "tokudata" + + + 4 // version + + + 4 // original_version + + + 4 // size + + + 8 // byte order verification + + + 8 // checkpoint_count + + + 8 // checkpoint_lsn + + + 4 // tree's nodesize + + + 8 // translation_size_on_disk + + + 8 // translation_address_on_disk + + + 4 // checksum + + + 8 // Number of blocks in old version. + + + 8 // diskoff + + + 4 // flags + ); + break; + default: + abort(); + } + + lazy_assert(size <= BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE); return size; } @@ -486,7 +503,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd, struct rbuf *rb, uint64_t *checkpoint_count, LSN *checkpoint_lsn, - uint32_t * version_p) + uint32_t *version_p) // Effect: Read and parse the header of a fractalal tree // // Simply reading the raw bytes of the header into an rbuf is insensitive @@ -496,18 +513,18 @@ int deserialize_ft_from_fd_into_rbuf(int fd, // file AND the header is useless { int r = 0; - const int64_t prefix_size = 8 + // magic ("tokudata") - 4 + // version - 4 + // build_id - 4; // size + const int64_t prefix_size = 8 + // magic ("tokudata") + 4 + // version + 4 + // build_id + 4; // size const int64_t read_size = roundup_to_multiple(512, prefix_size); unsigned char *XMALLOC_N_ALIGNED(512, read_size, prefix); rb->buf = NULL; int64_t n = toku_os_pread(fd, prefix, read_size, offset_of_header); if (n != read_size) { - if (n==0) { + if (n == 0) { r = TOKUDB_DICTIONARY_NO_HEADER; - } else if (n<0) { + } else if (n < 0) { r = get_error_errno(); } else { r = EINVAL; @@ -518,95 +535,102 @@ int deserialize_ft_from_fd_into_rbuf(int fd, rbuf_init(rb, prefix, prefix_size); - //Check magic number + // Check magic number const void *magic; rbuf_literal_bytes(rb, &magic, 8); - if (memcmp(magic,"tokudata",8)!=0) { - if ((*(uint64_t*)magic) == 0) { + if (memcmp(magic, "tokudata", 8) != 0) { + if ((*(uint64_t *)magic) == 0) { r = TOKUDB_DICTIONARY_NO_HEADER; } else { - r = EINVAL; //Not a tokudb file! Do not use. + r = EINVAL; // Not a tokudb file! Do not use. } goto exit; } - //Version MUST be in network order regardless of disk order. + // Version MUST be in network order regardless of disk order. uint32_t version; version = rbuf_network_int(rb); *version_p = version; if (version < FT_LAYOUT_MIN_SUPPORTED_VERSION) { - r = TOKUDB_DICTIONARY_TOO_OLD; //Cannot use + r = TOKUDB_DICTIONARY_TOO_OLD; // Cannot use goto exit; } else if (version > FT_LAYOUT_VERSION) { - r = TOKUDB_DICTIONARY_TOO_NEW; //Cannot use + r = TOKUDB_DICTIONARY_TOO_NEW; // Cannot use goto exit; } - //build_id MUST be in network order regardless of disk order. + // build_id MUST be in network order regardless of disk order. uint32_t build_id __attribute__((__unused__)); build_id = rbuf_network_int(rb); int64_t min_header_size; min_header_size = serialize_ft_min_size(version); - //Size MUST be in network order regardless of disk order. + // Size MUST be in network order regardless of disk order. uint32_t size; size = rbuf_network_int(rb); - //If too big, it is corrupt. We would probably notice during checksum - //but may have to do a multi-gigabyte malloc+read to find out. - //If its too small reading rbuf would crash, so verify. - if (size > block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE || size < min_header_size) { + // If too big, it is corrupt. We would probably notice during checksum + // but may have to do a multi-gigabyte malloc+read to find out. + // If its too small reading rbuf would crash, so verify. + if (size > BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE || + size < min_header_size) { r = TOKUDB_DICTIONARY_NO_HEADER; goto exit; } - lazy_assert(rb->ndone==prefix_size); + lazy_assert(rb->ndone == prefix_size); rb->size = size; { toku_free(rb->buf); uint32_t size_to_read = roundup_to_multiple(512, size); XMALLOC_N_ALIGNED(512, size_to_read, rb->buf); - assert(offset_of_header%512==0); + invariant(offset_of_header % 512 == 0); n = toku_os_pread(fd, rb->buf, size_to_read, offset_of_header); if (n != size_to_read) { if (n < 0) { r = get_error_errno(); } else { - r = EINVAL; //Header might be useless (wrong size) or could be a disk read error. + r = EINVAL; // Header might be useless (wrong size) or could be + // a disk read error. } goto exit; } } - //It's version 14 or later. Magic looks OK. - //We have an rbuf that represents the header. - //Size is within acceptable bounds. + // It's version 14 or later. Magic looks OK. + // We have an rbuf that represents the header. + // Size is within acceptable bounds. - //Verify checksum (FT_LAYOUT_VERSION_13 or later, when checksum function changed) + // Verify checksum (FT_LAYOUT_VERSION_13 or later, when checksum function + // changed) uint32_t calculated_x1764; - calculated_x1764 = toku_x1764_memory(rb->buf, rb->size-4); + calculated_x1764 = toku_x1764_memory(rb->buf, rb->size - 4); uint32_t stored_x1764; - stored_x1764 = toku_dtoh32(*(int*)(rb->buf+rb->size-4)); + stored_x1764 = toku_dtoh32(*(int *)(rb->buf + rb->size - 4)); if (calculated_x1764 != stored_x1764) { - r = TOKUDB_BAD_CHECKSUM; //Header useless - fprintf(stderr, "Header checksum failure: calc=0x%08x read=0x%08x\n", calculated_x1764, stored_x1764); + r = TOKUDB_BAD_CHECKSUM; // Header useless + fprintf(stderr, + "Header checksum failure: calc=0x%08x read=0x%08x\n", + calculated_x1764, + stored_x1764); goto exit; } - //Verify byte order + // Verify byte order const void *tmp_byte_order_check; lazy_assert((sizeof toku_byte_order_host) == 8); - rbuf_literal_bytes(rb, &tmp_byte_order_check, 8); //Must not translate byte order + rbuf_literal_bytes( + rb, &tmp_byte_order_check, 8); // Must not translate byte order int64_t byte_order_stored; - byte_order_stored = *(int64_t*)tmp_byte_order_check; + byte_order_stored = *(int64_t *)tmp_byte_order_check; if (byte_order_stored != toku_byte_order_host) { - r = TOKUDB_DICTIONARY_NO_HEADER; //Cannot use dictionary + r = TOKUDB_DICTIONARY_NO_HEADER; // Cannot use dictionary goto exit; } - //Load checkpoint count + // Load checkpoint count *checkpoint_count = rbuf_ulonglong(rb); *checkpoint_lsn = rbuf_LSN(rb); - //Restart at beginning during regular deserialization + // Restart at beginning during regular deserialization rb->ndone = 0; exit: @@ -620,11 +644,7 @@ int deserialize_ft_from_fd_into_rbuf(int fd, // Read ft from file into struct. Read both headers and use one. // We want the latest acceptable header whose checkpoint_lsn is no later // than max_acceptable_lsn. -int -toku_deserialize_ft_from(int fd, - LSN max_acceptable_lsn, - FT *ft) -{ +int toku_deserialize_ft_from(int fd, LSN max_acceptable_lsn, FT *ft) { struct rbuf rb_0; struct rbuf rb_1; uint64_t checkpoint_count_0 = 0; @@ -638,13 +658,23 @@ toku_deserialize_ft_from(int fd, int r0, r1, r; toku_off_t header_0_off = 0; - r0 = deserialize_ft_from_fd_into_rbuf(fd, header_0_off, &rb_0, &checkpoint_count_0, &checkpoint_lsn_0, &version_0); + r0 = deserialize_ft_from_fd_into_rbuf(fd, + header_0_off, + &rb_0, + &checkpoint_count_0, + &checkpoint_lsn_0, + &version_0); if (r0 == 0 && checkpoint_lsn_0.lsn <= max_acceptable_lsn.lsn) { h0_acceptable = true; } - toku_off_t header_1_off = block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE; - r1 = deserialize_ft_from_fd_into_rbuf(fd, header_1_off, &rb_1, &checkpoint_count_1, &checkpoint_lsn_1, &version_1); + toku_off_t header_1_off = BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE; + r1 = deserialize_ft_from_fd_into_rbuf(fd, + header_1_off, + &rb_1, + &checkpoint_count_1, + &checkpoint_lsn_1, + &version_1); if (r1 == 0 && checkpoint_lsn_1.lsn <= max_acceptable_lsn.lsn) { h1_acceptable = true; } @@ -655,24 +685,29 @@ toku_deserialize_ft_from(int fd, // We were unable to read either header or at least one is too // new. Certain errors are higher priority than others. Order of // these if/else if is important. - if (r0 == TOKUDB_DICTIONARY_TOO_NEW || r1 == TOKUDB_DICTIONARY_TOO_NEW) { + if (r0 == TOKUDB_DICTIONARY_TOO_NEW || + r1 == TOKUDB_DICTIONARY_TOO_NEW) { r = TOKUDB_DICTIONARY_TOO_NEW; - } else if (r0 == TOKUDB_DICTIONARY_TOO_OLD || r1 == TOKUDB_DICTIONARY_TOO_OLD) { + } else if (r0 == TOKUDB_DICTIONARY_TOO_OLD || + r1 == TOKUDB_DICTIONARY_TOO_OLD) { r = TOKUDB_DICTIONARY_TOO_OLD; } else if (r0 == TOKUDB_BAD_CHECKSUM && r1 == TOKUDB_BAD_CHECKSUM) { fprintf(stderr, "Both header checksums failed.\n"); r = TOKUDB_BAD_CHECKSUM; - } else if (r0 == TOKUDB_DICTIONARY_NO_HEADER || r1 == TOKUDB_DICTIONARY_NO_HEADER) { + } else if (r0 == TOKUDB_DICTIONARY_NO_HEADER || + r1 == TOKUDB_DICTIONARY_NO_HEADER) { r = TOKUDB_DICTIONARY_NO_HEADER; } else { - r = r0 ? r0 : r1; //Arbitrarily report the error from the - //first header, unless it's readable + r = r0 ? r0 : r1; // Arbitrarily report the error from the + // first header, unless it's readable } - // it should not be possible for both headers to be later than the max_acceptable_lsn - invariant(!((r0==0 && checkpoint_lsn_0.lsn > max_acceptable_lsn.lsn) && - (r1==0 && checkpoint_lsn_1.lsn > max_acceptable_lsn.lsn))); - invariant(r!=0); + // it should not be possible for both headers to be later than the + // max_acceptable_lsn + invariant( + !((r0 == 0 && checkpoint_lsn_0.lsn > max_acceptable_lsn.lsn) && + (r1 == 0 && checkpoint_lsn_1.lsn > max_acceptable_lsn.lsn))); + invariant(r != 0); goto exit; } @@ -682,8 +717,7 @@ toku_deserialize_ft_from(int fd, invariant(version_0 >= version_1); rb = &rb_0; version = version_0; - } - else { + } else { invariant(checkpoint_count_1 == checkpoint_count_0 + 1); invariant(version_1 >= version_0); rb = &rb_1; @@ -692,14 +726,18 @@ toku_deserialize_ft_from(int fd, } else if (h0_acceptable) { if (r1 == TOKUDB_BAD_CHECKSUM) { // print something reassuring - fprintf(stderr, "Header 2 checksum failed, but header 1 ok. Proceeding.\n"); + fprintf( + stderr, + "Header 2 checksum failed, but header 1 ok. Proceeding.\n"); } rb = &rb_0; version = version_0; } else if (h1_acceptable) { if (r0 == TOKUDB_BAD_CHECKSUM) { // print something reassuring - fprintf(stderr, "Header 1 checksum failed, but header 2 ok. Proceeding.\n"); + fprintf( + stderr, + "Header 1 checksum failed, but header 2 ok. Proceeding.\n"); } rb = &rb_1; version = version_1; @@ -718,15 +756,13 @@ toku_deserialize_ft_from(int fd, return r; } - -size_t toku_serialize_ft_size (FT_HEADER h) { +size_t toku_serialize_ft_size(FT_HEADER h) { size_t size = serialize_ft_min_size(h->layout_version); - //There is no dynamic data. - lazy_assert(size <= block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE); + // There is no dynamic data. + lazy_assert(size <= BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE); return size; } - void toku_serialize_ft_to_wbuf ( struct wbuf *wbuf, FT_HEADER h, @@ -771,52 +807,60 @@ void toku_serialize_ft_to_wbuf ( } void toku_serialize_ft_to(int fd, FT_HEADER h, block_table *bt, CACHEFILE cf) { - lazy_assert(h->type==FT_CHECKPOINT_INPROGRESS); + lazy_assert(h->type == FT_CHECKPOINT_INPROGRESS); struct wbuf w_translation; int64_t size_translation; int64_t address_translation; // Must serialize translation first, to get address,size for header. - bt->serialize_translation_to_wbuf(fd, &w_translation, - &address_translation, - &size_translation); - assert(size_translation == w_translation.ndone); + bt->serialize_translation_to_wbuf( + fd, &w_translation, &address_translation, &size_translation); + invariant(size_translation == w_translation.ndone); - // the number of bytes available in the buffer is 0 mod 512, and those last bytes are all initialized. - assert(w_translation.size % 512 == 0); + // the number of bytes available in the buffer is 0 mod 512, and those last + // bytes are all initialized. + invariant(w_translation.size % 512 == 0); struct wbuf w_main; - size_t size_main = toku_serialize_ft_size(h); + size_t size_main = toku_serialize_ft_size(h); size_t size_main_aligned = roundup_to_multiple(512, size_main); - assert(size_main_alignedcheckpoint_count & 0x1) ? 0 : block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE; + main_offset = (h->checkpoint_count & 0x1) + ? 0 + : BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE; toku_os_full_pwrite(fd, w_main.buf, size_main_aligned, main_offset); toku_free(w_main.buf); toku_free(w_translation.buf); diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc index c4f4886b6a03b..5914f8a1050e9 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc @@ -99,13 +99,11 @@ void toku_ft_serialize_layer_init(void) { num_cores = toku_os_get_number_active_processors(); int r = toku_thread_pool_create(&ft_pool, num_cores); lazy_assert_zero(r); - block_allocator::maybe_initialize_trace(); toku_serialize_in_parallel = false; } void toku_ft_serialize_layer_destroy(void) { toku_thread_pool_destroy(&ft_pool); - block_allocator::maybe_close_trace(); } enum { FILE_CHANGE_INCREMENT = (16 << 20) }; @@ -773,19 +771,23 @@ int toku_serialize_ftnode_to_memory(FTNODE node, return 0; } -int -toku_serialize_ftnode_to (int fd, BLOCKNUM blocknum, FTNODE node, FTNODE_DISK_DATA* ndd, bool do_rebalancing, FT ft, bool for_checkpoint) { - +int toku_serialize_ftnode_to(int fd, + BLOCKNUM blocknum, + FTNODE node, + FTNODE_DISK_DATA *ndd, + bool do_rebalancing, + FT ft, + bool for_checkpoint) { size_t n_to_write; size_t n_uncompressed_bytes; char *compressed_buf = nullptr; - // because toku_serialize_ftnode_to is only called for + // because toku_serialize_ftnode_to is only called for // in toku_ftnode_flush_callback, we pass false // for in_parallel. The reasoning is that when we write - // nodes to disk via toku_ftnode_flush_callback, we + // nodes to disk via toku_ftnode_flush_callback, we // assume that it is being done on a non-critical - // background thread (probably for checkpointing), and therefore + // background thread (probably for checkpointing), and therefore // should not hog CPU, // // Should the above facts change, we may want to revisit @@ -802,32 +804,32 @@ toku_serialize_ftnode_to (int fd, BLOCKNUM blocknum, FTNODE node, FTNODE_DISK_DA toku_unsafe_fetch(&toku_serialize_in_parallel), &n_to_write, &n_uncompressed_bytes, - &compressed_buf - ); + &compressed_buf); if (r != 0) { return r; } - // If the node has never been written, then write the whole buffer, including the zeros - invariant(blocknum.b>=0); + // If the node has never been written, then write the whole buffer, + // including the zeros + invariant(blocknum.b >= 0); DISKOFF offset; // Dirties the ft - ft->blocktable.realloc_on_disk(blocknum, n_to_write, &offset, - ft, fd, for_checkpoint, - // Allocations for nodes high in the tree are considered 'hot', - // as they are likely to move again in the next checkpoint. - node->height); + ft->blocktable.realloc_on_disk( + blocknum, n_to_write, &offset, ft, fd, for_checkpoint); tokutime_t t0 = toku_time_now(); toku_os_full_pwrite(fd, compressed_buf, n_to_write, offset); tokutime_t t1 = toku_time_now(); tokutime_t io_time = t1 - t0; - toku_ft_status_update_flush_reason(node, n_uncompressed_bytes, n_to_write, io_time, for_checkpoint); + toku_ft_status_update_flush_reason( + node, n_uncompressed_bytes, n_to_write, io_time, for_checkpoint); toku_free(compressed_buf); - node->dirty = 0; // See #1957. Must set the node to be clean after serializing it so that it doesn't get written again on the next checkpoint or eviction. + node->dirty = 0; // See #1957. Must set the node to be clean after + // serializing it so that it doesn't get written again on + // the next checkpoint or eviction. return 0; } @@ -994,6 +996,7 @@ BASEMENTNODE toku_clone_bn(BASEMENTNODE orig_bn) { bn->seqinsert = orig_bn->seqinsert; bn->stale_ancestor_messages_applied = orig_bn->stale_ancestor_messages_applied; bn->stat64_delta = orig_bn->stat64_delta; + bn->logical_rows_delta = orig_bn->logical_rows_delta; bn->data_buffer.clone(&orig_bn->data_buffer); return bn; } @@ -1004,6 +1007,7 @@ BASEMENTNODE toku_create_empty_bn_no_buffer(void) { bn->seqinsert = 0; bn->stale_ancestor_messages_applied = false; bn->stat64_delta = ZEROSTATS; + bn->logical_rows_delta = 0; bn->data_buffer.init_zero(); return bn; } @@ -1897,7 +1901,7 @@ read_and_decompress_block_from_fd_into_rbuf(int fd, BLOCKNUM blocknum, /* out */ int *layout_version_p); // This function upgrades a version 14 or 13 ftnode to the current -// verison. NOTE: This code assumes the first field of the rbuf has +// version. NOTE: This code assumes the first field of the rbuf has // already been read from the buffer (namely the layout_version of the // ftnode.) static int @@ -2488,9 +2492,12 @@ toku_serialize_rollback_log_to_memory_uncompressed(ROLLBACK_LOG_NODE log, SERIAL serialized->blocknum = log->blocknum; } -int -toku_serialize_rollback_log_to (int fd, ROLLBACK_LOG_NODE log, SERIALIZED_ROLLBACK_LOG_NODE serialized_log, bool is_serialized, - FT ft, bool for_checkpoint) { +int toku_serialize_rollback_log_to(int fd, + ROLLBACK_LOG_NODE log, + SERIALIZED_ROLLBACK_LOG_NODE serialized_log, + bool is_serialized, + FT ft, + bool for_checkpoint) { size_t n_to_write; char *compressed_buf; struct serialized_rollback_log_node serialized_local; @@ -2511,21 +2518,21 @@ toku_serialize_rollback_log_to (int fd, ROLLBACK_LOG_NODE log, SERIALIZED_ROLLBA serialized_log->n_sub_blocks, serialized_log->sub_block, ft->h->compression_method, - &n_to_write, &compressed_buf); + &n_to_write, + &compressed_buf); // Dirties the ft DISKOFF offset; - ft->blocktable.realloc_on_disk(blocknum, n_to_write, &offset, - ft, fd, for_checkpoint, - // We consider rollback log flushing the hottest possible allocation, - // since rollback logs are short-lived compared to FT nodes. - INT_MAX); + ft->blocktable.realloc_on_disk( + blocknum, n_to_write, &offset, ft, fd, for_checkpoint); toku_os_full_pwrite(fd, compressed_buf, n_to_write, offset); toku_free(compressed_buf); if (!is_serialized) { toku_static_serialized_rollback_log_destroy(&serialized_local); - log->dirty = 0; // See #1957. Must set the node to be clean after serializing it so that it doesn't get written again on the next checkpoint or eviction. + log->dirty = 0; // See #1957. Must set the node to be clean after + // serializing it so that it doesn't get written again + // on the next checkpoint or eviction. } return 0; } @@ -2704,7 +2711,7 @@ decompress_from_raw_block_into_rbuf(uint8_t *raw_block, size_t raw_block_size, s } static int decompress_from_raw_block_into_rbuf_versioned(uint32_t version, uint8_t *raw_block, size_t raw_block_size, struct rbuf *rb, BLOCKNUM blocknum) { - // This function exists solely to accomodate future changes in compression. + // This function exists solely to accommodate future changes in compression. int r = 0; if ((version == FT_LAYOUT_VERSION_13 || version == FT_LAYOUT_VERSION_14) || (FT_LAYOUT_VERSION_25 <= version && version <= FT_LAYOUT_VERSION_27) || diff --git a/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.cc b/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.cc new file mode 100644 index 0000000000000..922850fb3e096 --- /dev/null +++ b/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.cc @@ -0,0 +1,833 @@ +/*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILIT or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see . + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see . +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "ft/serialize/rbtree_mhs.h" +#include "portability/toku_assert.h" +#include "portability/toku_portability.h" +#include + +namespace MhsRbTree { + + Tree::Tree() : _root(NULL), _align(1) {} + + Tree::Tree(uint64_t align) : _root(NULL), _align(align) {} + + Tree::~Tree() { Destroy(); } + + void Tree::PreOrder(Node *tree) const { + if (tree != NULL) { + fprintf(stderr, "%" PRIu64 " ", rbn_offset(tree).ToInt()); + PreOrder(tree->_left); + PreOrder(tree->_right); + } + } + + void Tree::PreOrder() { PreOrder(_root); } + + void Tree::InOrder(Node *tree) const { + if (tree != NULL) { + InOrder(tree->_left); + fprintf(stderr, "%" PRIu64 " ", rbn_offset(tree).ToInt()); + InOrder(tree->_right); + } + } + + // yeah, i only care about in order visitor. -Jun + void Tree::InOrderVisitor(Node *tree, + void (*f)(void *, Node *, uint64_t), + void *extra, + uint64_t depth) { + if (tree != NULL) { + InOrderVisitor(tree->_left, f, extra, depth + 1); + f(extra, tree, depth); + InOrderVisitor(tree->_right, f, extra, depth + 1); + } + } + + void Tree::InOrderVisitor(void (*f)(void *, Node *, uint64_t), + void *extra) { + InOrderVisitor(_root, f, extra, 0); + } + + void Tree::InOrder() { InOrder(_root); } + + void Tree::PostOrder(Node *tree) const { + if (tree != NULL) { + PostOrder(tree->_left); + PostOrder(tree->_right); + fprintf(stderr, "%" PRIu64 " ", rbn_offset(tree).ToInt()); + } + } + + void Tree::PostOrder() { PostOrder(_root); } + + Node *Tree::SearchByOffset(uint64_t offset) { + Node *x = _root; + while ((x != NULL) && (rbn_offset(x).ToInt() != offset)) { + if (offset < rbn_offset(x).ToInt()) + x = x->_left; + else + x = x->_right; + } + + return x; + } + + // mostly for testing + Node *Tree::SearchFirstFitBySize(uint64_t size) { + if (EffectiveSize(_root) < size && rbn_left_mhs(_root) < size && + rbn_right_mhs(_root) < size) { + return nullptr; + } else { + return SearchFirstFitBySizeHelper(_root, size); + } + } + + Node *Tree::SearchFirstFitBySizeHelper(Node *x, uint64_t size) { + if (EffectiveSize(x) >= size) { + // only possible to go left + if (rbn_left_mhs(x) >= size) + return SearchFirstFitBySizeHelper(x->_left, size); + else + return x; + } + if (rbn_left_mhs(x) >= size) + return SearchFirstFitBySizeHelper(x->_left, size); + + if (rbn_right_mhs(x) >= size) + return SearchFirstFitBySizeHelper(x->_right, size); + + // this is an invalid state + Dump(); + ValidateBalance(); + ValidateMhs(); + invariant(0); + return NULL; + } + + Node *Tree::MinNode(Node *tree) { + if (tree == NULL) + return NULL; + + while (tree->_left != NULL) + tree = tree->_left; + return tree; + } + + Node *Tree::MinNode() { return MinNode(_root); } + + Node *Tree::MaxNode(Node *tree) { + if (tree == NULL) + return NULL; + + while (tree->_right != NULL) + tree = tree->_right; + return tree; + } + + Node *Tree::MaxNode() { return MaxNode(_root); } + + Node *Tree::SuccessorHelper(Node *y, Node *x) { + while ((y != NULL) && (x == y->_right)) { + x = y; + y = y->_parent; + } + return y; + } + Node *Tree::Successor(Node *x) { + if (x->_right != NULL) + return MinNode(x->_right); + + Node *y = x->_parent; + return SuccessorHelper(y, x); + } + + Node *Tree::PredecessorHelper(Node *y, Node *x) { + while ((y != NULL) && (x == y->_left)) { + x = y; + y = y->_parent; + } + + return y; + } + Node *Tree::Predecessor(Node *x) { + if (x->_left != NULL) + return MaxNode(x->_left); + + Node *y = x->_parent; + return SuccessorHelper(y, x); + } + + /* + * px px + * / / + * x y + * / \ --(left rotation)--> / \ # + * lx y x ry + * / \ / \ + * ly ry lx ly + * max_hole_size updates are pretty local + */ + + void Tree::LeftRotate(Node *&root, Node *x) { + Node *y = x->_right; + + x->_right = y->_left; + rbn_right_mhs(x) = rbn_left_mhs(y); + + if (y->_left != NULL) + y->_left->_parent = x; + + y->_parent = x->_parent; + + if (x->_parent == NULL) { + root = y; + } else { + if (x->_parent->_left == x) { + x->_parent->_left = y; + } else { + x->_parent->_right = y; + } + } + y->_left = x; + rbn_left_mhs(y) = mhs_of_subtree(x); + + x->_parent = y; + } + + /* py py + * / / + * y x + * / \ --(right rotate)--> / \ # + * x ry lx y + * / \ / \ # + * lx rx rx ry + * + */ + + void Tree::RightRotate(Node *&root, Node *y) { + Node *x = y->_left; + + y->_left = x->_right; + rbn_left_mhs(y) = rbn_right_mhs(x); + + if (x->_right != NULL) + x->_right->_parent = y; + + x->_parent = y->_parent; + + if (y->_parent == NULL) { + root = x; + } else { + if (y == y->_parent->_right) + y->_parent->_right = x; + else + y->_parent->_left = x; + } + + x->_right = y; + rbn_right_mhs(x) = mhs_of_subtree(y); + y->_parent = x; + } + + // walking from this node up to update the mhs info + // whenver there is change on left/right mhs or size we should recalculate. + // prerequisit: the children of the node are mhs up-to-date. + void Tree::RecalculateMhs(Node *node) { + uint64_t *p_node_mhs = 0; + Node *parent = node->_parent; + + if (!parent) + return; + + uint64_t max_mhs = mhs_of_subtree(node); + if (node == parent->_left) { + p_node_mhs = &rbn_left_mhs(parent); + } else if (node == parent->_right) { + p_node_mhs = &rbn_right_mhs(parent); + } else { + return; + } + if (*p_node_mhs != max_mhs) { + *p_node_mhs = max_mhs; + RecalculateMhs(parent); + } + } + + void Tree::IsNewNodeMergable(Node *pred, + Node *succ, + Node::BlockPair pair, + bool *left_merge, + bool *right_merge) { + if (pred) { + OUUInt64 end_of_pred = rbn_size(pred) + rbn_offset(pred); + if (end_of_pred < pair._offset) + *left_merge = false; + else { + invariant(end_of_pred == pair._offset); + *left_merge = true; + } + } + if (succ) { + OUUInt64 begin_of_succ = rbn_offset(succ); + OUUInt64 end_of_node = pair._offset + pair._size; + if (end_of_node < begin_of_succ) { + *right_merge = false; + } else { + invariant(end_of_node == begin_of_succ); + *right_merge = true; + } + } + } + + void Tree::AbsorbNewNode(Node *pred, + Node *succ, + Node::BlockPair pair, + bool left_merge, + bool right_merge, + bool is_right_child) { + invariant(left_merge || right_merge); + if (left_merge && right_merge) { + // merge to the succ + if (!is_right_child) { + rbn_size(succ) += pair._size; + rbn_offset(succ) = pair._offset; + // merge to the pred + rbn_size(pred) += rbn_size(succ); + // to keep the invariant of the tree -no overlapping holes + rbn_offset(succ) += rbn_size(succ); + rbn_size(succ) = 0; + RecalculateMhs(succ); + RecalculateMhs(pred); + // pred dominates succ. this is going to + // update the pred labels separately. + // remove succ + RawRemove(_root, succ); + } else { + rbn_size(pred) += pair._size; + rbn_offset(succ) = rbn_offset(pred); + rbn_size(succ) += rbn_size(pred); + rbn_offset(pred) += rbn_size(pred); + rbn_size(pred) = 0; + RecalculateMhs(pred); + RecalculateMhs(succ); + // now remove pred + RawRemove(_root, pred); + } + } else if (left_merge) { + rbn_size(pred) += pair._size; + RecalculateMhs(pred); + } else if (right_merge) { + rbn_offset(succ) -= pair._size; + rbn_size(succ) += pair._size; + RecalculateMhs(succ); + } + } + // this is the most tedious part, but not complicated: + // 1.find where to insert the pair + // 2.if the pred and succ can merge with the pair. merge with them. either + // pred + // or succ can be removed. + // 3. if only left-mergable or right-mergeable, just merge + // 4. non-mergable case. insert the node and run the fixup. + + int Tree::Insert(Node *&root, Node::BlockPair pair) { + Node *x = _root; + Node *y = NULL; + bool left_merge = false; + bool right_merge = false; + Node *node = NULL; + + while (x != NULL) { + y = x; + if (pair._offset < rbn_key(x)) + x = x->_left; + else + x = x->_right; + } + + // we found where to insert, lets find out the pred and succ for + // possible + // merges. + // node->parent = y; + Node *pred, *succ; + if (y != NULL) { + if (pair._offset < rbn_key(y)) { + // as the left child + pred = PredecessorHelper(y->_parent, y); + succ = y; + IsNewNodeMergable(pred, succ, pair, &left_merge, &right_merge); + if (left_merge || right_merge) { + AbsorbNewNode( + pred, succ, pair, left_merge, right_merge, false); + } else { + // construct the node + Node::Pair mhsp {0, 0}; + node = + new Node(EColor::BLACK, pair, mhsp, nullptr, nullptr, nullptr); + if (!node) + return -1; + y->_left = node; + node->_parent = y; + RecalculateMhs(node); + } + + } else { + // as the right child + pred = y; + succ = SuccessorHelper(y->_parent, y); + IsNewNodeMergable(pred, succ, pair, &left_merge, &right_merge); + if (left_merge || right_merge) { + AbsorbNewNode( + pred, succ, pair, left_merge, right_merge, true); + } else { + // construct the node + Node::Pair mhsp {0, 0}; + node = + new Node(EColor::BLACK, pair, mhsp, nullptr, nullptr, nullptr); + if (!node) + return -1; + y->_right = node; + node->_parent = y; + RecalculateMhs(node); + } + } + } else { + Node::Pair mhsp {0, 0}; + node = new Node(EColor::BLACK, pair, mhsp, nullptr, nullptr, nullptr); + if (!node) + return -1; + root = node; + } + if (!left_merge && !right_merge) { + invariant_notnull(node); + node->_color = EColor::RED; + return InsertFixup(root, node); + } + return 0; + } + + int Tree::InsertFixup(Node *&root, Node *node) { + Node *parent, *gparent; + while ((parent = rbn_parent(node)) && rbn_is_red(parent)) { + gparent = rbn_parent(parent); + if (parent == gparent->_left) { + { + Node *uncle = gparent->_right; + if (uncle && rbn_is_red(uncle)) { + rbn_set_black(uncle); + rbn_set_black(parent); + rbn_set_red(gparent); + node = gparent; + continue; + } + } + + if (parent->_right == node) { + Node *tmp; + LeftRotate(root, parent); + tmp = parent; + parent = node; + node = tmp; + } + + rbn_set_black(parent); + rbn_set_red(gparent); + RightRotate(root, gparent); + } else { + { + Node *uncle = gparent->_left; + if (uncle && rbn_is_red(uncle)) { + rbn_set_black(uncle); + rbn_set_black(parent); + rbn_set_red(gparent); + node = gparent; + continue; + } + } + + if (parent->_left == node) { + Node *tmp; + RightRotate(root, parent); + tmp = parent; + parent = node; + node = tmp; + } + rbn_set_black(parent); + rbn_set_red(gparent); + LeftRotate(root, gparent); + } + } + rbn_set_black(root); + return 0; + } + + int Tree::Insert(Node::BlockPair pair) { return Insert(_root, pair); } + + uint64_t Tree::Remove(size_t size) { + Node *node = SearchFirstFitBySize(size); + return Remove(_root, node, size); + } + + void Tree::RawRemove(Node *&root, Node *node) { + Node *child, *parent; + EColor color; + + if ((node->_left != NULL) && (node->_right != NULL)) { + Node *replace = node; + replace = replace->_right; + while (replace->_left != NULL) + replace = replace->_left; + + if (rbn_parent(node)) { + if (rbn_parent(node)->_left == node) + rbn_parent(node)->_left = replace; + else + rbn_parent(node)->_right = replace; + } else { + root = replace; + } + child = replace->_right; + parent = rbn_parent(replace); + color = rbn_color(replace); + + if (parent == node) { + parent = replace; + } else { + if (child) + rbn_parent(child) = parent; + + parent->_left = child; + rbn_left_mhs(parent) = rbn_right_mhs(replace); + RecalculateMhs(parent); + replace->_right = node->_right; + rbn_set_parent(node->_right, replace); + rbn_right_mhs(replace) = rbn_right_mhs(node); + } + + replace->_parent = node->_parent; + replace->_color = node->_color; + replace->_left = node->_left; + rbn_left_mhs(replace) = rbn_left_mhs(node); + node->_left->_parent = replace; + RecalculateMhs(replace); + if (color == EColor::BLACK) + RawRemoveFixup(root, child, parent); + delete node; + return; + } + + if (node->_left != NULL) + child = node->_left; + else + child = node->_right; + + parent = node->_parent; + color = node->_color; + + if (child) + child->_parent = parent; + + if (parent) { + if (parent->_left == node) { + parent->_left = child; + rbn_left_mhs(parent) = child ? mhs_of_subtree(child) : 0; + } else { + parent->_right = child; + rbn_right_mhs(parent) = child ? mhs_of_subtree(child) : 0; + } + RecalculateMhs(parent); + } else + root = child; + if (color == EColor::BLACK) + RawRemoveFixup(root, child, parent); + delete node; + } + + void Tree::RawRemove(uint64_t offset) { + Node *node = SearchByOffset(offset); + RawRemove(_root, node); + } + static inline uint64_t align(uint64_t value, uint64_t ba_alignment) { + return ((value + ba_alignment - 1) / ba_alignment) * ba_alignment; + } + uint64_t Tree::Remove(Node *&root, Node *node, size_t size) { + OUUInt64 n_offset = rbn_offset(node); + OUUInt64 n_size = rbn_size(node); + OUUInt64 answer_offset(align(rbn_offset(node).ToInt(), _align)); + + invariant((answer_offset + size) <= (n_offset + n_size)); + if (answer_offset == n_offset) { + rbn_offset(node) += size; + rbn_size(node) -= size; + RecalculateMhs(node); + if (rbn_size(node) == 0) { + RawRemove(root, node); + } + + } else { + if (answer_offset + size == n_offset + n_size) { + rbn_size(node) -= size; + RecalculateMhs(node); + } else { + // well, cut in the middle... + rbn_size(node) = answer_offset - n_offset; + RecalculateMhs(node); + Insert(_root, + {(answer_offset + size), + (n_offset + n_size) - (answer_offset + size)}); + } + } + return answer_offset.ToInt(); + } + + void Tree::RawRemoveFixup(Node *&root, Node *node, Node *parent) { + Node *other; + while ((!node || rbn_is_black(node)) && node != root) { + if (parent->_left == node) { + other = parent->_right; + if (rbn_is_red(other)) { + // Case 1: the brother of X, w, is read + rbn_set_black(other); + rbn_set_red(parent); + LeftRotate(root, parent); + other = parent->_right; + } + if ((!other->_left || rbn_is_black(other->_left)) && + (!other->_right || rbn_is_black(other->_right))) { + // Case 2: w is black and both of w's children are black + rbn_set_red(other); + node = parent; + parent = rbn_parent(node); + } else { + if (!other->_right || rbn_is_black(other->_right)) { + // Case 3: w is black and left child of w is red but + // right + // child is black + rbn_set_black(other->_left); + rbn_set_red(other); + RightRotate(root, other); + other = parent->_right; + } + // Case 4: w is black and right child of w is red, + // regardless of + // left child's color + rbn_set_color(other, rbn_color(parent)); + rbn_set_black(parent); + rbn_set_black(other->_right); + LeftRotate(root, parent); + node = root; + break; + } + } else { + other = parent->_left; + if (rbn_is_red(other)) { + // Case 1: w is red + rbn_set_black(other); + rbn_set_red(parent); + RightRotate(root, parent); + other = parent->_left; + } + if ((!other->_left || rbn_is_black(other->_left)) && + (!other->_right || rbn_is_black(other->_right))) { + // Case 2: w is black and both children are black + rbn_set_red(other); + node = parent; + parent = rbn_parent(node); + } else { + if (!other->_left || rbn_is_black(other->_left)) { + // Case 3: w is black and left child of w is red whereas + // right child is black + rbn_set_black(other->_right); + rbn_set_red(other); + LeftRotate(root, other); + other = parent->_left; + } + // Case 4:w is black and right child of w is red, regardless + // of + // the left child's color + rbn_set_color(other, rbn_color(parent)); + rbn_set_black(parent); + rbn_set_black(other->_left); + RightRotate(root, parent); + node = root; + break; + } + } + } + if (node) + rbn_set_black(node); + } + + void Tree::Destroy(Node *&tree) { + if (tree == NULL) + return; + + if (tree->_left != NULL) + Destroy(tree->_left); + if (tree->_right != NULL) + Destroy(tree->_right); + + delete tree; + tree = NULL; + } + + void Tree::Destroy() { Destroy(_root); } + + void Tree::Dump(Node *tree, Node::BlockPair pair, EDirection dir) { + if (tree != NULL) { + if (dir == EDirection::NONE) + fprintf(stderr, + "(%" PRIu64 ",%" PRIu64 ", mhs:(%" PRIu64 ",%" PRIu64 + "))(B) is root\n", + rbn_offset(tree).ToInt(), + rbn_size(tree).ToInt(), + rbn_left_mhs(tree), + rbn_right_mhs(tree)); + else + fprintf(stderr, + "(%" PRIu64 ",%" PRIu64 ",mhs:(%" PRIu64 ",%" PRIu64 + "))(%c) is %" PRIu64 "'s %s\n", + rbn_offset(tree).ToInt(), + rbn_size(tree).ToInt(), + rbn_left_mhs(tree), + rbn_right_mhs(tree), + rbn_is_red(tree) ? 'R' : 'B', + pair._offset.ToInt(), + dir == EDirection::RIGHT ? "right child" : "left child"); + + Dump(tree->_left, tree->_hole, EDirection::LEFT); + Dump(tree->_right, tree->_hole, EDirection::RIGHT); + } + } + + uint64_t Tree::EffectiveSize(Node *node) { + OUUInt64 offset = rbn_offset(node); + OUUInt64 size = rbn_size(node); + OUUInt64 end = offset + size; + OUUInt64 aligned_offset(align(offset.ToInt(), _align)); + if (aligned_offset > end) { + return 0; + } + return (end - aligned_offset).ToInt(); + } + + void Tree::Dump() { + if (_root != NULL) + Dump(_root, _root->_hole, (EDirection)0); + } + + static void vis_bal_f(void *extra, Node *node, uint64_t depth) { + uint64_t **p = (uint64_t **)extra; + uint64_t min = *p[0]; + uint64_t max = *p[1]; + if (node->_left) { + Node *left = node->_left; + invariant(node == left->_parent); + } + + if (node->_right) { + Node *right = node->_right; + invariant(node == right->_parent); + } + + if (!node->_left || !node->_right) { + if (min > depth) { + *p[0] = depth; + } else if (max < depth) { + *p[1] = depth; + } + } + } + + void Tree::ValidateBalance() { + uint64_t min_depth = 0xffffffffffffffff; + uint64_t max_depth = 0; + if (!_root) { + return; + } + uint64_t *p[2] = {&min_depth, &max_depth}; + InOrderVisitor(vis_bal_f, (void *)p); + invariant((min_depth + 1) * 2 >= max_depth + 1); + } + + static void vis_cmp_f(void *extra, Node *node, uint64_t UU(depth)) { + Node::BlockPair **p = (Node::BlockPair **)extra; + + invariant_notnull(*p); + invariant((*p)->_offset == node->_hole._offset); + + *p = *p + 1; + } + + // validate the input pairs matches with sorted pairs + void Tree::ValidateInOrder(Node::BlockPair *pairs) { + InOrderVisitor(vis_cmp_f, &pairs); + } + + uint64_t Tree::ValidateMhs(Node *node) { + if (!node) + return 0; + else { + uint64_t mhs_left = ValidateMhs(node->_left); + uint64_t mhs_right = ValidateMhs(node->_right); + if (mhs_left != rbn_left_mhs(node)) { + printf("assert failure: mhs_left = %" PRIu64 "\n", mhs_left); + Dump(node, node->_hole, (EDirection)0); + } + invariant(mhs_left == rbn_left_mhs(node)); + + if (mhs_right != rbn_right_mhs(node)) { + printf("assert failure: mhs_right = %" PRIu64 "\n", mhs_right); + Dump(node, node->_hole, (EDirection)0); + } + invariant(mhs_right == rbn_right_mhs(node)); + return std::max(EffectiveSize(node), std::max(mhs_left, mhs_right)); + } + } + + void Tree::ValidateMhs() { + if (!_root) + return; + uint64_t mhs_left = ValidateMhs(_root->_left); + uint64_t mhs_right = ValidateMhs(_root->_right); + invariant(mhs_left == rbn_left_mhs(_root)); + invariant(mhs_right == rbn_right_mhs(_root)); + } + +} // namespace MhsRbTree diff --git a/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h b/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h new file mode 100644 index 0000000000000..92f1e278e1a01 --- /dev/null +++ b/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h @@ -0,0 +1,351 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see . + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see . +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include + +#include "portability/toku_pthread.h" +#include "portability/toku_stdint.h" +#include "portability/toku_stdlib.h" + +// RBTree(Red-black tree) with max hole sizes for subtrees. + +// This is a tentative data struct to improve the block allocation time +// complexity from the linear time to the log time. Please be noted this DS only +// supports first-fit for now. It is actually easier to do it with +// best-fit.(just +// sort by size). + +// RBTree is a classic data struct with O(log(n)) for insertion, deletion and +// search. Many years have seen its efficiency. + +// a *hole* is the representation of an available BlockPair for allocation. +// defined as (start_address,size) or (offset, size) interchangably. + +// each node has a *label* to indicate a pair of the max hole sizes for its +// subtree. + +// We are implementing a RBTree with max hole sizes for subtree. It is a red +// black tree that is sorted by the start_address but also labeld with the max +// hole sizes of the subtrees. + +// [(6,3)] -> [(offset, size)], the hole +// [{2,5}] -> [{mhs_of_left, mhs_of_right}], the label +/* / \ */ +// [(0, 1)] [(10, 5)] +// [{0, 2}] [{0, 0}] +/* \ */ +// [(3, 2)] +// [{0, 0}] +// request of allocation size=2 goes from root to [(3,2)]. + +// above example shows a simplified RBTree_max_holes. +// it is easier to tell the search time is O(log(n)) as we can make a decision +// on each descent until we get to the target. + +// the only question is if we can keep the maintenance cost low -- and i think +// it is not a problem becoz an insertion/deletion is only going to update the +// max_hole_sizes of the nodes along the path from the root to the node to be +// deleted/inserted. The path can be cached and search is anyway O(log(n)). + +// unlike the typical rbtree, Tree has to handle the inserts and deletes +// with more care: an allocation that triggers the delete might leave some +// unused space which we can simply update the start_addr and size without +// worrying overlapping. An free might not only mean the insertion but also +// *merging* with the adjacent holes. + +namespace MhsRbTree { + +#define offset_t uint64_t + enum class EColor { RED, BLACK }; + enum class EDirection { NONE = 0, LEFT, RIGHT }; + + // I am a bit tired of fixing overflow/underflow, just quickly craft some + // int + // class that has an infinity-like max value and prevents overflow and + // underflow. If you got a file offset larger than MHS_MAX_VAL, it is not + // a problem here. :-/ - JYM + class OUUInt64 { + public: + static const uint64_t MHS_MAX_VAL = 0xffffffffffffffff; + OUUInt64() : _value(0) {} + OUUInt64(uint64_t s) : _value(s) {} + bool operator<(const OUUInt64 &r) const { + invariant(!(_value == MHS_MAX_VAL && r.ToInt() == MHS_MAX_VAL)); + return _value < r.ToInt(); + } + bool operator>(const OUUInt64 &r) const { + invariant(!(_value == MHS_MAX_VAL && r.ToInt() == MHS_MAX_VAL)); + return _value > r.ToInt(); + } + bool operator<=(const OUUInt64 &r) const { + invariant(!(_value == MHS_MAX_VAL && r.ToInt() == MHS_MAX_VAL)); + return _value <= r.ToInt(); + } + bool operator>=(const OUUInt64 &r) const { + invariant(!(_value == MHS_MAX_VAL && r.ToInt() == MHS_MAX_VAL)); + return _value >= r.ToInt(); + } + OUUInt64 operator+(const OUUInt64 &r) const { + if (_value == MHS_MAX_VAL || r.ToInt() == MHS_MAX_VAL) { + OUUInt64 tmp(MHS_MAX_VAL); + return tmp; + } else { + // detecting overflow + invariant((MHS_MAX_VAL - _value) >= r.ToInt()); + uint64_t plus = _value + r.ToInt(); + OUUInt64 tmp(plus); + return tmp; + } + } + OUUInt64 operator-(const OUUInt64 &r) const { + invariant(r.ToInt() != MHS_MAX_VAL); + if (_value == MHS_MAX_VAL) { + return *this; + } else { + invariant(_value >= r.ToInt()); + uint64_t minus = _value - r.ToInt(); + OUUInt64 tmp(minus); + return tmp; + } + } + OUUInt64 operator-=(const OUUInt64 &r) { + if (_value != MHS_MAX_VAL) { + invariant(r.ToInt() != MHS_MAX_VAL); + invariant(_value >= r.ToInt()); + _value -= r.ToInt(); + } + return *this; + } + OUUInt64 operator+=(const OUUInt64 &r) { + if (_value != MHS_MAX_VAL) { + if (r.ToInt() == MHS_MAX_VAL) { + _value = MHS_MAX_VAL; + } else { + invariant((MHS_MAX_VAL - _value) >= r.ToInt()); + this->_value += r.ToInt(); + } + } + return *this; + } + bool operator==(const OUUInt64 &r) const { + return _value == r.ToInt(); + } + bool operator!=(const OUUInt64 &r) const { + return _value != r.ToInt(); + } + OUUInt64 operator=(const OUUInt64 &r) { + _value = r.ToInt(); + return *this; + } + uint64_t ToInt() const { return _value; } + + private: + uint64_t _value; + }; + + class Node { + public: + struct BlockPair { + OUUInt64 _offset; + OUUInt64 _size; + + BlockPair() : _offset(0), _size(0) {} + BlockPair(uint64_t o, uint64_t s) : _offset(o), _size(s) {} + + BlockPair(OUUInt64 o, OUUInt64 s) : _offset(o), _size(s) {} + int operator<(const struct BlockPair &rhs) const { + return _offset < rhs._offset; + } + int operator<(const uint64_t &o) const { return _offset < o; } + }; + + struct Pair { + uint64_t _left; + uint64_t _right; + Pair(uint64_t l, uint64_t r) : _left(l), _right(r) {} + }; + + EColor _color; + struct BlockPair _hole; + struct Pair _label; + Node *_left; + Node *_right; + Node *_parent; + + Node(EColor c, + Node::BlockPair h, + struct Pair lb, + Node *l, + Node *r, + Node *p) + : _color(c), + _hole(h), + _label(lb), + _left(l), + _right(r), + _parent(p) {} + }; + + class Tree { + private: + Node *_root; + uint64_t _align; + + public: + Tree(); + Tree(uint64_t); + ~Tree(); + + void PreOrder(); + void InOrder(); + void PostOrder(); + // immutable operations + Node *SearchByOffset(uint64_t addr); + Node *SearchFirstFitBySize(uint64_t size); + + Node *MinNode(); + Node *MaxNode(); + + Node *Successor(Node *); + Node *Predecessor(Node *); + + // mapped from tree_allocator::free_block + int Insert(Node::BlockPair pair); + // mapped from tree_allocator::alloc_block + uint64_t Remove(size_t size); + // mapped from tree_allocator::alloc_block_after + + void RawRemove(uint64_t offset); + void Destroy(); + // print the tree + void Dump(); + // validation + // balance + void ValidateBalance(); + void ValidateInOrder(Node::BlockPair *); + void InOrderVisitor(void (*f)(void *, Node *, uint64_t), void *); + void ValidateMhs(); + + private: + void PreOrder(Node *node) const; + void InOrder(Node *node) const; + void PostOrder(Node *node) const; + Node *SearchByOffset(Node *node, offset_t addr) const; + Node *SearchFirstFitBySize(Node *node, size_t size) const; + + Node *MinNode(Node *node); + Node *MaxNode(Node *node); + + // rotations to fix up. we will have to update the labels too. + void LeftRotate(Node *&root, Node *x); + void RightRotate(Node *&root, Node *y); + + int Insert(Node *&root, Node::BlockPair pair); + int InsertFixup(Node *&root, Node *node); + + void RawRemove(Node *&root, Node *node); + uint64_t Remove(Node *&root, Node *node, size_t size); + void RawRemoveFixup(Node *&root, Node *node, Node *parent); + + void Destroy(Node *&tree); + void Dump(Node *tree, Node::BlockPair pair, EDirection dir); + void RecalculateMhs(Node *node); + void IsNewNodeMergable(Node *, Node *, Node::BlockPair, bool *, bool *); + void AbsorbNewNode(Node *, Node *, Node::BlockPair, bool, bool, bool); + Node *SearchFirstFitBySizeHelper(Node *x, uint64_t size); + + Node *SuccessorHelper(Node *y, Node *x); + + Node *PredecessorHelper(Node *y, Node *x); + + void InOrderVisitor(Node *, + void (*f)(void *, Node *, uint64_t), + void *, + uint64_t); + uint64_t ValidateMhs(Node *); + + uint64_t EffectiveSize(Node *); +// mixed with some macros..... +#define rbn_parent(r) ((r)->_parent) +#define rbn_color(r) ((r)->_color) +#define rbn_is_red(r) ((r)->_color == EColor::RED) +#define rbn_is_black(r) ((r)->_color == EColor::BLACK) +#define rbn_set_black(r) \ + do { \ + (r)->_color = EColor::BLACK; \ + } while (0) +#define rbn_set_red(r) \ + do { \ + (r)->_color = EColor::RED; \ + } while (0) +#define rbn_set_parent(r, p) \ + do { \ + (r)->_parent = (p); \ + } while (0) +#define rbn_set_color(r, c) \ + do { \ + (r)->_color = (c); \ + } while (0) +#define rbn_set_offset(r) \ + do { \ + (r)->_hole._offset = (c); \ + } while (0) +#define rbn_set_size(r, c) \ + do { \ + (r)->_hole._size = (c); \ + } while (0) +#define rbn_set_left_mhs(r, c) \ + do { \ + (r)->_label._left = (c); \ + } while (0) +#define rbn_set_right_mhs(r, c) \ + do { \ + (r)->_label._right = (c); \ + } while (0) +#define rbn_size(r) ((r)->_hole._size) +#define rbn_offset(r) ((r)->_hole._offset) +#define rbn_key(r) ((r)->_hole._offset) +#define rbn_left_mhs(r) ((r)->_label._left) +#define rbn_right_mhs(r) ((r)->_label._right) +#define mhs_of_subtree(y) \ + (std::max(std::max(rbn_left_mhs(y), rbn_right_mhs(y)), EffectiveSize(y))) + }; + +} // namespace MhsRbTree diff --git a/storage/tokudb/PerconaFT/ft/tests/block_allocator_strategy_test.cc b/storage/tokudb/PerconaFT/ft/tests/block_allocator_strategy_test.cc deleted file mode 100644 index 3670ef81cc2f7..0000000000000 --- a/storage/tokudb/PerconaFT/ft/tests/block_allocator_strategy_test.cc +++ /dev/null @@ -1,126 +0,0 @@ -/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ -// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: -#ident "$Id$" -/*====== -This file is part of PerconaFT. - - -Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - - PerconaFT is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License, version 2, - as published by the Free Software Foundation. - - PerconaFT is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with PerconaFT. If not, see . - ----------------------------------------- - - PerconaFT is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License, version 3, - as published by the Free Software Foundation. - - PerconaFT is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with PerconaFT. If not, see . -======= */ - -#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." - -#include "ft/tests/test.h" - -#include "ft/serialize/block_allocator_strategy.h" - -static const uint64_t alignment = 4096; - -static void test_first_vs_best_fit(void) { - struct block_allocator::blockpair pairs[] = { - block_allocator::blockpair(1 * alignment, 6 * alignment), - // hole between 7x align -> 8x align - block_allocator::blockpair(8 * alignment, 4 * alignment), - // hole between 12x align -> 16x align - block_allocator::blockpair(16 * alignment, 1 * alignment), - block_allocator::blockpair(17 * alignment, 2 * alignment), - // hole between 19 align -> 21x align - block_allocator::blockpair(21 * alignment, 2 * alignment), - }; - const uint64_t n_blocks = sizeof(pairs) / sizeof(pairs[0]); - - block_allocator::blockpair *bp; - - // first fit - bp = block_allocator_strategy::first_fit(pairs, n_blocks, 100, alignment); - assert(bp == &pairs[0]); - bp = block_allocator_strategy::first_fit(pairs, n_blocks, 4096, alignment); - assert(bp == &pairs[0]); - bp = block_allocator_strategy::first_fit(pairs, n_blocks, 3 * 4096, alignment); - assert(bp == &pairs[1]); - bp = block_allocator_strategy::first_fit(pairs, n_blocks, 5 * 4096, alignment); - assert(bp == nullptr); - - // best fit - bp = block_allocator_strategy::best_fit(pairs, n_blocks, 100, alignment); - assert(bp == &pairs[0]); - bp = block_allocator_strategy::best_fit(pairs, n_blocks, 4100, alignment); - assert(bp == &pairs[3]); - bp = block_allocator_strategy::best_fit(pairs, n_blocks, 3 * 4096, alignment); - assert(bp == &pairs[1]); - bp = block_allocator_strategy::best_fit(pairs, n_blocks, 5 * 4096, alignment); - assert(bp == nullptr); -} - -static void test_padded_fit(void) { - struct block_allocator::blockpair pairs[] = { - block_allocator::blockpair(1 * alignment, 1 * alignment), - // 4096 byte hole after bp[0] - block_allocator::blockpair(3 * alignment, 1 * alignment), - // 8192 byte hole after bp[1] - block_allocator::blockpair(6 * alignment, 1 * alignment), - // 16384 byte hole after bp[2] - block_allocator::blockpair(11 * alignment, 1 * alignment), - // 32768 byte hole after bp[3] - block_allocator::blockpair(17 * alignment, 1 * alignment), - // 116kb hole after bp[4] - block_allocator::blockpair(113 * alignment, 1 * alignment), - // 256kb hole after bp[5] - block_allocator::blockpair(371 * alignment, 1 * alignment), - }; - const uint64_t n_blocks = sizeof(pairs) / sizeof(pairs[0]); - - block_allocator::blockpair *bp; - - // padding for a 100 byte allocation will be < than standard alignment, - // so it should fit in the first 4096 byte hole. - bp = block_allocator_strategy::padded_fit(pairs, n_blocks, 4000, alignment); - assert(bp == &pairs[0]); - - // Even padded, a 12kb alloc will fit in a 16kb hole - bp = block_allocator_strategy::padded_fit(pairs, n_blocks, 3 * alignment, alignment); - assert(bp == &pairs[2]); - - // would normally fit in the 116kb hole but the padding will bring it over - bp = block_allocator_strategy::padded_fit(pairs, n_blocks, 116 * alignment, alignment); - assert(bp == &pairs[5]); - - bp = block_allocator_strategy::padded_fit(pairs, n_blocks, 127 * alignment, alignment); - assert(bp == &pairs[5]); -} - -int test_main(int argc, const char *argv[]) { - (void) argc; - (void) argv; - - test_first_vs_best_fit(); - test_padded_fit(); - - return 0; -} diff --git a/storage/tokudb/PerconaFT/ft/tests/block_allocator_test.cc b/storage/tokudb/PerconaFT/ft/tests/block_allocator_test.cc index d80ee83cbc95f..3eff52b915d66 100644 --- a/storage/tokudb/PerconaFT/ft/tests/block_allocator_test.cc +++ b/storage/tokudb/PerconaFT/ft/tests/block_allocator_test.cc @@ -38,253 +38,243 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "test.h" -static void ba_alloc(block_allocator *ba, uint64_t size, uint64_t *answer) { - ba->validate(); +static void ba_alloc(BlockAllocator *ba, uint64_t size, uint64_t *answer) { + ba->Validate(); uint64_t actual_answer; - const uint64_t heat = random() % 2; - ba->alloc_block(512 * size, heat, &actual_answer); - ba->validate(); + ba->AllocBlock(512 * size, &actual_answer); + ba->Validate(); - assert(actual_answer%512==0); - *answer = actual_answer/512; + invariant(actual_answer % 512 == 0); + *answer = actual_answer / 512; } -static void ba_free(block_allocator *ba, uint64_t offset) { - ba->validate(); - ba->free_block(offset * 512); - ba->validate(); +static void ba_free(BlockAllocator *ba, uint64_t offset, uint64_t size) { + ba->Validate(); + ba->FreeBlock(offset * 512, 512 * size); + ba->Validate(); } -static void ba_check_l(block_allocator *ba, uint64_t blocknum_in_layout_order, - uint64_t expected_offset, uint64_t expected_size) { +static void ba_check_l(BlockAllocator *ba, + uint64_t blocknum_in_layout_order, + uint64_t expected_offset, + uint64_t expected_size) { uint64_t actual_offset, actual_size; - int r = ba->get_nth_block_in_layout_order(blocknum_in_layout_order, &actual_offset, &actual_size); - assert(r==0); - assert(expected_offset*512 == actual_offset); - assert(expected_size *512 == actual_size); + int r = ba->NthBlockInLayoutOrder( + blocknum_in_layout_order, &actual_offset, &actual_size); + invariant(r == 0); + invariant(expected_offset * 512 == actual_offset); + invariant(expected_size * 512 == actual_size); } -static void ba_check_none(block_allocator *ba, uint64_t blocknum_in_layout_order) { +static void ba_check_none(BlockAllocator *ba, + uint64_t blocknum_in_layout_order) { uint64_t actual_offset, actual_size; - int r = ba->get_nth_block_in_layout_order(blocknum_in_layout_order, &actual_offset, &actual_size); - assert(r==-1); + int r = ba->NthBlockInLayoutOrder( + blocknum_in_layout_order, &actual_offset, &actual_size); + invariant(r == -1); } - // Simple block allocator test -static void test_ba0(block_allocator::allocation_strategy strategy) { - block_allocator allocator; - block_allocator *ba = &allocator; - ba->create(100*512, 1*512); - ba->set_strategy(strategy); - assert(ba->allocated_limit()==100*512); +static void test_ba0() { + BlockAllocator allocator; + BlockAllocator *ba = &allocator; + ba->Create(100 * 512, 1 * 512); + invariant(ba->AllocatedLimit() == 100 * 512); uint64_t b2, b3, b4, b5, b6, b7; - ba_alloc(ba, 100, &b2); - ba_alloc(ba, 100, &b3); - ba_alloc(ba, 100, &b4); - ba_alloc(ba, 100, &b5); - ba_alloc(ba, 100, &b6); - ba_alloc(ba, 100, &b7); - ba_free(ba, b2); - ba_alloc(ba, 100, &b2); - ba_free(ba, b4); - ba_free(ba, b6); + ba_alloc(ba, 100, &b2); + ba_alloc(ba, 100, &b3); + ba_alloc(ba, 100, &b4); + ba_alloc(ba, 100, &b5); + ba_alloc(ba, 100, &b6); + ba_alloc(ba, 100, &b7); + ba_free(ba, b2, 100); + ba_alloc(ba, 100, &b2); + ba_free(ba, b4, 100); + ba_free(ba, b6, 100); uint64_t b8, b9; - ba_alloc(ba, 100, &b4); - ba_free(ba, b2); - ba_alloc(ba, 100, &b6); - ba_alloc(ba, 100, &b8); - ba_alloc(ba, 100, &b9); - ba_free(ba, b6); - ba_free(ba, b7); - ba_free(ba, b8); - ba_alloc(ba, 100, &b6); - ba_alloc(ba, 100, &b7); - ba_free(ba, b4); - ba_alloc(ba, 100, &b4); - - ba->destroy(); + ba_alloc(ba, 100, &b4); + ba_free(ba, b2, 100); + ba_alloc(ba, 100, &b6); + ba_alloc(ba, 100, &b8); + ba_alloc(ba, 100, &b9); + ba_free(ba, b6, 100); + ba_free(ba, b7, 100); + ba_free(ba, b8, 100); + ba_alloc(ba, 100, &b6); + ba_alloc(ba, 100, &b7); + ba_free(ba, b4, 100); + ba_alloc(ba, 100, &b4); + + ba->Destroy(); } // Manually to get coverage of all the code in the block allocator. -static void -test_ba1(block_allocator::allocation_strategy strategy, int n_initial) { - block_allocator allocator; - block_allocator *ba = &allocator; - ba->create(0*512, 1*512); - ba->set_strategy(strategy); - - int n_blocks=0; +static void test_ba1(int n_initial) { + BlockAllocator allocator; + BlockAllocator *ba = &allocator; + ba->Create(0 * 512, 1 * 512); + + int n_blocks = 0; uint64_t blocks[1000]; for (int i = 0; i < 1000; i++) { - if (i < n_initial || random() % 2 == 0) { - if (n_blocks < 1000) { - ba_alloc(ba, 1, &blocks[n_blocks]); - //printf("A[%d]=%ld\n", n_blocks, blocks[n_blocks]); - n_blocks++; - } - } else { - if (n_blocks > 0) { - int blocknum = random()%n_blocks; - //printf("F[%d]%ld\n", blocknum, blocks[blocknum]); - ba_free(ba, blocks[blocknum]); - blocks[blocknum]=blocks[n_blocks-1]; - n_blocks--; - } - } + if (i < n_initial || random() % 2 == 0) { + if (n_blocks < 1000) { + ba_alloc(ba, 1, &blocks[n_blocks]); + // printf("A[%d]=%ld\n", n_blocks, blocks[n_blocks]); + n_blocks++; + } + } else { + if (n_blocks > 0) { + int blocknum = random() % n_blocks; + // printf("F[%d]=%ld\n", blocknum, blocks[blocknum]); + ba_free(ba, blocks[blocknum], 1); + blocks[blocknum] = blocks[n_blocks - 1]; + n_blocks--; + } + } } - - ba->destroy(); + + ba->Destroy(); } - + // Check to see if it is first fit or best fit. -static void -test_ba2 (void) -{ - block_allocator allocator; - block_allocator *ba = &allocator; +static void test_ba2(void) { + BlockAllocator allocator; + BlockAllocator *ba = &allocator; uint64_t b[6]; enum { BSIZE = 1024 }; - ba->create(100*512, BSIZE*512); - ba->set_strategy(block_allocator::BA_STRATEGY_FIRST_FIT); - assert(ba->allocated_limit()==100*512); - - ba_check_l (ba, 0, 0, 100); - ba_check_none (ba, 1); - - ba_alloc (ba, 100, &b[0]); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_none (ba, 2); - - ba_alloc (ba, BSIZE + 100, &b[1]); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_none (ba, 3); - - ba_alloc (ba, 100, &b[2]); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 3, 4*BSIZE, 100); - ba_check_none (ba, 4); - - ba_alloc (ba, 100, &b[3]); - ba_alloc (ba, 100, &b[4]); - ba_alloc (ba, 100, &b[5]); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 3, 4*BSIZE, 100); - ba_check_l (ba, 4, 5*BSIZE, 100); - ba_check_l (ba, 5, 6*BSIZE, 100); - ba_check_l (ba, 6, 7*BSIZE, 100); - ba_check_none (ba, 7); - - ba_free (ba, 4*BSIZE); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 3, 5*BSIZE, 100); - ba_check_l (ba, 4, 6*BSIZE, 100); - ba_check_l (ba, 5, 7*BSIZE, 100); - ba_check_none (ba, 6); + ba->Create(100 * 512, BSIZE * 512); + invariant(ba->AllocatedLimit() == 100 * 512); + + ba_check_l(ba, 0, 0, 100); + ba_check_none(ba, 1); + + ba_alloc(ba, 100, &b[0]); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_none(ba, 2); + + ba_alloc(ba, BSIZE + 100, &b[1]); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_none(ba, 3); + + ba_alloc(ba, 100, &b[2]); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 3, 4 * BSIZE, 100); + ba_check_none(ba, 4); + + ba_alloc(ba, 100, &b[3]); + ba_alloc(ba, 100, &b[4]); + ba_alloc(ba, 100, &b[5]); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 3, 4 * BSIZE, 100); + ba_check_l(ba, 4, 5 * BSIZE, 100); + ba_check_l(ba, 5, 6 * BSIZE, 100); + ba_check_l(ba, 6, 7 * BSIZE, 100); + ba_check_none(ba, 7); + + ba_free(ba, 4 * BSIZE, 100); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 3, 5 * BSIZE, 100); + ba_check_l(ba, 4, 6 * BSIZE, 100); + ba_check_l(ba, 5, 7 * BSIZE, 100); + ba_check_none(ba, 6); uint64_t b2; ba_alloc(ba, 100, &b2); - assert(b2==4*BSIZE); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 3, 4*BSIZE, 100); - ba_check_l (ba, 4, 5*BSIZE, 100); - ba_check_l (ba, 5, 6*BSIZE, 100); - ba_check_l (ba, 6, 7*BSIZE, 100); - ba_check_none (ba, 7); - - ba_free (ba, BSIZE); - ba_free (ba, 5*BSIZE); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 2, 4*BSIZE, 100); - ba_check_l (ba, 3, 6*BSIZE, 100); - ba_check_l (ba, 4, 7*BSIZE, 100); - ba_check_none (ba, 5); - - // This alloc will allocate the first block after the reserve space in the case of first fit. + invariant(b2 == 4 * BSIZE); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 3, 4 * BSIZE, 100); + ba_check_l(ba, 4, 5 * BSIZE, 100); + ba_check_l(ba, 5, 6 * BSIZE, 100); + ba_check_l(ba, 6, 7 * BSIZE, 100); + ba_check_none(ba, 7); + + ba_free(ba, BSIZE, 100); + ba_free(ba, 5 * BSIZE, 100); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 2, 4 * BSIZE, 100); + ba_check_l(ba, 3, 6 * BSIZE, 100); + ba_check_l(ba, 4, 7 * BSIZE, 100); + ba_check_none(ba, 5); + + // This alloc will allocate the first block after the reserve space in the + // case of first fit. uint64_t b3; ba_alloc(ba, 100, &b3); - assert(b3== BSIZE); // First fit. + invariant(b3 == BSIZE); // First fit. // if (b3==5*BSIZE) then it is next fit. // Now 5*BSIZE is free uint64_t b5; ba_alloc(ba, 100, &b5); - assert(b5==5*BSIZE); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 3, 4*BSIZE, 100); - ba_check_l (ba, 4, 5*BSIZE, 100); - ba_check_l (ba, 5, 6*BSIZE, 100); - ba_check_l (ba, 6, 7*BSIZE, 100); - ba_check_none (ba, 7); + invariant(b5 == 5 * BSIZE); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 3, 4 * BSIZE, 100); + ba_check_l(ba, 4, 5 * BSIZE, 100); + ba_check_l(ba, 5, 6 * BSIZE, 100); + ba_check_l(ba, 6, 7 * BSIZE, 100); + ba_check_none(ba, 7); // Now all blocks are busy uint64_t b6, b7, b8; ba_alloc(ba, 100, &b6); ba_alloc(ba, 100, &b7); ba_alloc(ba, 100, &b8); - assert(b6==8*BSIZE); - assert(b7==9*BSIZE); - assert(b8==10*BSIZE); - ba_check_l (ba, 0, 0, 100); - ba_check_l (ba, 1, BSIZE, 100); - ba_check_l (ba, 2, 2*BSIZE, BSIZE + 100); - ba_check_l (ba, 3, 4*BSIZE, 100); - ba_check_l (ba, 4, 5*BSIZE, 100); - ba_check_l (ba, 5, 6*BSIZE, 100); - ba_check_l (ba, 6, 7*BSIZE, 100); - ba_check_l (ba, 7, 8*BSIZE, 100); - ba_check_l (ba, 8, 9*BSIZE, 100); - ba_check_l (ba, 9, 10*BSIZE, 100); - ba_check_none (ba, 10); - - ba_free(ba, 9*BSIZE); - ba_free(ba, 7*BSIZE); + invariant(b6 == 8 * BSIZE); + invariant(b7 == 9 * BSIZE); + invariant(b8 == 10 * BSIZE); + ba_check_l(ba, 0, 0, 100); + ba_check_l(ba, 1, BSIZE, 100); + ba_check_l(ba, 2, 2 * BSIZE, BSIZE + 100); + ba_check_l(ba, 3, 4 * BSIZE, 100); + ba_check_l(ba, 4, 5 * BSIZE, 100); + ba_check_l(ba, 5, 6 * BSIZE, 100); + ba_check_l(ba, 6, 7 * BSIZE, 100); + ba_check_l(ba, 7, 8 * BSIZE, 100); + ba_check_l(ba, 8, 9 * BSIZE, 100); + ba_check_l(ba, 9, 10 * BSIZE, 100); + ba_check_none(ba, 10); + + ba_free(ba, 9 * BSIZE, 100); + ba_free(ba, 7 * BSIZE, 100); uint64_t b9; ba_alloc(ba, 100, &b9); - assert(b9==7*BSIZE); + invariant(b9 == 7 * BSIZE); - ba_free(ba, 5*BSIZE); - ba_free(ba, 2*BSIZE); + ba_free(ba, 5 * BSIZE, 100); + ba_free(ba, 2 * BSIZE, BSIZE + 100); uint64_t b10, b11; ba_alloc(ba, 100, &b10); - assert(b10==2*BSIZE); + invariant(b10 == 2 * BSIZE); ba_alloc(ba, 100, &b11); - assert(b11==3*BSIZE); + invariant(b11 == 3 * BSIZE); ba_alloc(ba, 100, &b11); - assert(b11==5*BSIZE); + invariant(b11 == 5 * BSIZE); - ba->destroy(); + ba->Destroy(); } -int -test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { - enum block_allocator::allocation_strategy strategies[] = { - block_allocator::BA_STRATEGY_FIRST_FIT, - block_allocator::BA_STRATEGY_BEST_FIT, - block_allocator::BA_STRATEGY_PADDED_FIT, - block_allocator::BA_STRATEGY_HEAT_ZONE, - }; - for (size_t i = 0; i < sizeof(strategies) / sizeof(strategies[0]); i++) { - test_ba0(strategies[i]); - test_ba1(strategies[i], 0); - test_ba1(strategies[i], 10); - test_ba1(strategies[i], 20); - } +int test_main(int argc __attribute__((__unused__)), + const char *argv[] __attribute__((__unused__))) { + test_ba0(); + test_ba1(0); + test_ba1(10); + test_ba1(20); test_ba2(); return 0; } diff --git a/storage/tokudb/PerconaFT/ft/tests/cachetable-5978.cc b/storage/tokudb/PerconaFT/ft/tests/cachetable-5978.cc index a7c48ef709af6..ee68ab3ef0bb9 100644 --- a/storage/tokudb/PerconaFT/ft/tests/cachetable-5978.cc +++ b/storage/tokudb/PerconaFT/ft/tests/cachetable-5978.cc @@ -45,7 +45,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. // #5978 is fixed. Here is what we do. We have four pairs with // blocknums and fullhashes of 1,2,3,4. The cachetable has only // two bucket mutexes, so 1 and 3 share a pair mutex, as do 2 and 4. -// We pin all four with expensive write locks. Then, on backgroud threads, +// We pin all four with expensive write locks. Then, on background threads, // we call get_and_pin_nonblocking on 3, where the unlockers unpins 2, and // we call get_and_pin_nonblocking on 4, where the unlockers unpins 1. Run this // enough times, and we should see a deadlock before the fix, and no deadlock diff --git a/storage/tokudb/PerconaFT/ft/tests/cachetable-simple-clone2.cc b/storage/tokudb/PerconaFT/ft/tests/cachetable-simple-clone2.cc index be4bae898bebb..51cf70c3e76a6 100644 --- a/storage/tokudb/PerconaFT/ft/tests/cachetable-simple-clone2.cc +++ b/storage/tokudb/PerconaFT/ft/tests/cachetable-simple-clone2.cc @@ -77,7 +77,7 @@ flush ( // // test the following things for simple cloning: -// - verifies that after teh checkpoint ends, the PAIR is properly +// - verifies that after the checkpoint ends, the PAIR is properly // dirty or clean based on the second unpin // static void diff --git a/storage/tokudb/PerconaFT/ft/tests/ft-bfe-query.cc b/storage/tokudb/PerconaFT/ft/tests/ft-bfe-query.cc index cb03a23e0fc65..7abd2267a7ea6 100644 --- a/storage/tokudb/PerconaFT/ft/tests/ft-bfe-query.cc +++ b/storage/tokudb/PerconaFT/ft/tests/ft-bfe-query.cc @@ -38,69 +38,72 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "test.h" -static int -int64_key_cmp (DB *db UU(), const DBT *a, const DBT *b) { - int64_t x = *(int64_t *) a->data; - int64_t y = *(int64_t *) b->data; - - if (xy) return 1; +static int int64_key_cmp(DB *db UU(), const DBT *a, const DBT *b) { + int64_t x = *(int64_t *)a->data; + int64_t y = *(int64_t *)b->data; + + if (x < y) + return -1; + if (x > y) + return 1; return 0; } -static void -test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) { +static void test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) { int r; FT_CURSOR XMALLOC(cursor); FTNODE dn = NULL; PAIR_ATTR attr; - + // first test that prefetching everything should work - memset(&cursor->range_lock_left_key, 0 , sizeof(DBT)); - memset(&cursor->range_lock_right_key, 0 , sizeof(DBT)); + memset(&cursor->range_lock_left_key, 0, sizeof(DBT)); + memset(&cursor->range_lock_right_key, 0, sizeof(DBT)); cursor->left_is_neg_infty = true; cursor->right_is_pos_infty = true; cursor->disable_prefetching = false; - + ftnode_fetch_extra bfe; // quick test to see that we have the right behavior when we set // disable_prefetching to true cursor->disable_prefetching = true; - bfe.create_for_prefetch( ft_h, cursor); + bfe.create_for_prefetch(ft_h, cursor); FTNODE_DISK_DATA ndd = NULL; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_ON_DISK); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_ON_DISK); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); bfe.destroy(); toku_ftnode_free(&dn); toku_free(ndd); // now enable prefetching again cursor->disable_prefetching = false; - - bfe.create_for_prefetch( ft_h, cursor); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_COMPRESSED); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_COMPRESSED); + + bfe.create_for_prefetch(ft_h, cursor); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_COMPRESSED); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_COMPRESSED); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_AVAIL); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_AVAIL); bfe.destroy(); toku_ftnode_free(&dn); toku_free(ndd); @@ -108,21 +111,23 @@ test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) { uint64_t left_key = 150; toku_fill_dbt(&cursor->range_lock_left_key, &left_key, sizeof(uint64_t)); cursor->left_is_neg_infty = false; - bfe.create_for_prefetch( ft_h, cursor); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_COMPRESSED); + bfe.create_for_prefetch(ft_h, cursor); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_COMPRESSED); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_AVAIL); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_AVAIL); bfe.destroy(); toku_ftnode_free(&dn); toku_free(ndd); @@ -130,63 +135,69 @@ test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) { uint64_t right_key = 151; toku_fill_dbt(&cursor->range_lock_right_key, &right_key, sizeof(uint64_t)); cursor->right_is_pos_infty = false; - bfe.create_for_prefetch( ft_h, cursor); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_ON_DISK); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_ON_DISK); + bfe.create_for_prefetch(ft_h, cursor); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_ON_DISK); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); bfe.destroy(); toku_ftnode_free(&dn); toku_free(ndd); left_key = 100000; right_key = 100000; - bfe.create_for_prefetch( ft_h, cursor); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_COMPRESSED); + bfe.create_for_prefetch(ft_h, cursor); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_COMPRESSED); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_AVAIL); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_AVAIL); bfe.destroy(); toku_free(ndd); toku_ftnode_free(&dn); left_key = 100; right_key = 100; - bfe.create_for_prefetch( ft_h, cursor); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_ON_DISK); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_COMPRESSED); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_ON_DISK); + bfe.create_for_prefetch(ft_h, cursor); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_COMPRESSED); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_ON_DISK); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); bfe.destroy(); toku_ftnode_free(&dn); toku_free(ndd); @@ -194,20 +205,19 @@ test_prefetch_read(int fd, FT_HANDLE UU(ft), FT ft_h) { toku_free(cursor); } -static void -test_subset_read(int fd, FT_HANDLE UU(ft), FT ft_h) { +static void test_subset_read(int fd, FT_HANDLE UU(ft), FT ft_h) { int r; FT_CURSOR XMALLOC(cursor); FTNODE dn = NULL; FTNODE_DISK_DATA ndd = NULL; PAIR_ATTR attr; - + // first test that prefetching everything should work - memset(&cursor->range_lock_left_key, 0 , sizeof(DBT)); - memset(&cursor->range_lock_right_key, 0 , sizeof(DBT)); + memset(&cursor->range_lock_left_key, 0, sizeof(DBT)); + memset(&cursor->range_lock_right_key, 0, sizeof(DBT)); cursor->left_is_neg_infty = true; cursor->right_is_pos_infty = true; - + uint64_t left_key = 150; uint64_t right_key = 151; DBT left, right; @@ -216,101 +226,106 @@ test_subset_read(int fd, FT_HANDLE UU(ft), FT ft_h) { ftnode_fetch_extra bfe; bfe.create_for_subset_read( - ft_h, - NULL, - &left, - &right, - false, - false, - false, - false - ); - + ft_h, NULL, &left, &right, false, false, false, false); + // fake the childnum to read // set disable_prefetching ON bfe.child_to_read = 2; bfe.disable_prefetching = true; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_AVAIL); - // need to call this twice because we had a subset read before, that touched the clock - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_COMPRESSED); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + // need to call this twice because we had a subset read before, that touched + // the clock + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_COMPRESSED); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_ON_DISK); - assert(BP_STATE(dn,2) == PT_AVAIL); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_ON_DISK); + invariant(BP_STATE(dn, 2) == PT_AVAIL); toku_ftnode_free(&dn); toku_free(ndd); // fake the childnum to read bfe.child_to_read = 2; bfe.disable_prefetching = false; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_AVAIL); - // need to call this twice because we had a subset read before, that touched the clock - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_AVAIL); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_COMPRESSED); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + // need to call this twice because we had a subset read before, that touched + // the clock + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_AVAIL); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_COMPRESSED); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_ON_DISK); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_AVAIL); + invariant(BP_STATE(dn, 0) == PT_ON_DISK); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_AVAIL); toku_ftnode_free(&dn); toku_free(ndd); // fake the childnum to read bfe.child_to_read = 0; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd, &bfe); - assert(r==0); - assert(dn->n_children == 3); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_ON_DISK); - // need to call this twice because we had a subset read before, that touched the clock - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_ON_DISK); - toku_ftnode_pe_callback(dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(dn,0) == PT_COMPRESSED); - assert(BP_STATE(dn,1) == PT_COMPRESSED); - assert(BP_STATE(dn,2) == PT_ON_DISK); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd, &bfe); + invariant(r == 0); + invariant(dn->n_children == 3); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); + // need to call this twice because we had a subset read before, that touched + // the clock + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); + toku_ftnode_pe_callback( + dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + invariant(BP_STATE(dn, 0) == PT_COMPRESSED); + invariant(BP_STATE(dn, 1) == PT_COMPRESSED); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); r = toku_ftnode_pf_callback(dn, ndd, &bfe, fd, &attr); - assert(BP_STATE(dn,0) == PT_AVAIL); - assert(BP_STATE(dn,1) == PT_AVAIL); - assert(BP_STATE(dn,2) == PT_ON_DISK); + invariant(BP_STATE(dn, 0) == PT_AVAIL); + invariant(BP_STATE(dn, 1) == PT_AVAIL); + invariant(BP_STATE(dn, 2) == PT_ON_DISK); toku_ftnode_free(&dn); toku_free(ndd); toku_free(cursor); } - -static void -test_prefetching(void) { +static void test_prefetching(void) { // struct ft_handle source_ft; struct ftnode sn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -327,7 +342,7 @@ test_prefetching(void) { uint64_t key1 = 100; uint64_t key2 = 200; - + MALLOC_N(sn.n_children, sn.bp); DBT pivotkeys[2]; toku_fill_dbt(&pivotkeys[0], &key1, sizeof(key1)); @@ -336,13 +351,13 @@ test_prefetching(void) { BP_BLOCKNUM(&sn, 0).b = 30; BP_BLOCKNUM(&sn, 1).b = 35; BP_BLOCKNUM(&sn, 2).b = 40; - BP_STATE(&sn,0) = PT_AVAIL; - BP_STATE(&sn,1) = PT_AVAIL; - BP_STATE(&sn,2) = PT_AVAIL; + BP_STATE(&sn, 0) = PT_AVAIL; + BP_STATE(&sn, 1) = PT_AVAIL; + BP_STATE(&sn, 2) = PT_AVAIL; set_BNC(&sn, 0, toku_create_empty_nl()); set_BNC(&sn, 1, toku_create_empty_nl()); set_BNC(&sn, 2, toku_create_empty_nl()); - //Create XIDS + // Create XIDS XIDS xids_0 = toku_xids_get_root_xids(); XIDS xids_123; XIDS xids_234; @@ -352,7 +367,7 @@ test_prefetching(void) { CKERR(r); // data in the buffers does not matter in this test - //Cleanup: + // Cleanup: toku_xids_destroy(&xids_0); toku_xids_destroy(&xids_123); toku_xids_destroy(&xids_234); @@ -363,41 +378,48 @@ test_prefetching(void) { make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft_h->cmp.create(int64_key_cmp, nullptr); ft->ft = ft_h; ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA ndd = NULL; - r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); - assert(r==0); + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); + invariant(r == 0); - test_prefetch_read(fd, ft, ft_h); + test_prefetch_read(fd, ft, ft_h); test_subset_read(fd, ft, ft_h); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); ft_h->cmp.destroy(); toku_free(ft_h->h); @@ -405,11 +427,12 @@ test_prefetching(void) { toku_free(ft); toku_free(ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -int -test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { +int test_main(int argc __attribute__((__unused__)), + const char *argv[] __attribute__((__unused__))) { test_prefetching(); return 0; diff --git a/storage/tokudb/PerconaFT/ft/tests/ft-clock-test.cc b/storage/tokudb/PerconaFT/ft/tests/ft-clock-test.cc index ceef3772e2a8b..26a3dae673cd9 100644 --- a/storage/tokudb/PerconaFT/ft/tests/ft-clock-test.cc +++ b/storage/tokudb/PerconaFT/ft/tests/ft-clock-test.cc @@ -40,38 +40,28 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "ft/cursor.h" -enum ftnode_verify_type { - read_all=1, - read_compressed, - read_none -}; +enum ftnode_verify_type { read_all = 1, read_compressed, read_none }; #ifndef MIN #define MIN(x, y) (((x) < (y)) ? (x) : (y)) #endif -static int -string_key_cmp(DB *UU(e), const DBT *a, const DBT *b) -{ +static int string_key_cmp(DB *UU(e), const DBT *a, const DBT *b) { char *CAST_FROM_VOIDP(s, a->data); char *CAST_FROM_VOIDP(t, b->data); return strcmp(s, t); } -static void -le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keylen, const char *val, int vallen) -{ +static void le_add_to_bn(bn_data *bn, + uint32_t idx, + const char *key, + int keylen, + const char *val, + int vallen) { LEAFENTRY r = NULL; uint32_t size_needed = LE_CLEAN_MEMSIZE(vallen); void *maybe_free = nullptr; - bn->get_space_for_insert( - idx, - key, - keylen, - size_needed, - &r, - &maybe_free - ); + bn->get_space_for_insert(idx, key, keylen, size_needed, &r, &maybe_free); if (maybe_free) { toku_free(maybe_free); } @@ -81,70 +71,67 @@ le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keylen, const char memcpy(r->u.clean.val, val, vallen); } - -static void -le_malloc(bn_data* bn, uint32_t idx, const char *key, const char *val) -{ +static void le_malloc(bn_data *bn, + uint32_t idx, + const char *key, + const char *val) { int keylen = strlen(key) + 1; int vallen = strlen(val) + 1; le_add_to_bn(bn, idx, key, keylen, val, vallen); } - -static void -test1(int fd, FT ft_h, FTNODE *dn) { +static void test1(int fd, FT ft_h, FTNODE *dn) { int r; ftnode_fetch_extra bfe_all; bfe_all.create_for_full_read(ft_h); FTNODE_DISK_DATA ndd = NULL; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_all); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, dn, &ndd, &bfe_all); bool is_leaf = ((*dn)->height == 0); - assert(r==0); + invariant(r == 0); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } // should sweep and NOT get rid of anything PAIR_ATTR attr; - memset(&attr,0,sizeof(attr)); + memset(&attr, 0, sizeof(attr)); toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } // should sweep and get compress all toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { if (!is_leaf) { - assert(BP_STATE(*dn,i) == PT_COMPRESSED); - } - else { - assert(BP_STATE(*dn,i) == PT_ON_DISK); + invariant(BP_STATE(*dn, i) == PT_COMPRESSED); + } else { + invariant(BP_STATE(*dn, i) == PT_ON_DISK); } } PAIR_ATTR size; bool req = toku_ftnode_pf_req_callback(*dn, &bfe_all); - assert(req); + invariant(req); toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size); toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } // should sweep and get compress all toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { if (!is_leaf) { - assert(BP_STATE(*dn,i) == PT_COMPRESSED); - } - else { - assert(BP_STATE(*dn,i) == PT_ON_DISK); + invariant(BP_STATE(*dn, i) == PT_COMPRESSED); + } else { + invariant(BP_STATE(*dn, i) == PT_ON_DISK); } - } + } req = toku_ftnode_pf_req_callback(*dn, &bfe_all); - assert(req); + invariant(req); toku_ftnode_pf_callback(*dn, ndd, &bfe_all, fd, &size); toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } (*dn)->dirty = 1; toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); @@ -152,101 +139,102 @@ test1(int fd, FT ft_h, FTNODE *dn) { toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } toku_free(ndd); toku_ftnode_free(dn); } - -static int search_cmp(const struct ft_search& UU(so), const DBT* UU(key)) { +static int search_cmp(const struct ft_search &UU(so), const DBT *UU(key)) { return 0; } -static void -test2(int fd, FT ft_h, FTNODE *dn) { +static void test2(int fd, FT ft_h, FTNODE *dn) { DBT left, right; DB dummy_db; memset(&dummy_db, 0, sizeof(dummy_db)); memset(&left, 0, sizeof(left)); memset(&right, 0, sizeof(right)); ft_search search; - + ftnode_fetch_extra bfe_subset; bfe_subset.create_for_subset_read( ft_h, - ft_search_init(&search, search_cmp, FT_SEARCH_LEFT, nullptr, nullptr, nullptr), + ft_search_init( + &search, search_cmp, FT_SEARCH_LEFT, nullptr, nullptr, nullptr), &left, &right, true, true, false, - false - ); + false); FTNODE_DISK_DATA ndd = NULL; - int r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_subset); - assert(r==0); + int r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, dn, &ndd, &bfe_subset); + invariant(r == 0); bool is_leaf = ((*dn)->height == 0); - // at this point, although both partitions are available, only the + // at this point, although both partitions are available, only the // second basement node should have had its clock // touched - assert(BP_STATE(*dn, 0) == PT_AVAIL); - assert(BP_STATE(*dn, 1) == PT_AVAIL); - assert(BP_SHOULD_EVICT(*dn, 0)); - assert(!BP_SHOULD_EVICT(*dn, 1)); + invariant(BP_STATE(*dn, 0) == PT_AVAIL); + invariant(BP_STATE(*dn, 1) == PT_AVAIL); + invariant(BP_SHOULD_EVICT(*dn, 0)); + invariant(!BP_SHOULD_EVICT(*dn, 1)); PAIR_ATTR attr; - memset(&attr,0,sizeof(attr)); + memset(&attr, 0, sizeof(attr)); toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(*dn, 0) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED); - assert(BP_STATE(*dn, 1) == PT_AVAIL); - assert(BP_SHOULD_EVICT(*dn, 1)); + invariant(BP_STATE(*dn, 0) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED); + invariant(BP_STATE(*dn, 1) == PT_AVAIL); + invariant(BP_SHOULD_EVICT(*dn, 1)); toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr); - assert(BP_STATE(*dn, 1) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED); + invariant(BP_STATE(*dn, 1) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED); bool req = toku_ftnode_pf_req_callback(*dn, &bfe_subset); - assert(req); + invariant(req); toku_ftnode_pf_callback(*dn, ndd, &bfe_subset, fd, &attr); - assert(BP_STATE(*dn, 0) == PT_AVAIL); - assert(BP_STATE(*dn, 1) == PT_AVAIL); - assert(BP_SHOULD_EVICT(*dn, 0)); - assert(!BP_SHOULD_EVICT(*dn, 1)); + invariant(BP_STATE(*dn, 0) == PT_AVAIL); + invariant(BP_STATE(*dn, 1) == PT_AVAIL); + invariant(BP_SHOULD_EVICT(*dn, 0)); + invariant(!BP_SHOULD_EVICT(*dn, 1)); toku_free(ndd); toku_ftnode_free(dn); } -static void -test3_leaf(int fd, FT ft_h, FTNODE *dn) { +static void test3_leaf(int fd, FT ft_h, FTNODE *dn) { DBT left, right; DB dummy_db; memset(&dummy_db, 0, sizeof(dummy_db)); memset(&left, 0, sizeof(left)); memset(&right, 0, sizeof(right)); - + ftnode_fetch_extra bfe_min; bfe_min.create_for_min_read(ft_h); FTNODE_DISK_DATA ndd = NULL; - int r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, &ndd, &bfe_min); - assert(r==0); + int r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, dn, &ndd, &bfe_min); + invariant(r == 0); // // make sure we have a leaf // - assert((*dn)->height == 0); + invariant((*dn)->height == 0); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn, i) == PT_ON_DISK); + invariant(BP_STATE(*dn, i) == PT_ON_DISK); } toku_ftnode_free(dn); toku_free(ndd); } -static void -test_serialize_nonleaf(void) { +static void test_serialize_nonleaf(void) { // struct ft_handle source_ft; struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -265,11 +253,11 @@ test_serialize_nonleaf(void) { sn.pivotkeys.create_from_dbts(toku_fill_dbt(&pivotkey, "hello", 6), 1); BP_BLOCKNUM(&sn, 0).b = 30; BP_BLOCKNUM(&sn, 1).b = 35; - BP_STATE(&sn,0) = PT_AVAIL; - BP_STATE(&sn,1) = PT_AVAIL; + BP_STATE(&sn, 0) = PT_AVAIL; + BP_STATE(&sn, 1) = PT_AVAIL; set_BNC(&sn, 0, toku_create_empty_nl()); set_BNC(&sn, 1, toku_create_empty_nl()); - //Create XIDS + // Create XIDS XIDS xids_0 = toku_xids_get_root_xids(); XIDS xids_123; XIDS xids_234; @@ -281,11 +269,38 @@ test_serialize_nonleaf(void) { toku::comparator cmp; cmp.create(string_key_cmp, nullptr); - toku_bnc_insert_msg(BNC(&sn, 0), "a", 2, "aval", 5, FT_NONE, next_dummymsn(), xids_0, true, cmp); - toku_bnc_insert_msg(BNC(&sn, 0), "b", 2, "bval", 5, FT_NONE, next_dummymsn(), xids_123, false, cmp); - toku_bnc_insert_msg(BNC(&sn, 1), "x", 2, "xval", 5, FT_NONE, next_dummymsn(), xids_234, true, cmp); - - //Cleanup: + toku_bnc_insert_msg(BNC(&sn, 0), + "a", + 2, + "aval", + 5, + FT_NONE, + next_dummymsn(), + xids_0, + true, + cmp); + toku_bnc_insert_msg(BNC(&sn, 0), + "b", + 2, + "bval", + 5, + FT_NONE, + next_dummymsn(), + xids_123, + false, + cmp); + toku_bnc_insert_msg(BNC(&sn, 1), + "x", + 2, + "xval", + 5, + FT_NONE, + next_dummymsn(), + xids_234, + true, + cmp); + + // Cleanup: toku_xids_destroy(&xids_0); toku_xids_destroy(&xids_123); toku_xids_destroy(&xids_234); @@ -297,35 +312,41 @@ test_serialize_nonleaf(void) { make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft_h->cmp.create(string_key_cmp, nullptr); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA ndd = NULL; - r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); - assert(r==0); + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); + invariant(r == 0); test1(fd, ft_h, &dn); test2(fd, ft_h, &dn); @@ -333,22 +354,26 @@ test_serialize_nonleaf(void) { toku_destroy_ftnode_internals(&sn); toku_free(ndd); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); ft_h->cmp.destroy(); toku_free(ft_h); toku_free(ft); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -static void -test_serialize_leaf(void) { +static void test_serialize_leaf(void) { // struct ft_handle source_ft; struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -364,8 +389,8 @@ test_serialize_leaf(void) { MALLOC_N(sn.n_children, sn.bp); DBT pivotkey; sn.pivotkeys.create_from_dbts(toku_fill_dbt(&pivotkey, "b", 2), 1); - BP_STATE(&sn,0) = PT_AVAIL; - BP_STATE(&sn,1) = PT_AVAIL; + BP_STATE(&sn, 0) = PT_AVAIL; + BP_STATE(&sn, 1) = PT_AVAIL; set_BLB(&sn, 0, toku_create_empty_bn()); set_BLB(&sn, 1, toku_create_empty_bn()); le_malloc(BLB_DATA(&sn, 0), 0, "a", "aval"); @@ -378,51 +403,59 @@ test_serialize_leaf(void) { make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA ndd = NULL; - r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); - assert(r==0); + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); + invariant(r == 0); test1(fd, ft_h, &dn); - test3_leaf(fd, ft_h,&dn); + test3_leaf(fd, ft_h, &dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); toku_free(ft); toku_free(ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -int -test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { +int test_main(int argc __attribute__((__unused__)), + const char *argv[] __attribute__((__unused__))) { initialize_dummymsn(); test_serialize_nonleaf(); test_serialize_leaf(); diff --git a/storage/tokudb/PerconaFT/ft/tests/ft-serialize-benchmark.cc b/storage/tokudb/PerconaFT/ft/tests/ft-serialize-benchmark.cc index 9828f49513c79..d50488ae19708 100644 --- a/storage/tokudb/PerconaFT/ft/tests/ft-serialize-benchmark.cc +++ b/storage/tokudb/PerconaFT/ft/tests/ft-serialize-benchmark.cc @@ -41,27 +41,21 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include #include "test.h" - - #ifndef MIN #define MIN(x, y) (((x) < (y)) ? (x) : (y)) #endif const double USECS_PER_SEC = 1000000.0; -static void -le_add_to_bn(bn_data* bn, uint32_t idx, char *key, int keylen, char *val, int vallen) -{ +static void le_add_to_bn(bn_data *bn, + uint32_t idx, + char *key, + int keylen, + char *val, + int vallen) { LEAFENTRY r = NULL; uint32_t size_needed = LE_CLEAN_MEMSIZE(vallen); void *maybe_free = nullptr; - bn->get_space_for_insert( - idx, - key, - keylen, - size_needed, - &r, - &maybe_free - ); + bn->get_space_for_insert(idx, key, keylen, size_needed, &r, &maybe_free); if (maybe_free) { toku_free(maybe_free); } @@ -71,20 +65,24 @@ le_add_to_bn(bn_data* bn, uint32_t idx, char *key, int keylen, char *val, int va memcpy(r->u.clean.val, val, vallen); } -static int -long_key_cmp(DB *UU(e), const DBT *a, const DBT *b) -{ +static int long_key_cmp(DB *UU(e), const DBT *a, const DBT *b) { const long *CAST_FROM_VOIDP(x, a->data); const long *CAST_FROM_VOIDP(y, b->data); return (*x > *y) - (*x < *y); } -static void -test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int deser_runs) { +static void test_serialize_leaf(int valsize, + int nelts, + double entropy, + int ser_runs, + int deser_runs) { // struct ft_handle source_ft; struct ftnode *sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -102,7 +100,7 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de MALLOC_N(sn->n_children, sn->bp); sn->pivotkeys.create_empty(); for (int i = 0; i < sn->n_children; ++i) { - BP_STATE(sn,i) = PT_AVAIL; + BP_STATE(sn, i) = PT_AVAIL; set_BLB(sn, i, toku_create_empty_bn()); } int nperbn = nelts / sn->n_children; @@ -112,24 +110,19 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de k = ck * nperbn + i; char buf[valsize]; int c; - for (c = 0; c < valsize * entropy; ) { - int *p = (int *) &buf[c]; + for (c = 0; c < valsize * entropy;) { + int *p = (int *)&buf[c]; *p = rand(); c += sizeof(*p); } memset(&buf[c], 0, valsize - c); le_add_to_bn( - BLB_DATA(sn,ck), - i, - (char *)&k, - sizeof k, - buf, - sizeof buf - ); + BLB_DATA(sn, ck), i, (char *)&k, sizeof k, buf, sizeof buf); } if (ck < 7) { DBT pivotkey; - sn->pivotkeys.insert_at(toku_fill_dbt(&pivotkey, &k, sizeof(k)), ck); + sn->pivotkeys.insert_at(toku_fill_dbt(&pivotkey, &k, sizeof(k)), + ck); } } @@ -139,31 +132,36 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft_h->cmp.create(long_key_cmp, nullptr); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } struct timeval total_start; @@ -176,8 +174,9 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de gettimeofday(&t[0], NULL); ndd = NULL; sn->dirty = 1; - r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, &ndd, true, ft->ft, false); - assert(r==0); + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), sn, &ndd, true, ft->ft, false); + invariant(r == 0); gettimeofday(&t[1], NULL); total_start.tv_sec += t[0].tv_sec; total_start.tv_usec += t[0].tv_usec; @@ -186,12 +185,14 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de toku_free(ndd); } double dt; - dt = (total_end.tv_sec - total_start.tv_sec) + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC); + dt = (total_end.tv_sec - total_start.tv_sec) + + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC); dt *= 1000; dt /= ser_runs; - printf("serialize leaf(ms): %0.05lf (average of %d runs)\n", dt, ser_runs); + printf( + "serialize leaf(ms): %0.05lf (average of %d runs)\n", dt, ser_runs); - //reset + // reset total_start.tv_sec = total_start.tv_usec = 0; total_end.tv_sec = total_end.tv_usec = 0; @@ -200,8 +201,9 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de bfe.create_for_full_read(ft_h); gettimeofday(&t[0], NULL); FTNODE_DISK_DATA ndd2 = NULL; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe); - assert(r==0); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd2, &bfe); + invariant(r == 0); gettimeofday(&t[1], NULL); total_start.tv_sec += t[0].tv_sec; @@ -212,35 +214,46 @@ test_serialize_leaf(int valsize, int nelts, double entropy, int ser_runs, int de toku_ftnode_free(&dn); toku_free(ndd2); } - dt = (total_end.tv_sec - total_start.tv_sec) + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC); + dt = (total_end.tv_sec - total_start.tv_sec) + + ((total_end.tv_usec - total_start.tv_usec) / USECS_PER_SEC); dt *= 1000; dt /= deser_runs; - printf("deserialize leaf(ms): %0.05lf (average of %d runs)\n", dt, deser_runs); - printf("io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf (average of %d runs)\n", - tokutime_to_seconds(bfe.io_time)*1000, - tokutime_to_seconds(bfe.decompress_time)*1000, - tokutime_to_seconds(bfe.deserialize_time)*1000, - deser_runs - ); + printf( + "deserialize leaf(ms): %0.05lf (average of %d runs)\n", dt, deser_runs); + printf( + "io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf " + "(average of %d runs)\n", + tokutime_to_seconds(bfe.io_time) * 1000, + tokutime_to_seconds(bfe.decompress_time) * 1000, + tokutime_to_seconds(bfe.deserialize_time) * 1000, + deser_runs); toku_ftnode_free(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); ft_h->cmp.destroy(); toku_free(ft_h->h); toku_free(ft_h); toku_free(ft); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -static void -test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int deser_runs) { +static void test_serialize_nonleaf(int valsize, + int nelts, + double entropy, + int ser_runs, + int deser_runs) { // struct ft_handle source_ft; struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -257,11 +270,11 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int MALLOC_N(sn.n_children, sn.bp); sn.pivotkeys.create_empty(); for (int i = 0; i < sn.n_children; ++i) { - BP_BLOCKNUM(&sn, i).b = 30 + (i*5); - BP_STATE(&sn,i) = PT_AVAIL; + BP_BLOCKNUM(&sn, i).b = 30 + (i * 5); + BP_STATE(&sn, i) = PT_AVAIL; set_BNC(&sn, i, toku_create_empty_nl()); } - //Create XIDS + // Create XIDS XIDS xids_0 = toku_xids_get_root_xids(); XIDS xids_123; r = toku_xids_create_child(xids_0, &xids_123, (TXNID)123); @@ -276,14 +289,23 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int k = ck * nperchild + i; char buf[valsize]; int c; - for (c = 0; c < valsize * entropy; ) { - int *p = (int *) &buf[c]; + for (c = 0; c < valsize * entropy;) { + int *p = (int *)&buf[c]; *p = rand(); c += sizeof(*p); } memset(&buf[c], 0, valsize - c); - toku_bnc_insert_msg(bnc, &k, sizeof k, buf, valsize, FT_NONE, next_dummymsn(), xids_123, true, cmp); + toku_bnc_insert_msg(bnc, + &k, + sizeof k, + buf, + valsize, + FT_NONE, + next_dummymsn(), + xids_123, + true, + cmp); } if (ck < 7) { DBT pivotkey; @@ -291,7 +313,7 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int } } - //Cleanup: + // Cleanup: toku_xids_destroy(&xids_0); toku_xids_destroy(&xids_123); cmp.destroy(); @@ -302,65 +324,78 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft_h->cmp.create(long_key_cmp, nullptr); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } struct timeval t[2]; gettimeofday(&t[0], NULL); FTNODE_DISK_DATA ndd = NULL; - r = toku_serialize_ftnode_to(fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); - assert(r==0); + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), &sn, &ndd, true, ft->ft, false); + invariant(r == 0); gettimeofday(&t[1], NULL); double dt; - dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC); + dt = (t[1].tv_sec - t[0].tv_sec) + + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC); dt *= 1000; - printf("serialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, ser_runs); + printf( + "serialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, ser_runs); ftnode_fetch_extra bfe; bfe.create_for_full_read(ft_h); gettimeofday(&t[0], NULL); FTNODE_DISK_DATA ndd2 = NULL; - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, &ndd2, &bfe); - assert(r==0); + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, &dn, &ndd2, &bfe); + invariant(r == 0); gettimeofday(&t[1], NULL); - dt = (t[1].tv_sec - t[0].tv_sec) + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC); + dt = (t[1].tv_sec - t[0].tv_sec) + + ((t[1].tv_usec - t[0].tv_usec) / USECS_PER_SEC); dt *= 1000; - printf("deserialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, deser_runs); - printf("io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf (IGNORED RUNS=%d)\n", - tokutime_to_seconds(bfe.io_time)*1000, - tokutime_to_seconds(bfe.decompress_time)*1000, - tokutime_to_seconds(bfe.deserialize_time)*1000, - deser_runs - ); + printf( + "deserialize nonleaf(ms): %0.05lf (IGNORED RUNS=%d)\n", dt, deser_runs); + printf( + "io time(ms) %lf decompress time(ms) %lf deserialize time(ms) %lf " + "(IGNORED RUNS=%d)\n", + tokutime_to_seconds(bfe.io_time) * 1000, + tokutime_to_seconds(bfe.decompress_time) * 1000, + tokutime_to_seconds(bfe.deserialize_time) * 1000, + deser_runs); toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); ft_h->cmp.destroy(); @@ -369,17 +404,21 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy, int ser_runs, int toku_free(ndd); toku_free(ndd2); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -int -test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { +int test_main(int argc __attribute__((__unused__)), + const char *argv[] __attribute__((__unused__))) { const int DEFAULT_RUNS = 5; long valsize, nelts, ser_runs = DEFAULT_RUNS, deser_runs = DEFAULT_RUNS; double entropy = 0.3; if (argc != 3 && argc != 5) { - fprintf(stderr, "Usage: %s [ ]\n", argv[0]); + fprintf(stderr, + "Usage: %s [ " + "]\n", + argv[0]); fprintf(stderr, "Default (and min) runs is %d\n", DEFAULT_RUNS); return 2; } diff --git a/storage/tokudb/PerconaFT/ft/tests/ft-serialize-test.cc b/storage/tokudb/PerconaFT/ft/tests/ft-serialize-test.cc index 332aaa0c170b1..0cddaf1965141 100644 --- a/storage/tokudb/PerconaFT/ft/tests/ft-serialize-test.cc +++ b/storage/tokudb/PerconaFT/ft/tests/ft-serialize-test.cc @@ -39,26 +39,20 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "test.h" #include "bndata.h" - - #ifndef MIN #define MIN(x, y) (((x) < (y)) ? (x) : (y)) #endif -static size_t -le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keysize, const char *val, int valsize) -{ +static size_t le_add_to_bn(bn_data *bn, + uint32_t idx, + const char *key, + int keysize, + const char *val, + int valsize) { LEAFENTRY r = NULL; uint32_t size_needed = LE_CLEAN_MEMSIZE(valsize); void *maybe_free = nullptr; - bn->get_space_for_insert( - idx, - key, - keysize, - size_needed, - &r, - &maybe_free - ); + bn->get_space_for_insert(idx, key, keysize, size_needed, &r, &maybe_free); if (maybe_free) { toku_free(maybe_free); } @@ -70,16 +64,19 @@ le_add_to_bn(bn_data* bn, uint32_t idx, const char *key, int keysize, const cha } class test_key_le_pair { - public: + public: uint32_t keylen; - char* keyp; + char *keyp; LEAFENTRY le; test_key_le_pair() : keylen(), keyp(), le() {} void init(const char *_keyp, const char *_val) { init(_keyp, strlen(_keyp) + 1, _val, strlen(_val) + 1); } - void init(const char * _keyp, uint32_t _keylen, const char*_val, uint32_t _vallen) { + void init(const char *_keyp, + uint32_t _keylen, + const char *_val, + uint32_t _vallen) { keylen = _keylen; CAST_FROM_VOIDP(le, toku_malloc(LE_CLEAN_MEMSIZE(_vallen))); @@ -95,126 +92,144 @@ class test_key_le_pair { } }; -enum ftnode_verify_type { - read_all=1, - read_compressed, - read_none -}; +enum ftnode_verify_type { read_all = 1, read_compressed, read_none }; -static int -string_key_cmp(DB *UU(e), const DBT *a, const DBT *b) -{ +static int string_key_cmp(DB *UU(e), const DBT *a, const DBT *b) { char *CAST_FROM_VOIDP(s, a->data); char *CAST_FROM_VOIDP(t, b->data); return strcmp(s, t); } -static void -setup_dn(enum ftnode_verify_type bft, int fd, FT ft_h, FTNODE *dn, FTNODE_DISK_DATA* ndd) { +static void setup_dn(enum ftnode_verify_type bft, + int fd, + FT ft_h, + FTNODE *dn, + FTNODE_DISK_DATA *ndd) { int r; if (bft == read_all) { ftnode_fetch_extra bfe; bfe.create_for_full_read(ft_h); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, ndd, &bfe); - assert(r==0); - } - else if (bft == read_compressed || bft == read_none) { + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, dn, ndd, &bfe); + invariant(r == 0); + } else if (bft == read_compressed || bft == read_none) { ftnode_fetch_extra bfe; bfe.create_for_min_read(ft_h); - r = toku_deserialize_ftnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, dn, ndd, &bfe); - assert(r==0); - // assert all bp's are compressed or on disk. + r = toku_deserialize_ftnode_from( + fd, make_blocknum(20), 0 /*pass zero for hash*/, dn, ndd, &bfe); + invariant(r == 0); + // invariant all bp's are compressed or on disk. for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_COMPRESSED || BP_STATE(*dn, i) == PT_ON_DISK); + invariant(BP_STATE(*dn, i) == PT_COMPRESSED || + BP_STATE(*dn, i) == PT_ON_DISK); } // if read_none, get rid of the compressed bp's if (bft == read_none) { if ((*dn)->height == 0) { - toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); - // assert all bp's are on disk + toku_ftnode_pe_callback(*dn, + make_pair_attr(0xffffffff), + ft_h, + def_pe_finalize_impl, + nullptr); + // invariant all bp's are on disk for (int i = 0; i < (*dn)->n_children; i++) { if ((*dn)->height == 0) { - assert(BP_STATE(*dn,i) == PT_ON_DISK); - assert(is_BNULL(*dn, i)); - } - else { - assert(BP_STATE(*dn,i) == PT_COMPRESSED); + invariant(BP_STATE(*dn, i) == PT_ON_DISK); + invariant(is_BNULL(*dn, i)); + } else { + invariant(BP_STATE(*dn, i) == PT_COMPRESSED); } } - } - else { + } else { // first decompress everything, and make sure // that it is available // then run partial eviction to get it compressed PAIR_ATTR attr; bfe.create_for_full_read(ft_h); - assert(toku_ftnode_pf_req_callback(*dn, &bfe)); + invariant(toku_ftnode_pf_req_callback(*dn, &bfe)); r = toku_ftnode_pf_callback(*dn, *ndd, &bfe, fd, &attr); - assert(r==0); - // assert all bp's are available + invariant(r == 0); + // invariant all bp's are available for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } - toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + toku_ftnode_pe_callback(*dn, + make_pair_attr(0xffffffff), + ft_h, + def_pe_finalize_impl, + nullptr); for (int i = 0; i < (*dn)->n_children; i++) { - // assert all bp's are still available, because we touched the clock - assert(BP_STATE(*dn,i) == PT_AVAIL); - // now assert all should be evicted - assert(BP_SHOULD_EVICT(*dn, i)); + // invariant all bp's are still available, because we touched + // the clock + invariant(BP_STATE(*dn, i) == PT_AVAIL); + // now invariant all should be evicted + invariant(BP_SHOULD_EVICT(*dn, i)); } - toku_ftnode_pe_callback(*dn, make_pair_attr(0xffffffff), ft_h, def_pe_finalize_impl, nullptr); + toku_ftnode_pe_callback(*dn, + make_pair_attr(0xffffffff), + ft_h, + def_pe_finalize_impl, + nullptr); for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_COMPRESSED); + invariant(BP_STATE(*dn, i) == PT_COMPRESSED); } } } // now decompress them bfe.create_for_full_read(ft_h); - assert(toku_ftnode_pf_req_callback(*dn, &bfe)); + invariant(toku_ftnode_pf_req_callback(*dn, &bfe)); PAIR_ATTR attr; r = toku_ftnode_pf_callback(*dn, *ndd, &bfe, fd, &attr); - assert(r==0); - // assert all bp's are available + invariant(r == 0); + // invariant all bp's are available for (int i = 0; i < (*dn)->n_children; i++) { - assert(BP_STATE(*dn,i) == PT_AVAIL); + invariant(BP_STATE(*dn, i) == PT_AVAIL); } // continue on with test - } - else { + } else { // if we get here, this is a test bug, NOT a bug in development code - assert(false); + invariant(false); } } -static void write_sn_to_disk(int fd, FT_HANDLE ft, FTNODE sn, FTNODE_DISK_DATA* src_ndd, bool do_clone) { +static void write_sn_to_disk(int fd, + FT_HANDLE ft, + FTNODE sn, + FTNODE_DISK_DATA *src_ndd, + bool do_clone) { int r; if (do_clone) { - void* cloned_node_v = NULL; + void *cloned_node_v = NULL; PAIR_ATTR attr; long clone_size; - toku_ftnode_clone_callback(sn, &cloned_node_v, &clone_size, &attr, false, ft->ft); + toku_ftnode_clone_callback( + sn, &cloned_node_v, &clone_size, &attr, false, ft->ft); FTNODE CAST_FROM_VOIDP(cloned_node, cloned_node_v); - r = toku_serialize_ftnode_to(fd, make_blocknum(20), cloned_node, src_ndd, false, ft->ft, false); - assert(r==0); + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), cloned_node, src_ndd, false, ft->ft, false); + invariant(r == 0); toku_ftnode_free(&cloned_node); - } - else { - r = toku_serialize_ftnode_to(fd, make_blocknum(20), sn, src_ndd, true, ft->ft, false); - assert(r==0); + } else { + r = toku_serialize_ftnode_to( + fd, make_blocknum(20), sn, src_ndd, true, ft->ft, false); + invariant(r == 0); } } -static void -test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_leaf_check_msn(enum ftnode_verify_type bft, + bool do_clone) { // struct ft_handle source_ft; struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; -#define PRESERIALIZE_MSN_ON_DISK ((MSN) { MIN_MSN.msn + 42 }) -#define POSTSERIALIZE_MSN_ON_DISK ((MSN) { MIN_MSN.msn + 84 }) +#define PRESERIALIZE_MSN_ON_DISK ((MSN){MIN_MSN.msn + 42}) +#define POSTSERIALIZE_MSN_ON_DISK ((MSN){MIN_MSN.msn + 84}) sn.max_msn_applied_to_node_on_disk = PRESERIALIZE_MSN_ON_DISK; sn.flags = 0x11223344; @@ -228,14 +243,14 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { MALLOC_N(sn.n_children, sn.bp); DBT pivotkey; sn.pivotkeys.create_from_dbts(toku_fill_dbt(&pivotkey, "b", 2), 1); - BP_STATE(&sn,0) = PT_AVAIL; - BP_STATE(&sn,1) = PT_AVAIL; + BP_STATE(&sn, 0) = PT_AVAIL; + BP_STATE(&sn, 1) = PT_AVAIL; set_BLB(&sn, 0, toku_create_empty_bn()); set_BLB(&sn, 1, toku_create_empty_bn()); le_add_to_bn(BLB_DATA(&sn, 0), 0, "a", 2, "aval", 5); le_add_to_bn(BLB_DATA(&sn, 0), 1, "b", 2, "bval", 5); le_add_to_bn(BLB_DATA(&sn, 1), 0, "x", 2, "xval", 5); - BLB_MAX_MSN_APPLIED(&sn, 0) = ((MSN) { MIN_MSN.msn + 73 }); + BLB_MAX_MSN_APPLIED(&sn, 0) = ((MSN){MIN_MSN.msn + 73}); BLB_MAX_MSN_APPLIED(&sn, 1) = POSTSERIALIZE_MSN_ON_DISK; FT_HANDLE XMALLOC(ft); @@ -244,30 +259,35 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } - //Want to use block #20 + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; FTNODE_DISK_DATA dest_ndd = NULL; @@ -276,16 +296,18 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - assert(dn->blocknum.b==20); + invariant(dn->blocknum.b == 20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); - assert(dn->layout_version_read_from_disk ==FT_LAYOUT_VERSION); - assert(dn->height == 0); - assert(dn->n_children>=1); - assert(dn->max_msn_applied_to_node_on_disk.msn == POSTSERIALIZE_MSN_ON_DISK.msn); + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); + invariant(dn->layout_version_read_from_disk == FT_LAYOUT_VERSION); + invariant(dn->height == 0); + invariant(dn->n_children >= 1); + invariant(dn->max_msn_applied_to_node_on_disk.msn == + POSTSERIALIZE_MSN_ON_DISK.msn); { - // Man, this is way too ugly. This entire test suite needs to be refactored. + // Man, this is way too ugly. This entire test suite needs to be + // refactored. // Create a dummy mempool and put the leaves there. Ugh. test_key_le_pair elts[3]; elts[0].init("a", "aval"); @@ -294,34 +316,41 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { const uint32_t npartitions = dn->n_children; uint32_t last_i = 0; for (uint32_t bn = 0; bn < npartitions; ++bn) { - assert(BLB_MAX_MSN_APPLIED(dn, bn).msn == POSTSERIALIZE_MSN_ON_DISK.msn); - assert(dest_ndd[bn].start > 0); - assert(dest_ndd[bn].size > 0); + invariant(BLB_MAX_MSN_APPLIED(dn, bn).msn == + POSTSERIALIZE_MSN_ON_DISK.msn); + invariant(dest_ndd[bn].start > 0); + invariant(dest_ndd[bn].size > 0); if (bn > 0) { - assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size); + invariant(dest_ndd[bn].start >= + dest_ndd[bn - 1].start + dest_ndd[bn - 1].size); } for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) { LEAFENTRY curr_le; uint32_t curr_keylen; - void* curr_key; - BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); - assert(leafentry_memsize(curr_le) == leafentry_memsize(elts[last_i].le)); - assert(memcmp(curr_le, elts[last_i].le, leafentry_memsize(curr_le)) == 0); - if (bn < npartitions-1) { - assert(strcmp((char*)dn->pivotkeys.get_pivot(bn).data, elts[last_i].keyp) <= 0); + void *curr_key; + BLB_DATA(dn, bn) + ->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); + invariant(leafentry_memsize(curr_le) == + leafentry_memsize(elts[last_i].le)); + invariant(memcmp(curr_le, + elts[last_i].le, + leafentry_memsize(curr_le)) == 0); + if (bn < npartitions - 1) { + invariant(strcmp((char *)dn->pivotkeys.get_pivot(bn).data, + elts[last_i].keyp) <= 0); } // TODO for later, get a key comparison here as well last_i++; } - } - assert(last_i == 3); + invariant(last_i == 3); } toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); @@ -329,17 +358,21 @@ test_serialize_leaf_check_msn(enum ftnode_verify_type bft, bool do_clone) { toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -static void -test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, + bool do_clone) { int r; struct ftnode sn, *dn; - const int keylens = 256*1024, vallens = 0; + const int keylens = 256 * 1024, vallens = 0; const uint32_t nrows = 8; - // assert(val_size > BN_MAX_SIZE); // BN_MAX_SIZE isn't visible - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + // invariant(val_size > BN_MAX_SIZE); // BN_MAX_SIZE isn't visible + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); sn.max_msn_applied_to_node_on_disk.msn = 0; sn.flags = 0x11223344; @@ -354,21 +387,27 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone MALLOC_N(sn.n_children, sn.bp); sn.pivotkeys.create_empty(); for (int i = 0; i < sn.n_children; ++i) { - BP_STATE(&sn,i) = PT_AVAIL; + BP_STATE(&sn, i) = PT_AVAIL; set_BLB(&sn, i, toku_create_empty_bn()); } for (uint32_t i = 0; i < nrows; ++i) { // one basement per row char key[keylens], val[vallens]; - key[keylens-1] = '\0'; + key[keylens - 1] = '\0'; char c = 'a' + i; - memset(key, c, keylens-1); - le_add_to_bn(BLB_DATA(&sn, i), 0, (char *) &key, sizeof(key), (char *) &val, sizeof(val)); - if (i < nrows-1) { + memset(key, c, keylens - 1); + le_add_to_bn(BLB_DATA(&sn, i), + 0, + (char *)&key, + sizeof(key), + (char *)&val, + sizeof(val)); + if (i < nrows - 1) { uint32_t keylen; - void* curr_key; + void *curr_key; BLB_DATA(&sn, i)->fetch_key_and_len(0, &keylen, &curr_key); DBT pivotkey; - sn.pivotkeys.insert_at(toku_fill_dbt(&pivotkey, curr_key, keylen), i); + sn.pivotkeys.insert_at(toku_fill_dbt(&pivotkey, curr_key, keylen), + i); } } @@ -378,29 +417,34 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; FTNODE_DISK_DATA dest_ndd = NULL; @@ -408,55 +452,64 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone write_sn_to_disk(fd, ft, &sn, &src_ndd, do_clone); setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - - assert(dn->blocknum.b==20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); + invariant(dn->blocknum.b == 20); + + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); { - // Man, this is way too ugly. This entire test suite needs to be refactored. + // Man, this is way too ugly. This entire test suite needs to be + // refactored. // Create a dummy mempool and put the leaves there. Ugh. test_key_le_pair *les = new test_key_le_pair[nrows]; { char key[keylens], val[vallens]; - key[keylens-1] = '\0'; + key[keylens - 1] = '\0'; for (uint32_t i = 0; i < nrows; ++i) { char c = 'a' + i; - memset(key, c, keylens-1); - les[i].init((char *) &key, sizeof(key), (char *) &val, sizeof(val)); + memset(key, c, keylens - 1); + les[i].init( + (char *)&key, sizeof(key), (char *)&val, sizeof(val)); } } const uint32_t npartitions = dn->n_children; uint32_t last_i = 0; for (uint32_t bn = 0; bn < npartitions; ++bn) { - assert(dest_ndd[bn].start > 0); - assert(dest_ndd[bn].size > 0); + invariant(dest_ndd[bn].start > 0); + invariant(dest_ndd[bn].size > 0); if (bn > 0) { - assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size); + invariant(dest_ndd[bn].start >= + dest_ndd[bn - 1].start + dest_ndd[bn - 1].size); } - assert(BLB_DATA(dn, bn)->num_klpairs() > 0); + invariant(BLB_DATA(dn, bn)->num_klpairs() > 0); for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) { LEAFENTRY curr_le; uint32_t curr_keylen; - void* curr_key; - BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); - assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le)); - assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0); - if (bn < npartitions-1) { - assert(strcmp((char*)dn->pivotkeys.get_pivot(bn).data, les[last_i].keyp) <= 0); + void *curr_key; + BLB_DATA(dn, bn) + ->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); + invariant(leafentry_memsize(curr_le) == + leafentry_memsize(les[last_i].le)); + invariant(memcmp(curr_le, + les[last_i].le, + leafentry_memsize(curr_le)) == 0); + if (bn < npartitions - 1) { + invariant(strcmp((char *)dn->pivotkeys.get_pivot(bn).data, + les[last_i].keyp) <= 0); } // TODO for later, get a key comparison here as well last_i++; } } - assert(last_i == nrows); + invariant(last_i == nrows); delete[] les; } toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); @@ -464,15 +517,19 @@ test_serialize_leaf_with_large_pivots(enum ftnode_verify_type bft, bool do_clone toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -static void -test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, + bool do_clone) { int r; struct ftnode sn, *dn; - const uint32_t nrows = 196*1024; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + const uint32_t nrows = 196 * 1024; + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); sn.max_msn_applied_to_node_on_disk.msn = 0; sn.flags = 0x11223344; @@ -487,14 +544,19 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) { XMALLOC_N(sn.n_children, sn.bp); sn.pivotkeys.create_empty(); for (int i = 0; i < sn.n_children; ++i) { - BP_STATE(&sn,i) = PT_AVAIL; - set_BLB(&sn, i, toku_create_empty_bn()); + BP_STATE(&sn, i) = PT_AVAIL; + set_BLB(&sn, i, toku_create_empty_bn()); } size_t total_size = 0; for (uint32_t i = 0; i < nrows; ++i) { uint32_t key = i; uint32_t val = i; - total_size += le_add_to_bn(BLB_DATA(&sn, 0), i, (char *) &key, sizeof(key), (char *) &val, sizeof(val)); + total_size += le_add_to_bn(BLB_DATA(&sn, 0), + i, + (char *)&key, + sizeof(key), + (char *)&val, + sizeof(val)); } FT_HANDLE XMALLOC(ft); @@ -503,30 +565,35 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) { make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; @@ -535,56 +602,66 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) { setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - assert(dn->blocknum.b==20); + invariant(dn->blocknum.b == 20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); { - // Man, this is way too ugly. This entire test suite needs to be refactored. + // Man, this is way too ugly. This entire test suite needs to be + // refactored. // Create a dummy mempool and put the leaves there. Ugh. test_key_le_pair *les = new test_key_le_pair[nrows]; { int key = 0, val = 0; for (uint32_t i = 0; i < nrows; ++i, key++, val++) { - les[i].init((char *) &key, sizeof(key), (char *) &val, sizeof(val)); + les[i].init( + (char *)&key, sizeof(key), (char *)&val, sizeof(val)); } } const uint32_t npartitions = dn->n_children; uint32_t last_i = 0; for (uint32_t bn = 0; bn < npartitions; ++bn) { - assert(dest_ndd[bn].start > 0); - assert(dest_ndd[bn].size > 0); + invariant(dest_ndd[bn].start > 0); + invariant(dest_ndd[bn].size > 0); if (bn > 0) { - assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size); + invariant(dest_ndd[bn].start >= + dest_ndd[bn - 1].start + dest_ndd[bn - 1].size); } - assert(BLB_DATA(dn, bn)->num_klpairs() > 0); + invariant(BLB_DATA(dn, bn)->num_klpairs() > 0); for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) { LEAFENTRY curr_le; uint32_t curr_keylen; - void* curr_key; - BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); - assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le)); - assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0); - if (bn < npartitions-1) { - uint32_t *CAST_FROM_VOIDP(pivot, dn->pivotkeys.get_pivot(bn).data); - void* tmp = les[last_i].keyp; + void *curr_key; + BLB_DATA(dn, bn) + ->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); + invariant(leafentry_memsize(curr_le) == + leafentry_memsize(les[last_i].le)); + invariant(memcmp(curr_le, + les[last_i].le, + leafentry_memsize(curr_le)) == 0); + if (bn < npartitions - 1) { + uint32_t *CAST_FROM_VOIDP(pivot, + dn->pivotkeys.get_pivot(bn).data); + void *tmp = les[last_i].keyp; uint32_t *CAST_FROM_VOIDP(item, tmp); - assert(*pivot >= *item); + invariant(*pivot >= *item); } // TODO for later, get a key comparison here as well last_i++; } // don't check soft_copy_is_up_to_date or seqinsert - assert(BLB_DATA(dn, bn)->get_disk_size() < 128*1024); // BN_MAX_SIZE, apt to change + invariant(BLB_DATA(dn, bn)->get_disk_size() < + 128 * 1024); // BN_MAX_SIZE, apt to change } - assert(last_i == nrows); + invariant(last_i == nrows); delete[] les; } toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); @@ -592,19 +669,22 @@ test_serialize_leaf_with_many_rows(enum ftnode_verify_type bft, bool do_clone) { toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } - -static void -test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, + bool do_clone) { int r; struct ftnode sn, *dn; const uint32_t nrows = 7; const size_t key_size = 8; - const size_t val_size = 512*1024; - // assert(val_size > BN_MAX_SIZE); // BN_MAX_SIZE isn't visible - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + const size_t val_size = 512 * 1024; + // invariant(val_size > BN_MAX_SIZE); // BN_MAX_SIZE isn't visible + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); sn.max_msn_applied_to_node_on_disk.msn = 0; sn.flags = 0x11223344; @@ -615,21 +695,21 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone) sn.n_children = 1; sn.dirty = 1; sn.oldest_referenced_xid_known = TXNID_NONE; - + MALLOC_N(sn.n_children, sn.bp); sn.pivotkeys.create_empty(); for (int i = 0; i < sn.n_children; ++i) { - BP_STATE(&sn,i) = PT_AVAIL; + BP_STATE(&sn, i) = PT_AVAIL; set_BLB(&sn, i, toku_create_empty_bn()); } for (uint32_t i = 0; i < nrows; ++i) { char key[key_size], val[val_size]; - key[key_size-1] = '\0'; - val[val_size-1] = '\0'; + key[key_size - 1] = '\0'; + val[val_size - 1] = '\0'; char c = 'a' + i; - memset(key, c, key_size-1); - memset(val, c, val_size-1); - le_add_to_bn(BLB_DATA(&sn, 0), i,key, 8, val, val_size); + memset(key, c, key_size - 1); + memset(val, c, val_size - 1); + le_add_to_bn(BLB_DATA(&sn, 0), i, key, 8, val, val_size); } FT_HANDLE XMALLOC(ft); @@ -638,30 +718,35 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone) make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; @@ -670,58 +755,66 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone) setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - assert(dn->blocknum.b==20); + invariant(dn->blocknum.b == 20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); { - // Man, this is way too ugly. This entire test suite needs to be refactored. + // Man, this is way too ugly. This entire test suite needs to be + // refactored. // Create a dummy mempool and put the leaves there. Ugh. test_key_le_pair *les = new test_key_le_pair[nrows]; { char key[key_size], val[val_size]; - key[key_size-1] = '\0'; - val[val_size-1] = '\0'; + key[key_size - 1] = '\0'; + val[val_size - 1] = '\0'; for (uint32_t i = 0; i < nrows; ++i) { char c = 'a' + i; - memset(key, c, key_size-1); - memset(val, c, val_size-1); + memset(key, c, key_size - 1); + memset(val, c, val_size - 1); les[i].init(key, key_size, val, val_size); } } const uint32_t npartitions = dn->n_children; - assert(npartitions == nrows); + invariant(npartitions == nrows); uint32_t last_i = 0; for (uint32_t bn = 0; bn < npartitions; ++bn) { - assert(dest_ndd[bn].start > 0); - assert(dest_ndd[bn].size > 0); + invariant(dest_ndd[bn].start > 0); + invariant(dest_ndd[bn].size > 0); if (bn > 0) { - assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size); + invariant(dest_ndd[bn].start >= + dest_ndd[bn - 1].start + dest_ndd[bn - 1].size); } - assert(BLB_DATA(dn, bn)->num_klpairs() > 0); + invariant(BLB_DATA(dn, bn)->num_klpairs() > 0); for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) { LEAFENTRY curr_le; uint32_t curr_keylen; - void* curr_key; - BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); - assert(leafentry_memsize(curr_le) == leafentry_memsize(les[last_i].le)); - assert(memcmp(curr_le, les[last_i].le, leafentry_memsize(curr_le)) == 0); - if (bn < npartitions-1) { - assert(strcmp((char*)dn->pivotkeys.get_pivot(bn).data, (char*)(les[last_i].keyp)) <= 0); + void *curr_key; + BLB_DATA(dn, bn) + ->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); + invariant(leafentry_memsize(curr_le) == + leafentry_memsize(les[last_i].le)); + invariant(memcmp(curr_le, + les[last_i].le, + leafentry_memsize(curr_le)) == 0); + if (bn < npartitions - 1) { + invariant(strcmp((char *)dn->pivotkeys.get_pivot(bn).data, + (char *)(les[last_i].keyp)) <= 0); } // TODO for later, get a key comparison here as well last_i++; } // don't check soft_copy_is_up_to_date or seqinsert } - assert(last_i == 7); + invariant(last_i == 7); delete[] les; } toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); @@ -729,15 +822,19 @@ test_serialize_leaf_with_large_rows(enum ftnode_verify_type bft, bool do_clone) toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } - -static void -test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_leaf_with_empty_basement_nodes( + enum ftnode_verify_type bft, + bool do_clone) { struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -760,7 +857,7 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool toku_fill_dbt(&pivotkeys[5], "x", 2); sn.pivotkeys.create_from_dbts(pivotkeys, 6); for (int i = 0; i < sn.n_children; ++i) { - BP_STATE(&sn,i) = PT_AVAIL; + BP_STATE(&sn, i) = PT_AVAIL; set_BLB(&sn, i, toku_create_empty_bn()); BLB_SEQINSERT(&sn, i) = 0; } @@ -774,30 +871,35 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; FTNODE_DISK_DATA dest_ndd = NULL; @@ -805,17 +907,18 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - assert(dn->blocknum.b==20); + invariant(dn->blocknum.b == 20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); - assert(dn->layout_version_read_from_disk ==FT_LAYOUT_VERSION); - assert(dn->height == 0); - assert(dn->n_children>0); + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); + invariant(dn->layout_version_read_from_disk == FT_LAYOUT_VERSION); + invariant(dn->height == 0); + invariant(dn->n_children > 0); { test_key_le_pair elts[3]; - // Man, this is way too ugly. This entire test suite needs to be refactored. + // Man, this is way too ugly. This entire test suite needs to be + // refactored. // Create a dummy mempool and put the leaves there. Ugh. elts[0].init("a", "aval"); elts[1].init("b", "bval"); @@ -823,33 +926,39 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool const uint32_t npartitions = dn->n_children; uint32_t last_i = 0; for (uint32_t bn = 0; bn < npartitions; ++bn) { - assert(dest_ndd[bn].start > 0); - assert(dest_ndd[bn].size > 0); + invariant(dest_ndd[bn].start > 0); + invariant(dest_ndd[bn].size > 0); if (bn > 0) { - assert(dest_ndd[bn].start >= dest_ndd[bn-1].start + dest_ndd[bn-1].size); + invariant(dest_ndd[bn].start >= + dest_ndd[bn - 1].start + dest_ndd[bn - 1].size); } for (uint32_t i = 0; i < BLB_DATA(dn, bn)->num_klpairs(); i++) { LEAFENTRY curr_le; uint32_t curr_keylen; - void* curr_key; - BLB_DATA(dn, bn)->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); - assert(leafentry_memsize(curr_le) == leafentry_memsize(elts[last_i].le)); - assert(memcmp(curr_le, elts[last_i].le, leafentry_memsize(curr_le)) == 0); - if (bn < npartitions-1) { - assert(strcmp((char*)dn->pivotkeys.get_pivot(bn).data, (char*)(elts[last_i].keyp)) <= 0); + void *curr_key; + BLB_DATA(dn, bn) + ->fetch_klpair(i, &curr_le, &curr_keylen, &curr_key); + invariant(leafentry_memsize(curr_le) == + leafentry_memsize(elts[last_i].le)); + invariant(memcmp(curr_le, + elts[last_i].le, + leafentry_memsize(curr_le)) == 0); + if (bn < npartitions - 1) { + invariant(strcmp((char *)dn->pivotkeys.get_pivot(bn).data, + (char *)(elts[last_i].keyp)) <= 0); } // TODO for later, get a key comparison here as well last_i++; } - } - assert(last_i == 3); + invariant(last_i == 3); } toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); @@ -857,14 +966,19 @@ test_serialize_leaf_with_empty_basement_nodes(enum ftnode_verify_type bft, bool toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -static void -test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_leaf_with_multiple_empty_basement_nodes( + enum ftnode_verify_type bft, + bool do_clone) { struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -884,7 +998,7 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b toku_fill_dbt(&pivotkeys[2], "A", 2); sn.pivotkeys.create_from_dbts(pivotkeys, 3); for (int i = 0; i < sn.n_children; ++i) { - BP_STATE(&sn,i) = PT_AVAIL; + BP_STATE(&sn, i) = PT_AVAIL; set_BLB(&sn, i, toku_create_empty_bn()); } @@ -894,30 +1008,35 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; @@ -926,29 +1045,31 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - assert(dn->blocknum.b==20); + invariant(dn->blocknum.b == 20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); - assert(dn->layout_version_read_from_disk ==FT_LAYOUT_VERSION); - assert(dn->height == 0); - assert(dn->n_children == 1); + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); + invariant(dn->layout_version_read_from_disk == FT_LAYOUT_VERSION); + invariant(dn->height == 0); + invariant(dn->n_children == 1); { const uint32_t npartitions = dn->n_children; for (uint32_t i = 0; i < npartitions; ++i) { - assert(dest_ndd[i].start > 0); - assert(dest_ndd[i].size > 0); + invariant(dest_ndd[i].start > 0); + invariant(dest_ndd[i].size > 0); if (i > 0) { - assert(dest_ndd[i].start >= dest_ndd[i-1].start + dest_ndd[i-1].size); + invariant(dest_ndd[i].start >= + dest_ndd[i - 1].start + dest_ndd[i - 1].size); } - assert(BLB_DATA(dn, i)->num_klpairs() == 0); + invariant(BLB_DATA(dn, i)->num_klpairs() == 0); } } - + toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); toku_free(ft_h->h); toku_free(ft_h); @@ -956,16 +1077,18 @@ test_serialize_leaf_with_multiple_empty_basement_nodes(enum ftnode_verify_type b toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } - -static void -test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { +static void test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { // struct ft_handle source_ft; struct ftnode sn, *dn; - int fd = open(TOKU_TEST_FILENAME, O_RDWR|O_CREAT|O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd >= 0); + int fd = open(TOKU_TEST_FILENAME, + O_RDWR | O_CREAT | O_BINARY, + S_IRWXU | S_IRWXG | S_IRWXO); + invariant(fd >= 0); int r; @@ -984,11 +1107,11 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { sn.pivotkeys.create_from_dbts(toku_fill_dbt(&pivotkey, "hello", 6), 1); BP_BLOCKNUM(&sn, 0).b = 30; BP_BLOCKNUM(&sn, 1).b = 35; - BP_STATE(&sn,0) = PT_AVAIL; - BP_STATE(&sn,1) = PT_AVAIL; + BP_STATE(&sn, 0) = PT_AVAIL; + BP_STATE(&sn, 1) = PT_AVAIL; set_BNC(&sn, 0, toku_create_empty_nl()); set_BNC(&sn, 1, toku_create_empty_nl()); - //Create XIDS + // Create XIDS XIDS xids_0 = toku_xids_get_root_xids(); XIDS xids_123; XIDS xids_234; @@ -1000,11 +1123,38 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { toku::comparator cmp; cmp.create(string_key_cmp, nullptr); - toku_bnc_insert_msg(BNC(&sn, 0), "a", 2, "aval", 5, FT_NONE, next_dummymsn(), xids_0, true, cmp); - toku_bnc_insert_msg(BNC(&sn, 0), "b", 2, "bval", 5, FT_NONE, next_dummymsn(), xids_123, false, cmp); - toku_bnc_insert_msg(BNC(&sn, 1), "x", 2, "xval", 5, FT_NONE, next_dummymsn(), xids_234, true, cmp); - - //Cleanup: + toku_bnc_insert_msg(BNC(&sn, 0), + "a", + 2, + "aval", + 5, + FT_NONE, + next_dummymsn(), + xids_0, + true, + cmp); + toku_bnc_insert_msg(BNC(&sn, 0), + "b", + 2, + "bval", + 5, + FT_NONE, + next_dummymsn(), + xids_123, + false, + cmp); + toku_bnc_insert_msg(BNC(&sn, 1), + "x", + 2, + "xval", + 5, + FT_NONE, + next_dummymsn(), + xids_234, + true, + cmp); + + // Cleanup: toku_xids_destroy(&xids_0); toku_xids_destroy(&xids_123); toku_xids_destroy(&xids_234); @@ -1016,31 +1166,36 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { make_blocknum(0), ZERO_LSN, TXNID_NONE, - 4*1024*1024, - 128*1024, + 4 * 1024 * 1024, + 128 * 1024, TOKU_DEFAULT_COMPRESSION_METHOD, 16); ft_h->cmp.create(string_key_cmp, nullptr); ft->ft = ft_h; - + ft_h->blocktable.create(); - { int r_truncate = ftruncate(fd, 0); CKERR(r_truncate); } - //Want to use block #20 + { + int r_truncate = ftruncate(fd, 0); + CKERR(r_truncate); + } + // Want to use block #20 BLOCKNUM b = make_blocknum(0); while (b.b < 20) { ft_h->blocktable.allocate_blocknum(&b, ft_h); } - assert(b.b == 20); + invariant(b.b == 20); { DISKOFF offset; DISKOFF size; - ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false, 0); - assert(offset==(DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.realloc_on_disk(b, 100, &offset, ft_h, fd, false); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); ft_h->blocktable.translate_blocknum_to_offset_size(b, &offset, &size); - assert(offset == (DISKOFF)block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); - assert(size == 100); + invariant(offset == + (DISKOFF)BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + invariant(size == 100); } FTNODE_DISK_DATA src_ndd = NULL; FTNODE_DISK_DATA dest_ndd = NULL; @@ -1048,30 +1203,31 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { setup_dn(bft, fd, ft_h, &dn, &dest_ndd); - assert(dn->blocknum.b==20); + invariant(dn->blocknum.b == 20); - assert(dn->layout_version ==FT_LAYOUT_VERSION); - assert(dn->layout_version_original ==FT_LAYOUT_VERSION); - assert(dn->layout_version_read_from_disk ==FT_LAYOUT_VERSION); - assert(dn->height == 1); - assert(dn->n_children==2); - assert(strcmp((char*)dn->pivotkeys.get_pivot(0).data, "hello")==0); - assert(dn->pivotkeys.get_pivot(0).size==6); - assert(BP_BLOCKNUM(dn,0).b==30); - assert(BP_BLOCKNUM(dn,1).b==35); + invariant(dn->layout_version == FT_LAYOUT_VERSION); + invariant(dn->layout_version_original == FT_LAYOUT_VERSION); + invariant(dn->layout_version_read_from_disk == FT_LAYOUT_VERSION); + invariant(dn->height == 1); + invariant(dn->n_children == 2); + invariant(strcmp((char *)dn->pivotkeys.get_pivot(0).data, "hello") == 0); + invariant(dn->pivotkeys.get_pivot(0).size == 6); + invariant(BP_BLOCKNUM(dn, 0).b == 30); + invariant(BP_BLOCKNUM(dn, 1).b == 35); message_buffer *src_msg_buffer1 = &BNC(&sn, 0)->msg_buffer; message_buffer *src_msg_buffer2 = &BNC(&sn, 1)->msg_buffer; message_buffer *dest_msg_buffer1 = &BNC(dn, 0)->msg_buffer; message_buffer *dest_msg_buffer2 = &BNC(dn, 1)->msg_buffer; - assert(src_msg_buffer1->equals(dest_msg_buffer1)); - assert(src_msg_buffer2->equals(dest_msg_buffer2)); + invariant(src_msg_buffer1->equals(dest_msg_buffer1)); + invariant(src_msg_buffer2->equals(dest_msg_buffer2)); toku_ftnode_free(&dn); toku_destroy_ftnode_internals(&sn); - ft_h->blocktable.block_free(block_allocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE); + ft_h->blocktable.block_free( + BlockAllocator::BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE, 100); ft_h->blocktable.destroy(); ft_h->cmp.destroy(); toku_free(ft_h->h); @@ -1080,11 +1236,12 @@ test_serialize_nonleaf(enum ftnode_verify_type bft, bool do_clone) { toku_free(src_ndd); toku_free(dest_ndd); - r = close(fd); assert(r != -1); + r = close(fd); + invariant(r != -1); } -int -test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { +int test_main(int argc __attribute__((__unused__)), + const char *argv[] __attribute__((__unused__))) { initialize_dummymsn(); test_serialize_nonleaf(read_none, false); @@ -1103,10 +1260,12 @@ test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute_ test_serialize_leaf_with_multiple_empty_basement_nodes(read_none, false); test_serialize_leaf_with_multiple_empty_basement_nodes(read_all, false); - test_serialize_leaf_with_multiple_empty_basement_nodes(read_compressed, false); + test_serialize_leaf_with_multiple_empty_basement_nodes(read_compressed, + false); test_serialize_leaf_with_multiple_empty_basement_nodes(read_none, true); test_serialize_leaf_with_multiple_empty_basement_nodes(read_all, true); - test_serialize_leaf_with_multiple_empty_basement_nodes(read_compressed, true); + test_serialize_leaf_with_multiple_empty_basement_nodes(read_compressed, + true); test_serialize_leaf_with_empty_basement_nodes(read_none, false); test_serialize_leaf_with_empty_basement_nodes(read_all, false); diff --git a/storage/tokudb/PerconaFT/ft/tests/ft-test.cc b/storage/tokudb/PerconaFT/ft/tests/ft-test.cc index 598a1cc7085c2..706bd94fbc3af 100644 --- a/storage/tokudb/PerconaFT/ft/tests/ft-test.cc +++ b/storage/tokudb/PerconaFT/ft/tests/ft-test.cc @@ -164,17 +164,16 @@ static void test_read_what_was_written (void) { int r; const int NVALS=10000; - if (verbose) printf("test_read_what_was_written(): "); fflush(stdout); + if (verbose) { + printf("test_read_what_was_written(): "); fflush(stdout); + } unlink(fname); - toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0); r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0); - toku_cachetable_close(&ct); - - + toku_cachetable_close(&ct); /* Now see if we can read an empty tree in. */ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); @@ -189,8 +188,6 @@ static void test_read_what_was_written (void) { r = toku_close_ft_handle_nolsn(ft, 0); assert(r==0); toku_cachetable_close(&ct); - - /* Now see if we can read it in and get the value. */ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); assert(r==0); diff --git a/storage/tokudb/PerconaFT/ft/tests/pqueue-test.cc b/storage/tokudb/PerconaFT/ft/tests/pqueue-test.cc index 53973794eae74..aeb5a897c488f 100644 --- a/storage/tokudb/PerconaFT/ft/tests/pqueue-test.cc +++ b/storage/tokudb/PerconaFT/ft/tests/pqueue-test.cc @@ -109,7 +109,9 @@ static int run_test(void) r = pqueue_pop(pq, &node); assert(r==0); if (verbose) printf("%d : %d\n", i, *(int*)(node->key->data)); if ( *(int*)(node->key->data) != i ) { - if (verbose) printf("FAIL\n"); return -1; + if (verbose) + printf("FAIL\n"); + return -1; } } pqueue_free(pq); diff --git a/storage/tokudb/PerconaFT/ft/tests/test-leafentry-nested.cc b/storage/tokudb/PerconaFT/ft/tests/test-leafentry-nested.cc index a78f787cdf299..f200496486250 100644 --- a/storage/tokudb/PerconaFT/ft/tests/test-leafentry-nested.cc +++ b/storage/tokudb/PerconaFT/ft/tests/test-leafentry-nested.cc @@ -793,7 +793,7 @@ static void test_le_garbage_collection_birdie(void) { do_garbage_collect = ule_worth_running_garbage_collection(&ule, 200); invariant(do_garbage_collect); - // It is definately worth doing when the above case is true + // It is definitely worth doing when the above case is true // and there is more than one provisional entry. ule.num_cuxrs = 1; ule.num_puxrs = 2; diff --git a/storage/tokudb/PerconaFT/ft/tests/test-oldest-referenced-xid-flush.cc b/storage/tokudb/PerconaFT/ft/tests/test-oldest-referenced-xid-flush.cc index 419af550545c4..71357a1e16ad2 100644 --- a/storage/tokudb/PerconaFT/ft/tests/test-oldest-referenced-xid-flush.cc +++ b/storage/tokudb/PerconaFT/ft/tests/test-oldest-referenced-xid-flush.cc @@ -72,7 +72,7 @@ static void dummy_update_status(FTNODE UU(child), int UU(dirtied), void* UU(extr enum { NODESIZE = 1024, KSIZE=NODESIZE-100, TOKU_PSIZE=20 }; -static void test_oldest_referenced_xid_gets_propogated(void) { +static void test_oldest_referenced_xid_gets_propagated(void) { int r; CACHETABLE ct; FT_HANDLE t; @@ -166,7 +166,7 @@ static void test_oldest_referenced_xid_gets_propogated(void) { toku_ft_flush_some_child(t->ft, node, &fa); // pin the child, verify that oldest referenced xid was - // propogated from parent to child during the flush + // propagated from parent to child during the flush toku_pin_ftnode( t->ft, child_nonleaf_blocknum, @@ -185,6 +185,6 @@ static void test_oldest_referenced_xid_gets_propogated(void) { int test_main(int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) { default_parse_args(argc, argv); - test_oldest_referenced_xid_gets_propogated(); + test_oldest_referenced_xid_gets_propagated(); return 0; } diff --git a/storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.h b/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-with-mhs.cc similarity index 55% rename from storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.h rename to storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-with-mhs.cc index 8aded3898c1de..ea4f9374dc336 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/block_allocator_strategy.h +++ b/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-with-mhs.cc @@ -36,30 +36,62 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#pragma once - -#include - -#include "ft/serialize/block_allocator.h" - -// Block allocation strategy implementations - -class block_allocator_strategy { -public: - static struct block_allocator::blockpair * - first_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment); - - static struct block_allocator::blockpair * - best_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment); - - static struct block_allocator::blockpair * - padded_fit(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment); - - static struct block_allocator::blockpair * - heat_zone(struct block_allocator::blockpair *blocks_array, - uint64_t n_blocks, uint64_t size, uint64_t alignment, - uint64_t heat); -}; +#include "ft/serialize/rbtree_mhs.h" +#include "test.h" +#include +#include +#include +#include + +static void test_insert_remove(void) { + uint64_t i; + MhsRbTree::Tree *tree = new MhsRbTree::Tree(); + verbose = 0; + + tree->Insert({0, 100}); + + for (i = 0; i < 10; i++) { + tree->Remove(3); + tree->Remove(2); + } + tree->ValidateBalance(); + tree->ValidateMhs(); + + for (i = 0; i < 10; i++) { + tree->Insert({5 * i, 3}); + } + tree->ValidateBalance(); + tree->ValidateMhs(); + + uint64_t offset = tree->Remove(2); + invariant(offset == 0); + offset = tree->Remove(10); + invariant(offset == 50); + offset = tree->Remove(3); + invariant(offset == 5); + tree->ValidateBalance(); + tree->ValidateMhs(); + + tree->Insert({48, 2}); + tree->Insert({50, 10}); + + tree->ValidateBalance(); + tree->ValidateMhs(); + + tree->Insert({3, 7}); + offset = tree->Remove(10); + invariant(offset == 2); + tree->ValidateBalance(); + tree->ValidateMhs(); + tree->Dump(); + delete tree; +} + +int test_main(int argc, const char *argv[]) { + default_parse_args(argc, argv); + + test_insert_remove(); + if (verbose) + printf("test ok\n"); + return 0; +} diff --git a/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc b/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc new file mode 100644 index 0000000000000..85f29ce9813ef --- /dev/null +++ b/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc @@ -0,0 +1,102 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see . + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see . +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "ft/serialize/rbtree_mhs.h" +#include "test.h" +#include +#include +#include +#include + +#define N 1000000 +std::vector input_vector; +MhsRbTree::Node::BlockPair old_vector[N]; + +static int myrandom(int i) { return std::rand() % i; } + +static void generate_random_input() { + std::srand(unsigned(std::time(0))); + + // set some values: + for (uint64_t i = 1; i < N; ++i) { + input_vector.push_back({i, 0}); + old_vector[i] = {i, 0}; + } + // using built-in random generator: + std::random_shuffle(input_vector.begin(), input_vector.end(), myrandom); +} + +static void test_insert_remove(void) { + int i; + MhsRbTree::Tree *tree = new MhsRbTree::Tree(); + verbose = 0; + generate_random_input(); + if (verbose) { + printf("\n we are going to insert the following block offsets\n"); + for (i = 0; i < N; i++) + printf("%" PRIu64 "\t", input_vector[i]._offset.ToInt()); + } + for (i = 0; i < N; i++) { + tree->Insert(input_vector[i]); + // tree->ValidateBalance(); + } + tree->ValidateBalance(); + MhsRbTree::Node::BlockPair *p_bps = &old_vector[0]; + tree->ValidateInOrder(p_bps); + printf("min node of the tree:%" PRIu64 "\n", + rbn_offset(tree->MinNode()).ToInt()); + printf("max node of the tree:%" PRIu64 "\n", + rbn_offset(tree->MaxNode()).ToInt()); + + for (i = 0; i < N; i++) { + // tree->ValidateBalance(); + tree->RawRemove(input_vector[i]._offset.ToInt()); + } + + tree->Destroy(); + delete tree; +} + +int test_main(int argc, const char *argv[]) { + default_parse_args(argc, argv); + + test_insert_remove(); + if (verbose) + printf("test ok\n"); + return 0; +} diff --git a/storage/tokudb/PerconaFT/ft/txn/roll.cc b/storage/tokudb/PerconaFT/ft/txn/roll.cc index 407116b983c1a..90eee1e580a80 100644 --- a/storage/tokudb/PerconaFT/ft/txn/roll.cc +++ b/storage/tokudb/PerconaFT/ft/txn/roll.cc @@ -49,7 +49,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. // functionality provided by roll.c is exposed by an autogenerated // header file, logheader.h // -// this (poorly) explains the absense of "roll.h" +// this (poorly) explains the absence of "roll.h" // these flags control whether or not we send commit messages for // various operations diff --git a/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc b/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc index df830afd0df68..c9464c3ed60a3 100644 --- a/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc +++ b/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc @@ -169,7 +169,7 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) { txn->roll_info.spilled_rollback_head = ROLLBACK_NONE; txn->roll_info.spilled_rollback_tail = ROLLBACK_NONE; } - // if we're commiting a child rollback, put its entries into the parent + // if we're committing a child rollback, put its entries into the parent // by pinning both child and parent and then linking the child log entry // list to the end of the parent log entry list. if (txn_has_current_rollback_log(txn)) { diff --git a/storage/tokudb/PerconaFT/ft/txn/rollback-ct-callbacks.cc b/storage/tokudb/PerconaFT/ft/txn/rollback-ct-callbacks.cc index 68c94c2ad1192..08d7c8874e50e 100644 --- a/storage/tokudb/PerconaFT/ft/txn/rollback-ct-callbacks.cc +++ b/storage/tokudb/PerconaFT/ft/txn/rollback-ct-callbacks.cc @@ -59,21 +59,18 @@ rollback_log_destroy(ROLLBACK_LOG_NODE log) { // flush an ununused log to disk, by allocating a size 0 blocknum in // the blocktable -static void -toku_rollback_flush_unused_log( - ROLLBACK_LOG_NODE log, - BLOCKNUM logname, - int fd, - FT ft, - bool write_me, - bool keep_me, - bool for_checkpoint, - bool is_clone - ) -{ +static void toku_rollback_flush_unused_log(ROLLBACK_LOG_NODE log, + BLOCKNUM logname, + int fd, + FT ft, + bool write_me, + bool keep_me, + bool for_checkpoint, + bool is_clone) { if (write_me) { DISKOFF offset; - ft->blocktable.realloc_on_disk(logname, 0, &offset, ft, fd, for_checkpoint, INT_MAX); + ft->blocktable.realloc_on_disk( + logname, 0, &offset, ft, fd, for_checkpoint); } if (!keep_me && !is_clone) { toku_free(log); diff --git a/storage/tokudb/PerconaFT/ft/ule.cc b/storage/tokudb/PerconaFT/ft/ule.cc index ac393fbf17991..e3dce6d27dd8d 100644 --- a/storage/tokudb/PerconaFT/ft/ule.cc +++ b/storage/tokudb/PerconaFT/ft/ule.cc @@ -587,8 +587,8 @@ bool toku_le_worth_running_garbage_collection( // by new txns. // 2.) There is only one committed entry, but the outermost // provisional entry is older than the oldest known referenced -// xid, so it must have commited. Therefor we can promote it to -// committed and get rid of the old commited entry. +// xid, so it must have committed. Therefor we can promote it to +// committed and get rid of the old committed entry. if (le->type != LE_MVCC) { return false; } diff --git a/storage/tokudb/PerconaFT/portability/tests/test-max-data.cc b/storage/tokudb/PerconaFT/portability/tests/test-max-data.cc index 880f9a3a9bbf7..dbbea974a49ff 100644 --- a/storage/tokudb/PerconaFT/portability/tests/test-max-data.cc +++ b/storage/tokudb/PerconaFT/portability/tests/test-max-data.cc @@ -64,7 +64,7 @@ int main(int argc, char *const argv[]) { if (verbose) printf("maxdata=%" PRIu64 " 0x%" PRIx64 "\n", maxdata, maxdata); // check the data size -#if __x86_64__ +#if defined(__x86_64__) || defined(__aarch64__) assert(maxdata > (1ULL << 32)); #elif __i386__ assert(maxdata < (1ULL << 32)); diff --git a/storage/tokudb/PerconaFT/portability/toku_config.h.in b/storage/tokudb/PerconaFT/portability/toku_config.h.in index e1412cc9e14d5..1a34bf1ef45a7 100644 --- a/storage/tokudb/PerconaFT/portability/toku_config.h.in +++ b/storage/tokudb/PerconaFT/portability/toku_config.h.in @@ -42,7 +42,6 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #cmakedefine TOKU_DEBUG_PARANOID 1 #cmakedefine USE_VALGRIND 1 - #cmakedefine HAVE_ALLOCA_H 1 #cmakedefine HAVE_ARPA_INET_H 1 #cmakedefine HAVE_BYTESWAP_H 1 diff --git a/storage/tokudb/PerconaFT/portability/toku_time.h b/storage/tokudb/PerconaFT/portability/toku_time.h index 11a3f3aa2b99c..a1278ef033731 100644 --- a/storage/tokudb/PerconaFT/portability/toku_time.h +++ b/storage/tokudb/PerconaFT/portability/toku_time.h @@ -98,9 +98,17 @@ double tokutime_to_seconds(tokutime_t) __attribute__((__visibility__("default") // Get the value of tokutime for right now. We want this to be fast, so we expose the implementation as RDTSC. static inline tokutime_t toku_time_now(void) { +#if defined(__x86_64__) || defined(__i386__) uint32_t lo, hi; __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); return (uint64_t)hi << 32 | lo; +#elif defined (__aarch64__) + uint64_t result; + __asm __volatile__ ("mrs %[rt], cntvct_el0" : [rt] "=r" (result)); + return result; +#else +#error No timer implementation for this platform +#endif } static inline uint64_t toku_current_time_microsec(void) { diff --git a/storage/tokudb/PerconaFT/src/indexer-internal.h b/storage/tokudb/PerconaFT/src/indexer-internal.h index 48e62ee49b2d7..fdaa561e3d027 100644 --- a/storage/tokudb/PerconaFT/src/indexer-internal.h +++ b/storage/tokudb/PerconaFT/src/indexer-internal.h @@ -42,7 +42,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include // the indexer_commit_keys is an ordered set of keys described by a DBT in the keys array. -// the array is a resizeable array with max size "max_keys" and current size "current_keys". +// the array is a resizable array with max size "max_keys" and current size "current_keys". // the ordered set is used by the hotindex undo function to collect the commit keys. struct indexer_commit_keys { int max_keys; // max number of keys diff --git a/storage/tokudb/PerconaFT/src/indexer-undo-do.cc b/storage/tokudb/PerconaFT/src/indexer-undo-do.cc index 8d0b080b9fe88..4c7f5336161da 100644 --- a/storage/tokudb/PerconaFT/src/indexer-undo-do.cc +++ b/storage/tokudb/PerconaFT/src/indexer-undo-do.cc @@ -528,7 +528,7 @@ indexer_find_prev_xr(DB_INDEXER *UU(indexer), ULEHANDLE ule, uint64_t xrindex, u } // inject "delete" message into ft with logging in recovery and rollback logs, -// and making assocation between txn and ft +// and making association between txn and ft static int indexer_ft_delete_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids, TOKUTXN txn) { int result = 0; @@ -577,7 +577,7 @@ indexer_ft_delete_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xi } // inject "insert" message into ft with logging in recovery and rollback logs, -// and making assocation between txn and ft +// and making association between txn and ft static int indexer_ft_insert_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids, TOKUTXN txn) { int result = 0; diff --git a/storage/tokudb/PerconaFT/src/tests/hotindexer-undo-do-tests/commit.i0.test b/storage/tokudb/PerconaFT/src/tests/hotindexer-undo-do-tests/commit.i0.test index 20df13923e6d5..7cce68e6ff8e5 100644 --- a/storage/tokudb/PerconaFT/src/tests/hotindexer-undo-do-tests/commit.i0.test +++ b/storage/tokudb/PerconaFT/src/tests/hotindexer-undo-do-tests/commit.i0.test @@ -1,3 +1,3 @@ -# commited insert +# committed insert key k1 insert committed 0 v100 diff --git a/storage/tokudb/PerconaFT/src/tests/loader-dup-test.cc b/storage/tokudb/PerconaFT/src/tests/loader-dup-test.cc index 3f2f8d7455a98..aaf77c503cc42 100644 --- a/storage/tokudb/PerconaFT/src/tests/loader-dup-test.cc +++ b/storage/tokudb/PerconaFT/src/tests/loader-dup-test.cc @@ -51,7 +51,7 @@ int DISALLOW_PUTS=0; int COMPRESS=0; enum {MAGIC=311}; -bool dup_row_at_end = false; // false: duplicate at the begining. true: duplicate at the end. The duplicated row is row 0. +bool dup_row_at_end = false; // false: duplicate at the beginning. true: duplicate at the end. The duplicated row is row 0. int dup_row_id = 0; // 0 means to use row 1 if inserting at the end, row NUM_ROWS if inserting at the beginning. Otherwise insert the row specified here. // diff --git a/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc b/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc index a4dc0ea9236b4..2c905c5ff122e 100644 --- a/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc +++ b/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc @@ -156,7 +156,7 @@ do_args(int argc, char * const argv[]) { choices[i] = -1; } - char c; + int c; while ((c = getopt(argc, argv, "vqhcrO:A:B:C:D:E:F:G:H:I:X:")) != -1) { switch(c) { case 'v': diff --git a/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc b/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc index a2b48e443cdfd..48843a0bd3257 100644 --- a/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc +++ b/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc @@ -166,7 +166,7 @@ run_test (void) { DB_BTREE_STAT64 s; r = db->stat64(db, NULL, &s); CKERR(r); - assert(s.bt_nkeys == 0); + assert(s.bt_nkeys == 1); r = db->close(db, 0); CKERR(r); @@ -176,7 +176,7 @@ run_test (void) { r = txn->commit(txn, 0); CKERR(r); r = db->stat64(db, NULL, &s); CKERR(r); - assert(s.bt_nkeys == 0); + assert(s.bt_nkeys == 1); } // verify update callback overwrites the row diff --git a/storage/tokudb/PerconaFT/src/tests/test_insert_many_gc.cc b/storage/tokudb/PerconaFT/src/tests/test_insert_many_gc.cc index 8e5109cd2a973..f6111d4b67c0b 100644 --- a/storage/tokudb/PerconaFT/src/tests/test_insert_many_gc.cc +++ b/storage/tokudb/PerconaFT/src/tests/test_insert_many_gc.cc @@ -78,7 +78,7 @@ static void test_insert_many_gc(void) { // from having an MVCC stack of size 'N'. At the time of this // writing, we run full GC on leaf-inject when the leaf is // 32mb or larger. A good invariant is that the max LE size - // never grew larger than 35mb and that the max commited xr stack + // never grew larger than 35mb and that the max committed xr stack // length never exceeded 35 const uint64_t le_max_memsize = get_engine_status_val(env, "LE_MAX_MEMSIZE"); const uint64_t le_max_committed_xr = get_engine_status_val(env, "LE_MAX_COMMITTED_XR"); diff --git a/storage/tokudb/PerconaFT/src/tests/test_stress0.cc b/storage/tokudb/PerconaFT/src/tests/test_stress0.cc index aaafe284906a0..88140dd173184 100644 --- a/storage/tokudb/PerconaFT/src/tests/test_stress0.cc +++ b/storage/tokudb/PerconaFT/src/tests/test_stress0.cc @@ -53,7 +53,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. // This test is a micro stress test that does multithreaded updates on a fixed size table. // There is also a thread that scans the table with bulk fetch, ensuring the sum is zero. // -// This test is targetted at stressing the locktree, hence the small table and many update threads. +// This test is targeted at stressing the locktree, hence the small table and many update threads. // static int UU() lock_escalation_op(DB_TXN *UU(txn), ARG arg, void* operation_extra, void *UU(stats_extra)) { diff --git a/storage/tokudb/PerconaFT/src/tests/test_txn_abort5a.cc b/storage/tokudb/PerconaFT/src/tests/test_txn_abort5a.cc index fec454b80093b..301eed1560e13 100644 --- a/storage/tokudb/PerconaFT/src/tests/test_txn_abort5a.cc +++ b/storage/tokudb/PerconaFT/src/tests/test_txn_abort5a.cc @@ -123,7 +123,8 @@ test_main(int argc, char *const argv[]) { continue; } } - if (verbose>0) printf("%s", __FILE__); if (verbose>1) printf("\n"); + if (verbose>0) printf("%s", __FILE__); + if (verbose>1) printf("\n"); for (i=1; i<100; i++) test_txn_abort(i); if (verbose>1) printf("%s OK\n", __FILE__); diff --git a/storage/tokudb/PerconaFT/src/ydb-internal.h b/storage/tokudb/PerconaFT/src/ydb-internal.h index 462a2a3d861e3..2d6c84126e168 100644 --- a/storage/tokudb/PerconaFT/src/ydb-internal.h +++ b/storage/tokudb/PerconaFT/src/ydb-internal.h @@ -114,7 +114,7 @@ struct __toku_db_env_internal { char *real_data_dir; // data dir used when the env is opened (relative to cwd, or absolute with leading /) char *real_log_dir; // log dir used when the env is opened (relative to cwd, or absolute with leading /) - char *real_tmp_dir; // tmp dir used for temporary files (relative to cwd, or absoulte with leading /) + char *real_tmp_dir; // tmp dir used for temporary files (relative to cwd, or absolute with leading /) fs_redzone_state fs_state; uint64_t fs_seq; // how many times has fs_poller run? diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess index da8331460888a..7501b1bee019d 100755 --- a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess +++ b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess @@ -1,10 +1,10 @@ #! /bin/sh # Attempt to guess a canonical system name. # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 -# Free Software Foundation, Inc. +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012 Free Software Foundation, Inc. -timestamp='2009-04-27' +timestamp='2016-06-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -17,9 +17,7 @@ timestamp='2009-04-27' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA -# 02110-1301, USA. +# along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a @@ -27,16 +25,16 @@ timestamp='2009-04-27' # the same distribution terms that you use for the rest of that program. -# Originally written by Per Bothner . -# Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. +# Originally written by Per Bothner. Please send patches (context +# diff format) to and include a ChangeLog +# entry. # # This script attempts to guess a canonical system name similar to # config.sub. If it succeeds, it prints the system name on stdout, and # exits with 0. Otherwise, it exits with 1. # -# The plan is that this can be called by configure scripts if you -# don't specify an explicit build system type. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD me=`echo "$0" | sed -e 's,.*/,,'` @@ -56,8 +54,9 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, -2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, +2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 +Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -144,7 +143,7 @@ UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently # switched to ELF, *-*-netbsd* would select the old # object file format. This provides both forward @@ -170,7 +169,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep __ELF__ >/dev/null + | grep -q __ELF__ then # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). # Return netbsd for either. FIX? @@ -180,7 +179,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in fi ;; *) - os=netbsd + os=netbsd ;; esac # The OS release @@ -223,7 +222,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on @@ -269,7 +268,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - exit ;; + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; Alpha\ *:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # Should we change UNAME_MACHINE based on the output of uname instead @@ -295,7 +297,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo s390-ibm-zvmoe exit ;; *:OS400:*:*) - echo powerpc-ibm-os400 + echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix${UNAME_RELEASE} @@ -333,6 +335,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux${UNAME_RELEASE} + exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval $set_cc_for_build SUN_ARCH="i386" @@ -391,23 +396,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint${UNAME_RELEASE} exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} - exit ;; + exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint${UNAME_RELEASE} exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} - exit ;; + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} - exit ;; + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} - exit ;; + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; m68k:machten:*:*) echo m68k-apple-machten${UNAME_RELEASE} exit ;; @@ -477,8 +482,8 @@ EOF echo m88k-motorola-sysv3 exit ;; AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] then if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ @@ -491,7 +496,7 @@ EOF else echo i586-dg-dgux${UNAME_RELEASE} fi - exit ;; + exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) echo m88k-dolphin-sysv3 exit ;; @@ -548,7 +553,7 @@ EOF echo rs6000-ibm-aix3.2 fi exit ;; - *:AIX:*:[456]) + *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 @@ -591,52 +596,52 @@ EOF 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 - esac ;; - esac + esac ;; + esac fi if [ "${HP_ARCH}" = "" ]; then eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + sed 's/^ //' << EOF >$dummy.c - #define _HPUX_SOURCE - #include - #include + #define _HPUX_SOURCE + #include + #include - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } EOF (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa @@ -656,7 +661,7 @@ EOF # => hppa64-hp-hpux11.23 if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | - grep __LP64__ >/dev/null + grep -q __LP64__ then HP_ARCH="hppa2.0w" else @@ -727,22 +732,22 @@ EOF exit ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) echo c1-convex-bsd - exit ;; + exit ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit ;; + exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) echo c34-convex-bsd - exit ;; + exit ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) echo c38-convex-bsd - exit ;; + exit ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) echo c4-convex-bsd - exit ;; + exit ;; CRAY*Y-MP:*:*:*) echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; @@ -766,14 +771,14 @@ EOF exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} @@ -785,13 +790,12 @@ EOF echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} exit ;; *:FreeBSD:*:*) - case ${UNAME_MACHINE} in - pc98) - echo i386-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=`/usr/bin/uname -p` + case ${UNAME_PROCESSOR} in amd64) echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; *) - echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; esac exit ;; i*:CYGWIN*:*) @@ -800,19 +804,22 @@ EOF *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 exit ;; + i*:MSYS*:*) + echo ${UNAME_MACHINE}-pc-msys + exit ;; i*:windows32*:*) - # uname -m includes "-pc" on this system. - echo ${UNAME_MACHINE}-mingw32 + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 exit ;; i*:PW*:*) echo ${UNAME_MACHINE}-pc-pw32 exit ;; - *:Interix*:[3456]*) - case ${UNAME_MACHINE} in + *:Interix*:*) + case ${UNAME_MACHINE} in x86) echo i586-pc-interix${UNAME_RELEASE} exit ;; - EM64T | authenticamd | genuineintel) + authenticamd | genuineintel | EM64T) echo x86_64-unknown-interix${UNAME_RELEASE} exit ;; IA64) @@ -822,6 +829,9 @@ EOF [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) echo i${UNAME_MACHINE}-pc-mks exit ;; + 8664:Windows_NT:*) + echo x86_64-pc-mks + exit ;; i*:Windows_NT*:* | Pentium*:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we @@ -851,6 +861,27 @@ EOF i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix exit ;; + aarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi + echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + exit ;; arm*:Linux:*:*) eval $set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ @@ -858,20 +889,40 @@ EOF then echo ${UNAME_MACHINE}-unknown-linux-gnu else - echo ${UNAME_MACHINE}-unknown-linux-gnueabi + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo ${UNAME_MACHINE}-unknown-linux-gnueabi + else + echo ${UNAME_MACHINE}-unknown-linux-gnueabihf + fi fi exit ;; avr32*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; cris:Linux:*:*) - echo cris-axis-linux-gnu + echo ${UNAME_MACHINE}-axis-linux-gnu exit ;; crisv32:Linux:*:*) - echo crisv32-axis-linux-gnu + echo ${UNAME_MACHINE}-axis-linux-gnu exit ;; frv:Linux:*:*) - echo frv-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + hexagon:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + i*86:Linux:*:*) + LIBC=gnu + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #ifdef __dietlibc__ + LIBC=dietlibc + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + echo "${UNAME_MACHINE}-pc-linux-${LIBC}" exit ;; ia64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu @@ -882,78 +933,34 @@ EOF m68*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; - mips:Linux:*:*) + mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #undef CPU - #undef mips - #undef mipsel + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mipsel + CPU=${UNAME_MACHINE}el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips + CPU=${UNAME_MACHINE} #else CPU= #endif #endif EOF - eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n ' - /^CPU/{ - s: ::g - p - }'`" - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } - ;; - mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #undef CPU - #undef mips64 - #undef mips64el - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mips64el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips64 - #else - CPU= - #endif - #endif -EOF - eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n ' - /^CPU/{ - s: ::g - p - }'`" + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } ;; or32:Linux:*:*) - echo or32-unknown-linux-gnu - exit ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu - exit ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu - exit ;; - alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; padre:Linux:*:*) echo sparc-unknown-linux-gnu exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-gnu + exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in @@ -962,14 +969,17 @@ EOF *) echo hppa-unknown-linux-gnu ;; esac exit ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-gnu + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-gnu exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo ${UNAME_MACHINE}-ibm-linux exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; sh*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu @@ -977,75 +987,18 @@ EOF sparc:Linux:*:* | sparc64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; + tile*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; vax:Linux:*:*) echo ${UNAME_MACHINE}-dec-linux-gnu exit ;; x86_64:Linux:*:*) - echo x86_64-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; - i*86:Linux:*:*) - # The BFD linker knows what the default object file format is, so - # first see if it will tell us. cd to the root directory to prevent - # problems with other programs or directories called `ld' in the path. - # Set LC_ALL=C to ensure ld outputs messages in English. - ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ - | sed -ne '/supported targets:/!d - s/[ ][ ]*/ /g - s/.*supported targets: *// - s/ .*// - p'` - case "$ld_supported_targets" in - elf32-i386) - TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu" - ;; - a.out-i386-linux) - echo "${UNAME_MACHINE}-pc-linux-gnuaout" - exit ;; - "") - # Either a pre-BFD a.out linker (linux-gnuoldld) or - # one that does not give us useful --help. - echo "${UNAME_MACHINE}-pc-linux-gnuoldld" - exit ;; - esac - # Determine whether the default compiler is a.out or elf - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #ifdef __ELF__ - # ifdef __GLIBC__ - # if __GLIBC__ >= 2 - LIBC=gnu - # else - LIBC=gnulibc1 - # endif - # else - LIBC=gnulibc1 - # endif - #else - #if defined(__INTEL_COMPILER) || defined(__PGI) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) - LIBC=gnu - #else - LIBC=gnuaout - #endif - #endif - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval "`$CC_FOR_BUILD -E $dummy.c 2>/dev/null | sed -n ' - /^LIBC/{ - s: ::g - p - }'`" - test x"${LIBC}" != x && { - echo "${UNAME_MACHINE}-pc-linux-${LIBC}" - exit - } - test x"${TENTATIVE}" != x && { echo "${TENTATIVE}"; exit; } - ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both @@ -1053,11 +1006,11 @@ EOF echo i386-sequent-sysv4 exit ;; i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. + # Use sysv4.2uw... so that sysv4* matches it. echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} exit ;; i*86:OS/2:*:*) @@ -1074,7 +1027,7 @@ EOF i*86:syllable:*:*) echo ${UNAME_MACHINE}-pc-syllable exit ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) echo i386-unknown-lynxos${UNAME_RELEASE} exit ;; i*86:*DOS:*:*) @@ -1089,7 +1042,7 @@ EOF fi exit ;; i*86:*:5:[678]*) - # UnixWare 7.x, OpenUNIX and OpenServer 6. + # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; @@ -1117,13 +1070,13 @@ EOF exit ;; pc:*:*:*) # Left here for compatibility: - # uname -m prints for DJGPP always 'pc', but it prints nothing about - # the processor, so we play safe by assuming i586. + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub # prints for the "djgpp" host, or else GDB configury will decide that # this is a cross-build. echo i586-pc-msdosdjgpp - exit ;; + exit ;; Intel:Mach:3*:*) echo i386-pc-mach3 exit ;; @@ -1158,8 +1111,8 @@ EOF /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4; exit; } ;; + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; NCR*:*:4.2:* | MPRAS*:*:4.2:*) OS_REL='.3' test -r /etc/.relid \ @@ -1182,7 +1135,7 @@ EOF rs6000:LynxOS:2.*:*) echo rs6000-unknown-lynxos${UNAME_RELEASE} exit ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) echo powerpc-unknown-lynxos${UNAME_RELEASE} exit ;; SM[BE]S:UNIX_SV:*:*) @@ -1202,10 +1155,10 @@ EOF echo ns32k-sni-sysv fi exit ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - echo i586-unisys-sysv4 - exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes . # How about differentiating between stratus architectures? -djm @@ -1231,11 +1184,11 @@ EOF exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv${UNAME_RELEASE} else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv${UNAME_RELEASE} fi - exit ;; + exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. echo powerpc-be-beos exit ;; @@ -1275,6 +1228,16 @@ EOF *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown case $UNAME_PROCESSOR in + i386) + eval $set_cc_for_build + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + UNAME_PROCESSOR="x86_64" + fi + fi ;; unknown) UNAME_PROCESSOR=powerpc ;; esac echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} @@ -1290,6 +1253,9 @@ EOF *:QNX:*:4*) echo i386-pc-qnx exit ;; + NEO-?:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk${UNAME_RELEASE} + exit ;; NSE-?:NONSTOP_KERNEL:*:*) echo nse-tandem-nsk${UNAME_RELEASE} exit ;; @@ -1335,13 +1301,13 @@ EOF echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} + echo mips-sei-seiux${UNAME_RELEASE} exit ;; *:DragonFly:*:*) echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; *:*VMS:*:*) - UNAME_MACHINE=`(uname -p) 2>/dev/null` + UNAME_MACHINE=`(uname -p) 2>/dev/null` case "${UNAME_MACHINE}" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; @@ -1359,6 +1325,9 @@ EOF i*86:AROS:*:*) echo ${UNAME_MACHINE}-pc-aros exit ;; + x86_64:VMkernel:*:*) + echo ${UNAME_MACHINE}-unknown-esx + exit ;; esac #echo '(No uname command or uname output not recognized.)' 1>&2 @@ -1381,11 +1350,11 @@ main () #include printf ("m68k-sony-newsos%s\n", #ifdef NEWSOS4 - "4" + "4" #else - "" + "" #endif - ); exit (0); + ); exit (0); #endif #endif diff --git a/storage/tokudb/PerconaFT/tools/CMakeLists.txt b/storage/tokudb/PerconaFT/tools/CMakeLists.txt index d3808483fea60..30ca883c4ba96 100644 --- a/storage/tokudb/PerconaFT/tools/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/tools/CMakeLists.txt @@ -1,6 +1,6 @@ set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPRECATE_ERRNO) -set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify ba_replay) +set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify) foreach(tool ${tools}) add_executable(${tool} ${tool}.cc) add_dependencies(${tool} install_tdb_h) @@ -14,4 +14,3 @@ target_link_libraries(ftverify m) install(TARGETS tokuftdump DESTINATION bin COMPONENT tokukv_tools) install(TARGETS tokuft_logprint DESTINATION bin COMPONENT tokukv_tools) - diff --git a/storage/tokudb/PerconaFT/tools/ba_replay.cc b/storage/tokudb/PerconaFT/tools/ba_replay.cc deleted file mode 100644 index cade7e5dfafc6..0000000000000 --- a/storage/tokudb/PerconaFT/tools/ba_replay.cc +++ /dev/null @@ -1,629 +0,0 @@ -/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ -// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: -#ident "$Id$" -/*====== -This file is part of PerconaFT. - - -Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - - PerconaFT is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License, version 2, - as published by the Free Software Foundation. - - PerconaFT is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with PerconaFT. If not, see . - ----------------------------------------- - - PerconaFT is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License, version 3, - as published by the Free Software Foundation. - - PerconaFT is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with PerconaFT. If not, see . -======= */ - -#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." - -// Replay a block allocator trace against different strategies and compare -// the results - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "ft/serialize/block_allocator.h" - -using std::map; -using std::set; -using std::string; -using std::vector; - -static int verbose = false; - -static void ba_replay_assert(bool pred, const char *msg, const char *line, int line_num) { - if (!pred) { - fprintf(stderr, "%s, line (#%d): %s\n", msg, line_num, line); - abort(); - } -} - -static char *trim_whitespace(char *line) { - // skip leading whitespace - while (isspace(*line)) { - line++; - } - return line; -} - -static int64_t parse_number(char **ptr, int line_num, int base) { - *ptr = trim_whitespace(*ptr); - char *line = *ptr; - - char *new_ptr; - int64_t n = strtoll(line, &new_ptr, base); - ba_replay_assert(n >= 0, "malformed trace (bad numeric token)", line, line_num); - ba_replay_assert(new_ptr > *ptr, "malformed trace (missing numeric token)", line, line_num); - *ptr = new_ptr; - return n; -} - -static uint64_t parse_uint64(char **ptr, int line_num) { - int64_t n = parse_number(ptr, line_num, 10); - // we happen to know that the uint64's we deal with will - // take less than 63 bits (they come from pointers) - return static_cast(n); -} - -static string parse_token(char **ptr, int line_num) { - *ptr = trim_whitespace(*ptr); - char *line = *ptr; - - // parse the first token, which represents the traced function - char token[64]; - int r = sscanf(*ptr, "%64s", token); - ba_replay_assert(r == 1, "malformed trace (missing string token)", line, line_num); - *ptr += strlen(token); - return string(token); -} - -static block_allocator::blockpair parse_blockpair(char **ptr, int line_num) { - *ptr = trim_whitespace(*ptr); - char *line = *ptr; - - uint64_t offset, size; - int bytes_read; - int r = sscanf(line, "[%" PRIu64 " %" PRIu64 "]%n", &offset, &size, &bytes_read); - ba_replay_assert(r == 2, "malformed trace (bad offset/size pair)", line, line_num); - *ptr += bytes_read; - return block_allocator::blockpair(offset, size); -} - -static char *strip_newline(char *line, bool *found) { - char *ptr = strchr(line, '\n'); - if (ptr != nullptr) { - if (found != nullptr) { - *found = true; - } - *ptr = '\0'; - } - return line; -} - -static char *read_trace_line(FILE *file) { - const int buf_size = 4096; - char buf[buf_size]; - std::stringstream ss; - while (true) { - if (fgets(buf, buf_size, file) == nullptr) { - break; - } - bool has_newline = false; - ss << strip_newline(buf, &has_newline); - if (has_newline) { - // end of the line, we're done out - break; - } - } - std::string s = ss.str(); - return s.size() ? toku_strdup(s.c_str()) : nullptr; -} - -static vector canonicalize_trace_from(FILE *file) { - // new trace, canonicalized from a raw trace - vector canonicalized_trace; - - // raw allocator id -> canonical allocator id - // - // keeps track of allocators that were created as part of the trace, - // and therefore will be part of the canonicalized trace. - uint64_t allocator_id_seq_num = 0; - map allocator_ids; - - // allocated offset -> allocation seq num - // - uint64_t allocation_seq_num = 0; - static const uint64_t ASN_NONE = (uint64_t) -1; - typedef map offset_seq_map; - - // raw allocator id -> offset_seq_map that tracks its allocations - map offset_to_seq_num_maps; - - int line_num = 0; - char *line; - while ((line = read_trace_line(file)) != nullptr) { - line_num++; - char *ptr = line; - - string fn = parse_token(&ptr, line_num); - int64_t allocator_id = parse_number(&ptr, line_num, 16); - - std::stringstream ss; - if (fn.find("ba_trace_create") != string::npos) { - ba_replay_assert(allocator_ids.count(allocator_id) == 0, "corrupted trace: double create", line, line_num); - ba_replay_assert(fn == "ba_trace_create" || fn == "ba_trace_create_from_blockpairs", - "corrupted trace: bad fn", line, line_num); - - // we only convert the allocator_id to an allocator_id_seq_num - // in the canonical trace and leave the rest of the line as-is. - allocator_ids[allocator_id] = allocator_id_seq_num; - ss << fn << ' ' << allocator_id_seq_num << ' ' << trim_whitespace(ptr) << std::endl; - allocator_id_seq_num++; - - // First, read passed the reserve / alignment values. - (void) parse_uint64(&ptr, line_num); - (void) parse_uint64(&ptr, line_num); - if (fn == "ba_trace_create_from_blockpairs") { - // For each blockpair created by this traceline, add its offset to the offset seq map - // with asn ASN_NONE so that later canonicalizations of `free' know whether to write - // down the asn or the raw offset. - offset_seq_map *map = &offset_to_seq_num_maps[allocator_id]; - while (*trim_whitespace(ptr) != '\0') { - const block_allocator::blockpair bp = parse_blockpair(&ptr, line_num); - (*map)[bp.offset] = ASN_NONE; - } - } - } else { - ba_replay_assert(allocator_ids.count(allocator_id) > 0, "corrupted trace: unknown allocator", line, line_num); - uint64_t canonical_allocator_id = allocator_ids[allocator_id]; - - // this is the map that tracks allocations for this allocator - offset_seq_map *map = &offset_to_seq_num_maps[allocator_id]; - - if (fn == "ba_trace_alloc") { - const uint64_t size = parse_uint64(&ptr, line_num); - const uint64_t heat = parse_uint64(&ptr, line_num); - const uint64_t offset = parse_uint64(&ptr, line_num); - ba_replay_assert(map->count(offset) == 0, "corrupted trace: double alloc", line, line_num); - - // remember that an allocation at `offset' has the current alloc seq num - (*map)[offset] = allocation_seq_num; - - // translate `offset = alloc(size)' to `asn = alloc(size)' - ss << fn << ' ' << canonical_allocator_id << ' ' << size << ' ' << heat << ' ' << allocation_seq_num << std::endl; - allocation_seq_num++; - } else if (fn == "ba_trace_free") { - const uint64_t offset = parse_uint64(&ptr, line_num); - ba_replay_assert(map->count(offset) != 0, "corrupted trace: invalid free", line, line_num); - - // get the alloc seq num for an allcation that occurred at `offset' - const uint64_t asn = (*map)[offset]; - map->erase(offset); - - // if there's an asn, then a corresponding ba_trace_alloc occurred and we should - // write `free(asn)'. otherwise, the blockpair was initialized from create_from_blockpairs - // and we write the original offset. - if (asn != ASN_NONE) { - ss << "ba_trace_free_asn" << ' ' << canonical_allocator_id << ' ' << asn << std::endl; - } else { - ss << "ba_trace_free_offset" << ' ' << canonical_allocator_id << ' ' << offset << std::endl; - } - } else if (fn == "ba_trace_destroy") { - // Remove this allocator from both maps - allocator_ids.erase(allocator_id); - offset_to_seq_num_maps.erase(allocator_id); - - // translate `destroy(ptr_id) to destroy(canonical_id)' - ss << fn << ' ' << canonical_allocator_id << ' ' << std::endl; - } else { - ba_replay_assert(false, "corrupted trace: bad fn", line, line_num); - } - } - canonicalized_trace.push_back(ss.str()); - - toku_free(line); - } - - if (allocator_ids.size() != 0) { - fprintf(stderr, "warning: leaked allocators. this might be ok if the tracing process is still running"); - } - - return canonicalized_trace; -} - -struct streaming_variance_calculator { - int64_t n_samples; - int64_t mean; - int64_t variance; - - // math credit: AoCP, Donald Knuth, '62 - void add_sample(int64_t x) { - n_samples++; - if (n_samples == 1) { - mean = x; - variance = 0; - } else { - int64_t old_mean = mean; - mean = old_mean + ((x - old_mean) / n_samples); - variance = (((n_samples - 1) * variance) + - ((x - old_mean) * (x - mean))) / n_samples; - } - } -}; - -struct canonical_trace_stats { - uint64_t n_lines_replayed; - - uint64_t n_create; - uint64_t n_create_from_blockpairs; - uint64_t n_alloc_hot; - uint64_t n_alloc_cold; - uint64_t n_free; - uint64_t n_destroy; - - struct streaming_variance_calculator alloc_hot_bytes; - struct streaming_variance_calculator alloc_cold_bytes; - - canonical_trace_stats() { - memset(this, 0, sizeof(*this)); - } -}; - -struct fragmentation_report { - TOKU_DB_FRAGMENTATION_S beginning; - TOKU_DB_FRAGMENTATION_S end; - fragmentation_report() { - memset(this, 0, sizeof(*this)); - } - void merge(const struct fragmentation_report &src_report) { - for (int i = 0; i < 2; i++) { - TOKU_DB_FRAGMENTATION_S *dst = i == 0 ? &beginning : &end; - const TOKU_DB_FRAGMENTATION_S *src = i == 0 ? &src_report.beginning : &src_report.end; - dst->file_size_bytes += src->file_size_bytes; - dst->data_bytes += src->data_bytes; - dst->data_blocks += src->data_blocks; - dst->checkpoint_bytes_additional += src->checkpoint_bytes_additional; - dst->checkpoint_blocks_additional += src->checkpoint_blocks_additional; - dst->unused_bytes += src->unused_bytes; - dst->unused_blocks += src->unused_blocks; - dst->largest_unused_block += src->largest_unused_block; - } - } -}; - -static void replay_canonicalized_trace(const vector &canonicalized_trace, - block_allocator::allocation_strategy strategy, - map *reports, - struct canonical_trace_stats *stats) { - // maps an allocator id to its block allocator - map allocator_map; - - // maps allocation seq num to allocated offset - map seq_num_to_offset; - - for (vector::const_iterator it = canonicalized_trace.begin(); - it != canonicalized_trace.end(); it++) { - const int line_num = stats->n_lines_replayed++; - - char *line = toku_strdup(it->c_str()); - line = strip_newline(line, nullptr); - - char *ptr = trim_whitespace(line); - - // canonical allocator id is in base 10, not 16 - string fn = parse_token(&ptr, line_num); - int64_t allocator_id = parse_number(&ptr, line_num, 10); - - if (fn.find("ba_trace_create") != string::npos) { - const uint64_t reserve_at_beginning = parse_uint64(&ptr, line_num); - const uint64_t alignment = parse_uint64(&ptr, line_num); - ba_replay_assert(allocator_map.count(allocator_id) == 0, - "corrupted canonical trace: double create", line, line_num); - - block_allocator *ba = new block_allocator(); - if (fn == "ba_trace_create") { - ba->create(reserve_at_beginning, alignment); - stats->n_create++; - } else { - ba_replay_assert(fn == "ba_trace_create_from_blockpairs", - "corrupted canonical trace: bad create fn", line, line_num); - vector pairs; - while (*trim_whitespace(ptr) != '\0') { - const block_allocator::blockpair bp = parse_blockpair(&ptr, line_num); - pairs.push_back(bp); - } - ba->create_from_blockpairs(reserve_at_beginning, alignment, &pairs[0], pairs.size()); - stats->n_create_from_blockpairs++; - } - ba->set_strategy(strategy); - - TOKU_DB_FRAGMENTATION_S report; - ba->get_statistics(&report); - (*reports)[allocator_id].beginning = report; - allocator_map[allocator_id] = ba; - } else { - ba_replay_assert(allocator_map.count(allocator_id) > 0, - "corrupted canonical trace: no such allocator", line, line_num); - - block_allocator *ba = allocator_map[allocator_id]; - if (fn == "ba_trace_alloc") { - // replay an `alloc' whose result will be associated with a certain asn - const uint64_t size = parse_uint64(&ptr, line_num); - const uint64_t heat = parse_uint64(&ptr, line_num); - const uint64_t asn = parse_uint64(&ptr, line_num); - ba_replay_assert(seq_num_to_offset.count(asn) == 0, - "corrupted canonical trace: double alloc (asn in use)", line, line_num); - - uint64_t offset; - ba->alloc_block(size, heat, &offset); - seq_num_to_offset[asn] = offset; - heat ? stats->n_alloc_hot++ : stats->n_alloc_cold++; - heat ? stats->alloc_hot_bytes.add_sample(size) : stats->alloc_cold_bytes.add_sample(size); - } else if (fn == "ba_trace_free_asn") { - // replay a `free' on a block whose offset is the result of an alloc with an asn - const uint64_t asn = parse_uint64(&ptr, line_num); - ba_replay_assert(seq_num_to_offset.count(asn) == 1, - "corrupted canonical trace: double free (asn unused)", line, line_num); - - const uint64_t offset = seq_num_to_offset[asn]; - ba->free_block(offset); - seq_num_to_offset.erase(asn); - stats->n_free++; - } else if (fn == "ba_trace_free_offset") { - // replay a `free' on a block whose offset was explicitly set during a create_from_blockpairs - const uint64_t offset = parse_uint64(&ptr, line_num); - ba->free_block(offset); - stats->n_free++; - } else if (fn == "ba_trace_destroy") { - TOKU_DB_FRAGMENTATION_S report; - ba->get_statistics(&report); - ba->destroy(); - (*reports)[allocator_id].end = report; - allocator_map.erase(allocator_id); - stats->n_destroy++; - } else { - ba_replay_assert(false, "corrupted canonical trace: bad fn", line, line_num); - } - } - - toku_free(line); - } -} - -static const char *strategy_to_cstring(block_allocator::allocation_strategy strategy) { - switch (strategy) { - case block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT: - return "first-fit"; - case block_allocator::allocation_strategy::BA_STRATEGY_BEST_FIT: - return "best-fit"; - case block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE: - return "heat-zone"; - case block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT: - return "padded-fit"; - default: - abort(); - } -} - -static block_allocator::allocation_strategy cstring_to_strategy(const char *str) { - if (strcmp(str, "first-fit") == 0) { - return block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT; - } - if (strcmp(str, "best-fit") == 0) { - return block_allocator::allocation_strategy::BA_STRATEGY_BEST_FIT; - } - if (strcmp(str, "heat-zone") == 0) { - return block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE; - } - if (strcmp(str, "padded-fit") != 0) { - fprintf(stderr, "bad strategy string: %s\n", str); - abort(); - } - return block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT; -} - -static void print_result_verbose(uint64_t allocator_id, - block_allocator::allocation_strategy strategy, - const struct fragmentation_report &report) { - if (report.end.data_bytes + report.end.unused_bytes + - report.beginning.data_bytes + report.beginning.unused_bytes - < 32UL * 1024 * 1024) { - printf(" ...skipping allocator_id %" PRId64 " (total bytes < 32mb)\n", allocator_id); - return; - } - - printf(" allocator_id: %20" PRId64 "\n", allocator_id); - printf(" strategy: %20s\n", strategy_to_cstring(strategy)); - - for (int i = 0; i < 2; i++) { - const TOKU_DB_FRAGMENTATION_S *r = i == 0 ? &report.beginning : &report.end; - printf("%s\n", i == 0 ? "BEFORE" : "AFTER"); - - uint64_t total_bytes = r->data_bytes + r->unused_bytes; - uint64_t total_blocks = r->data_blocks + r->unused_blocks; - - // byte statistics - printf(" total bytes: %20" PRId64 "\n", total_bytes); - printf(" used bytes: %20" PRId64 " (%.3lf)\n", r->data_bytes, - static_cast(r->data_bytes) / total_bytes); - printf(" unused bytes: %20" PRId64 " (%.3lf)\n", r->unused_bytes, - static_cast(r->unused_bytes) / total_bytes); - - // block statistics - printf(" total blocks: %20" PRId64 "\n", total_blocks); - printf(" used blocks: %20" PRId64 " (%.3lf)\n", r->data_blocks, - static_cast(r->data_blocks) / total_blocks); - printf(" unused blocks: %20" PRId64 " (%.3lf)\n", r->unused_blocks, - static_cast(r->unused_blocks) / total_blocks); - - // misc - printf(" largest unused: %20" PRId64 "\n", r->largest_unused_block); - } -} - -static void print_result(uint64_t allocator_id, - block_allocator::allocation_strategy strategy, - const struct fragmentation_report &report) { - const TOKU_DB_FRAGMENTATION_S *beginning = &report.beginning; - const TOKU_DB_FRAGMENTATION_S *end = &report.end; - - uint64_t total_beginning_bytes = beginning->data_bytes + beginning->unused_bytes; - uint64_t total_end_bytes = end->data_bytes + end->unused_bytes; - if (total_end_bytes + total_beginning_bytes < 32UL * 1024 * 1024) { - if (verbose) { - printf("\n"); - printf(" ...skipping allocator_id %" PRId64 " (total bytes < 32mb)\n", allocator_id); - } - return; - } - printf("\n"); - if (verbose) { - print_result_verbose(allocator_id, strategy, report); - } else { - printf(" %-15s: allocator %" PRId64 ", %.3lf used bytes (%.3lf before)\n", - strategy_to_cstring(strategy), allocator_id, - static_cast(report.end.data_bytes) / total_end_bytes, - static_cast(report.beginning.data_bytes) / total_beginning_bytes); - } -} - -static int only_aggregate_reports; - -static struct option getopt_options[] = { - { "verbose", no_argument, &verbose, 1 }, - { "only-aggregate-reports", no_argument, &only_aggregate_reports, 1 }, - { "include-strategy", required_argument, nullptr, 'i' }, - { "exclude-strategy", required_argument, nullptr, 'x' }, - { nullptr, 0, nullptr, 0 }, -}; - -int main(int argc, char *argv[]) { - int opt; - set candidate_strategies, excluded_strategies; - while ((opt = getopt_long(argc, argv, "", getopt_options, nullptr)) != -1) { - switch (opt) { - case 0: - break; - case 'i': - candidate_strategies.insert(cstring_to_strategy(optarg)); - break; - case 'x': - excluded_strategies.insert(cstring_to_strategy(optarg)); - break; - case '?': - default: - abort(); - }; - } - // Default to everything if nothing was explicitly included. - if (candidate_strategies.empty()) { - candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT); - candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_BEST_FIT); - candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT); - candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE); - } - // ..but remove anything that was explicitly excluded - for (set::const_iterator it = excluded_strategies.begin(); - it != excluded_strategies.end(); it++) { - candidate_strategies.erase(*it); - } - - // Run the real trace - // - // First, read the raw trace from stdin - vector canonicalized_trace = canonicalize_trace_from(stdin); - - if (!only_aggregate_reports) { - printf("\n"); - printf("Individual reports, by allocator:\n"); - } - - struct canonical_trace_stats stats; - map reports_by_strategy; - for (set::const_iterator it = candidate_strategies.begin(); - it != candidate_strategies.end(); it++) { - const block_allocator::allocation_strategy strategy(*it); - - // replay the canonicalized trace against the current strategy. - // - // we provided the allocator map so we can gather statistics later - struct canonical_trace_stats dummy_stats; - map reports; - replay_canonicalized_trace(canonicalized_trace, strategy, &reports, - // Only need to gather canonical trace stats once - it == candidate_strategies.begin() ? &stats : &dummy_stats); - - struct fragmentation_report aggregate_report; - memset(&aggregate_report, 0, sizeof(aggregate_report)); - for (map::iterator rp = reports.begin(); - rp != reports.end(); rp++) { - const struct fragmentation_report &report = rp->second; - aggregate_report.merge(report); - if (!only_aggregate_reports) { - print_result(rp->first, strategy, report); - } - } - reports_by_strategy[strategy] = aggregate_report; - } - - printf("\n"); - printf("Aggregate reports, by strategy:\n"); - - for (map::iterator it = reports_by_strategy.begin(); - it != reports_by_strategy.end(); it++) { - print_result(0, it->first, it->second); - } - - printf("\n"); - printf("Overall trace stats:\n"); - printf("\n"); - printf(" n_lines_played: %15" PRIu64 "\n", stats.n_lines_replayed); - printf(" n_create: %15" PRIu64 "\n", stats.n_create); - printf(" n_create_from_blockpairs: %15" PRIu64 "\n", stats.n_create_from_blockpairs); - printf(" n_alloc_hot: %15" PRIu64 "\n", stats.n_alloc_hot); - printf(" n_alloc_cold: %15" PRIu64 "\n", stats.n_alloc_cold); - printf(" n_free: %15" PRIu64 "\n", stats.n_free); - printf(" n_destroy: %15" PRIu64 "\n", stats.n_destroy); - printf("\n"); - printf(" avg_alloc_hot: %15" PRIu64 "\n", stats.alloc_hot_bytes.mean); - printf(" stddev_alloc_hot: %15" PRIu64 "\n", (uint64_t) sqrt(stats.alloc_hot_bytes.variance)); - printf(" avg_alloc_cold: %15" PRIu64 "\n", stats.alloc_cold_bytes.mean); - printf(" stddev_alloc_cold: %15" PRIu64 "\n", (uint64_t) sqrt(stats.alloc_cold_bytes.variance)); - printf("\n"); - - return 0; -} diff --git a/storage/tokudb/PerconaFT/tools/ftverify.cc b/storage/tokudb/PerconaFT/tools/ftverify.cc index 5920be8dedaa7..2324249ba00f2 100644 --- a/storage/tokudb/PerconaFT/tools/ftverify.cc +++ b/storage/tokudb/PerconaFT/tools/ftverify.cc @@ -148,7 +148,7 @@ deserialize_headers(int fd, struct ft **h1p, struct ft **h2p) } } { - toku_off_t header_1_off = block_allocator::BLOCK_ALLOCATOR_HEADER_RESERVE; + toku_off_t header_1_off = BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE; r1 = deserialize_ft_from_fd_into_rbuf( fd, header_1_off, diff --git a/storage/tokudb/PerconaFT/tools/tokuftdump.cc b/storage/tokudb/PerconaFT/tools/tokuftdump.cc index 23ef72218ac31..f6d777b416113 100644 --- a/storage/tokudb/PerconaFT/tools/tokuftdump.cc +++ b/storage/tokudb/PerconaFT/tools/tokuftdump.cc @@ -192,6 +192,7 @@ static void dump_header(FT ft) { dump_descriptor(&ft->descriptor); printf(" estimated numrows=%" PRId64 "\n", ft->in_memory_stats.numrows); printf(" estimated numbytes=%" PRId64 "\n", ft->in_memory_stats.numbytes); + printf(" logical row count=%" PRId64 "\n", ft->in_memory_logical_rows); } static int64_t getRootNode(FT ft) { diff --git a/storage/tokudb/PerconaFT/util/tests/x1764-test.cc b/storage/tokudb/PerconaFT/util/tests/x1764-test.cc index 48ff28e89af15..76b1d9c713e48 100644 --- a/storage/tokudb/PerconaFT/util/tests/x1764-test.cc +++ b/storage/tokudb/PerconaFT/util/tests/x1764-test.cc @@ -110,7 +110,7 @@ test2 (void) { static void test3 (void) -// Compare the simple version to the highly optimized verison. +// Compare the simple version to the highly optimized version. { const int datalen = 1000; char data[datalen]; diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 25f77301696b2..a77f46de9d0b1 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -369,17 +369,17 @@ void TOKUDB_SHARE::update_row_count( pct_of_rows_changed_to_trigger = ((_rows * auto_threshold) / 100); if (_row_delta_activity >= pct_of_rows_changed_to_trigger) { char msg[200]; - snprintf( - msg, - sizeof(msg), - "TokuDB: Auto %s background analysis for %s, delta_activity " - "%llu is greater than %llu percent of %llu rows.", - tokudb::sysvars::analyze_in_background(thd) > 0 ? - "scheduling" : "running", - full_table_name(), - _row_delta_activity, - auto_threshold, - (ulonglong)(_rows)); + snprintf(msg, + sizeof(msg), + "TokuDB: Auto %s analysis for %s, delta_activity %llu is " + "greater than %llu percent of %llu rows.", + tokudb::sysvars::analyze_in_background(thd) > 0 + ? "scheduling background" + : "running foreground", + full_table_name(), + _row_delta_activity, + auto_threshold, + (ulonglong)(_rows)); // analyze_standard will unlock _mutex regardless of success/failure int ret = analyze_standard(thd, NULL); @@ -4096,7 +4096,7 @@ int ha_tokudb::write_row(uchar * record) { goto cleanup; } if (curr_num_DBs == 1) { - error = insert_row_to_main_dictionary(record,&prim_key, &row, txn); + error = insert_row_to_main_dictionary(record, &prim_key, &row, txn); if (error) { goto cleanup; } } else { error = insert_rows_to_dictionaries_mult(&prim_key, &row, txn, thd); @@ -6130,7 +6130,7 @@ int ha_tokudb::info(uint flag) { // we should always have a primary key assert_always(share->file != NULL); - error = estimate_num_rows(share->file,&num_rows, txn); + error = estimate_num_rows(share->file, &num_rows, txn); if (error == 0) { share->set_row_count(num_rows, false); stats.records = num_rows; diff --git a/storage/tokudb/ha_tokudb_admin.cc b/storage/tokudb/ha_tokudb_admin.cc index db3d6c112d499..6d8e7173c8d8b 100644 --- a/storage/tokudb/ha_tokudb_admin.cc +++ b/storage/tokudb/ha_tokudb_admin.cc @@ -7,7 +7,7 @@ This file is part of TokuDB Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - TokuDBis is free software: you can redistribute it and/or modify + TokuDB is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2, as published by the Free Software Foundation. @@ -43,13 +43,11 @@ class recount_rows_t : public tokudb::background::job_manager_t::job_t { virtual ~recount_rows_t(); virtual const char* key(); - - virtual void status( - char* database, - char* table, - char* type, - char* params, - char* status); + virtual const char* database(); + virtual const char* table(); + virtual const char* type(); + virtual const char* parameters(); + virtual const char* status(); protected: virtual void on_run(); @@ -64,6 +62,8 @@ class recount_rows_t : public tokudb::background::job_manager_t::job_t { ulonglong _throttle; // for recount rows status reporting + char _parameters[256]; + char _status[1024]; int _result; ulonglong _recount_start; // in microseconds ulonglong _total_elapsed_time; // in microseconds @@ -78,7 +78,6 @@ class recount_rows_t : public tokudb::background::job_manager_t::job_t { uint64_t deleted, void* extra); int analyze_recount_rows_progress(uint64_t count, uint64_t deleted); - void get_analyze_status(char*); }; void* recount_rows_t::operator new(size_t sz) { @@ -114,10 +113,19 @@ recount_rows_t::recount_rows_t( } _throttle = tokudb::sysvars::analyze_throttle(thd); + + snprintf(_parameters, + sizeof(_parameters), + "TOKUDB_ANALYZE_THROTTLE=%llu;", + _throttle); + _status[0] = '\0'; } recount_rows_t::~recount_rows_t() { } void recount_rows_t::on_run() { + const char* orig_proc_info = NULL; + if (_thd) + orig_proc_info = tokudb_thd_get_proc_info(_thd); _recount_start = tokudb::time::microsec(); _total_elapsed_time = 0; @@ -171,6 +179,8 @@ void recount_rows_t::on_run() { _result, _share->row_count()); error: + if(_thd) + tokudb_thd_set_proc_info(_thd, orig_proc_info); return; } void recount_rows_t::on_destroy() { @@ -179,18 +189,21 @@ void recount_rows_t::on_destroy() { const char* recount_rows_t::key() { return _share->full_table_name(); } -void recount_rows_t::status( - char* database, - char* table, - char* type, - char* params, - char* status) { - - strcpy(database, _share->database_name()); - strcpy(table, _share->table_name()); - strcpy(type, "TOKUDB_ANALYZE_MODE_RECOUNT_ROWS"); - sprintf(params, "TOKUDB_ANALYZE_THROTTLE=%llu;", _throttle); - get_analyze_status(status); +const char* recount_rows_t::database() { + return _share->database_name(); +} +const char* recount_rows_t::table() { + return _share->table_name(); +} +const char* recount_rows_t::type() { + static const char* type = "TOKUDB_ANALYZE_MODE_RECOUNT_ROWS"; + return type; +} +const char* recount_rows_t::parameters() { + return _parameters; +} +const char* recount_rows_t::status() { + return _status; } int recount_rows_t::analyze_recount_rows_progress( uint64_t count, @@ -217,12 +230,32 @@ int recount_rows_t::analyze_recount_rows_progress( return ER_ABORTING_CONNECTION; } + // rebuild status + // There is a slight race condition here, + // _status is used here for tokudb_thd_set_proc_info and it is also used + // for the status column in i_s.background_job_status. + // If someone happens to be querying/building the i_s table + // at the exact same time that the status is being rebuilt here, + // the i_s table could get some garbage status. + // This solution is a little heavy handed but it works, it prevents us + // from changing the status while someone might be immediately observing + // us and it prevents someone from observing us while we change the + // status + tokudb::background::_job_manager->lock(); + snprintf(_status, + sizeof(_status), + "recount_rows %s.%s counted %llu rows and %llu deleted " + "in %llu seconds.", + _share->database_name(), + _share->table_name(), + _rows, + _deleted_rows, + _total_elapsed_time / tokudb::time::MICROSECONDS); + tokudb::background::_job_manager->unlock(); + // report - if (_thd) { - char status[256]; - get_analyze_status(status); - thd_proc_info(_thd, status); - } + if (_thd) + tokudb_thd_set_proc_info(_thd, _status); // throttle // given the throttle value, lets calculate the maximum number of rows @@ -238,18 +271,6 @@ int recount_rows_t::analyze_recount_rows_progress( } return 0; } -void recount_rows_t::get_analyze_status(char* msg) { - sprintf( - msg, - "recount_rows %s.%s counted %llu rows and %llu deleted in %llu " - "seconds.", - _share->database_name(), - _share->table_name(), - _rows, - _deleted_rows, - _total_elapsed_time / tokudb::time::MICROSECONDS); -} - class standard_t : public tokudb::background::job_manager_t::job_t { public: @@ -261,13 +282,11 @@ class standard_t : public tokudb::background::job_manager_t::job_t { virtual ~standard_t(); virtual const char* key(void); - - virtual void status( - char* database, - char* table, - char* type, - char* params, - char* status); + virtual const char* database(); + virtual const char* table(); + virtual const char* type(); + virtual const char* parameters(); + virtual const char* status(); protected: virtual void on_run(); @@ -284,6 +303,8 @@ class standard_t : public tokudb::background::job_manager_t::job_t { double _delete_fraction; // for analyze status reporting, may also use other state + char _parameters[256]; + char _status[1024]; int _result; ulonglong _analyze_start; // in microseconds ulonglong _total_elapsed_time; // in microseconds @@ -305,7 +326,6 @@ class standard_t : public tokudb::background::job_manager_t::job_t { uint64_t deleted_rows); bool analyze_standard_cursor_callback(uint64_t deleted_rows); - void get_analyze_status(char*); int analyze_key_progress(); int analyze_key(uint64_t* rec_per_key_part); }; @@ -351,6 +371,16 @@ standard_t::standard_t( _time_limit = tokudb::sysvars::analyze_time(thd) * tokudb::time::MICROSECONDS; _delete_fraction = tokudb::sysvars::analyze_delete_fraction(thd); + + snprintf(_parameters, + sizeof(_parameters), + "TOKUDB_ANALYZE_DELETE_FRACTION=%f; " + "TOKUDB_ANALYZE_TIME=%llu; TOKUDB_ANALYZE_THROTTLE=%llu;", + _delete_fraction, + _time_limit / tokudb::time::MICROSECONDS, + _throttle); + + _status[0] = '\0'; } standard_t::~standard_t() { } @@ -358,6 +388,10 @@ void standard_t::on_run() { DB_BTREE_STAT64 stat64; uint64_t rec_per_key_part[_share->_max_key_parts]; uint64_t total_key_parts = 0; + const char* orig_proc_info = NULL; + if (_thd) + orig_proc_info = tokudb_thd_get_proc_info(_thd); + _analyze_start = tokudb::time::microsec(); _half_time = _time_limit > 0 ? _time_limit/2 : 0; @@ -395,7 +429,7 @@ void standard_t::on_run() { _result = HA_ADMIN_FAILED; } if (_thd && (_result == HA_ADMIN_FAILED || - (double)_deleted_rows > + static_cast(_deleted_rows) > _delete_fraction * (_rows + _deleted_rows))) { char name[256]; int namelen; @@ -460,8 +494,9 @@ void standard_t::on_run() { } error: + if (_thd) + tokudb_thd_set_proc_info(_thd, orig_proc_info); return; - } void standard_t::on_destroy() { _share->lock(); @@ -472,24 +507,21 @@ void standard_t::on_destroy() { const char* standard_t::key() { return _share->full_table_name(); } -void standard_t::status( - char* database, - char* table, - char* type, - char* params, - char* status) { - - strcpy(database, _share->database_name()); - strcpy(table, _share->table_name()); - strcpy(type, "TOKUDB_ANALYZE_MODE_STANDARD"); - sprintf( - params, - "TOKUDB_ANALYZE_DELETE_FRACTION=%f; " - "TOKUDB_ANALYZE_TIME=%llu; TOKUDB_ANALYZE_THROTTLE=%llu;", - _delete_fraction, - _time_limit / tokudb::time::MICROSECONDS, - _throttle); - get_analyze_status(status); +const char* standard_t::database() { + return _share->database_name(); +} +const char* standard_t::table() { + return _share->table_name(); +} +const char* standard_t::type() { + static const char* type = "TOKUDB_ANALYZE_MODE_STANDARD"; + return type; +} +const char* standard_t::parameters() { + return _parameters; +} +const char* standard_t::status() { + return _status; } bool standard_t::analyze_standard_cursor_callback( void* extra, @@ -502,41 +534,6 @@ bool standard_t::analyze_standard_cursor_callback(uint64_t deleted_rows) { _ticks += deleted_rows; return analyze_key_progress() != 0; } -void standard_t::get_analyze_status(char* msg) { - static const char* scan_direction_str[] = { - "not scanning", - "scanning forward", - "scanning backward", - "scan unknown" - }; - - const char* scan_direction = NULL; - switch (_scan_direction) { - case 0: scan_direction = scan_direction_str[0]; break; - case DB_NEXT: scan_direction = scan_direction_str[1]; break; - case DB_PREV: scan_direction = scan_direction_str[2]; break; - default: scan_direction = scan_direction_str[3]; break; - } - - float progress_rows = 0.0; - if (_share->row_count() > 0) - progress_rows = (float) _rows / (float) _share->row_count(); - float progress_time = 0.0; - if (_time_limit > 0) - progress_time = (float) _key_elapsed_time / (float) _time_limit; - sprintf( - msg, - "analyze table standard %s.%s.%s %llu of %u %.lf%% rows %.lf%% time, " - "%s", - _share->database_name(), - _share->table_name(), - _share->_key_descriptors[_current_key]._name, - _current_key, - _share->_keys, - progress_rows * 100.0, - progress_time * 100.0, - scan_direction); -} int standard_t::analyze_key_progress(void) { if (_ticks > 1000) { _ticks = 0; @@ -546,19 +543,72 @@ int standard_t::analyze_key_progress(void) { if ((_thd && thd_killed(_thd)) || cancelled()) { // client killed return ER_ABORTING_CONNECTION; - } else if(_time_limit > 0 && - (uint64_t)_key_elapsed_time > _time_limit) { + } else if (_time_limit > 0 && + static_cast(_key_elapsed_time) > _time_limit) { // time limit reached return ETIME; } - // report - if (_thd) { - char status[256]; - get_analyze_status(status); - thd_proc_info(_thd, status); + // rebuild status + // There is a slight race condition here, + // _status is used here for tokudb_thd_set_proc_info and it is also used + // for the status column in i_s.background_job_status. + // If someone happens to be querying/building the i_s table + // at the exact same time that the status is being rebuilt here, + // the i_s table could get some garbage status. + // This solution is a little heavy handed but it works, it prevents us + // from changing the status while someone might be immediately observing + // us and it prevents someone from observing us while we change the + // status. + static const char* scan_direction_str[] = {"not scanning", + "scanning forward", + "scanning backward", + "scan unknown"}; + + const char* scan_direction = NULL; + switch (_scan_direction) { + case 0: + scan_direction = scan_direction_str[0]; + break; + case DB_NEXT: + scan_direction = scan_direction_str[1]; + break; + case DB_PREV: + scan_direction = scan_direction_str[2]; + break; + default: + scan_direction = scan_direction_str[3]; + break; } + float progress_rows = 0.0; + if (_share->row_count() > 0) + progress_rows = static_cast(_rows) / + static_cast(_share->row_count()); + float progress_time = 0.0; + if (_time_limit > 0) + progress_time = static_cast(_key_elapsed_time) / + static_cast(_time_limit); + tokudb::background::_job_manager->lock(); + snprintf( + _status, + sizeof(_status), + "analyze table standard %s.%s.%s %llu of %u %.lf%% rows %.lf%% " + "time, %s", + _share->database_name(), + _share->table_name(), + _share->_key_descriptors[_current_key]._name, + _current_key, + _share->_keys, + progress_rows * 100.0, + progress_time * 100.0, + scan_direction); + tokudb::background::_job_manager->unlock(); + + // report + if (_thd) + tokudb_thd_set_proc_info(_thd, _status); + // throttle // given the throttle value, lets calculate the maximum number of rows // we should have seen so far in a .1 sec resolution @@ -694,6 +744,11 @@ int standard_t::analyze_key(uint64_t* rec_per_key_part) { assert_always(close_error == 0); done: + // in case we timed out (bunch of deleted records) without hitting a + // single row + if (_rows == 0) + _rows = 1; + // return cardinality for (uint64_t i = 0; i < num_key_parts; i++) { rec_per_key_part[i] = _rows / unique_rows[i]; @@ -733,7 +788,6 @@ int TOKUDB_SHARE::analyze_recount_rows(THD* thd,DB_TXN* txn) { assert_always(thd != NULL); - const char *orig_proc_info = tokudb_thd_get_proc_info(thd); int result = HA_ADMIN_OK; tokudb::analyze::recount_rows_t* job @@ -753,8 +807,6 @@ int TOKUDB_SHARE::analyze_recount_rows(THD* thd,DB_TXN* txn) { result = HA_ADMIN_FAILED; } - thd_proc_info(thd, orig_proc_info); - TOKUDB_HANDLER_DBUG_RETURN(result); } @@ -778,8 +830,6 @@ int TOKUDB_SHARE::analyze_standard(THD* thd, DB_TXN* txn) { TOKUDB_HANDLER_DBUG_RETURN(result); } - const char *orig_proc_info = tokudb_thd_get_proc_info(thd); - tokudb::analyze::standard_t* job = new tokudb::analyze::standard_t(txn == NULL ? false : true, thd, this, txn); @@ -808,8 +858,6 @@ int TOKUDB_SHARE::analyze_standard(THD* thd, DB_TXN* txn) { lock(); - thd_proc_info(thd, orig_proc_info); - TOKUDB_HANDLER_DBUG_RETURN(result); } diff --git a/storage/tokudb/hatoku_defines.h b/storage/tokudb/hatoku_defines.h index 0269c47ffa37a..83f8a8a21d21b 100644 --- a/storage/tokudb/hatoku_defines.h +++ b/storage/tokudb/hatoku_defines.h @@ -7,7 +7,7 @@ This file is part of TokuDB Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - TokuDBis is free software: you can redistribute it and/or modify + TokuDB is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2, as published by the Free Software Foundation. @@ -232,9 +232,12 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. // mysql 5.6.15 removed the test macro, so we define our own #define tokudb_test(e) ((e) ? 1 : 0) -inline const char* tokudb_thd_get_proc_info(const THD *thd) { +inline const char* tokudb_thd_get_proc_info(const THD* thd) { return thd->proc_info; } +inline void tokudb_thd_set_proc_info(THD* thd, const char* proc_info) { + thd_proc_info(thd, proc_info); +} // uint3korr reads 4 bytes and valgrind reports an error, so we use this function instead inline uint tokudb_uint3korr(const uchar *a) { diff --git a/storage/tokudb/mysql-test/tokudb/r/background_job_manager.result b/storage/tokudb/mysql-test/tokudb/r/background_job_manager.result index 5282f0ec9ae90..8b24c3c9f729c 100644 --- a/storage/tokudb/mysql-test/tokudb/r/background_job_manager.result +++ b/storage/tokudb/mysql-test/tokudb/r/background_job_manager.result @@ -25,7 +25,7 @@ TokuDB_background_job_status CREATE TEMPORARY TABLE `TokuDB_background_job_statu `scheduler` varchar(32) NOT NULL DEFAULT '', `scheduled_time` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `started_time` datetime DEFAULT NULL, - `status` varchar(256) DEFAULT NULL + `status` varchar(1024) DEFAULT NULL ) ENGINE=MEMORY DEFAULT CHARSET=utf8 create table t1 (a int not null auto_increment, b int, c int, primary key(a), key kb(b), key kc(c), key kabc(a,b,c), key kab(a,b), key kbc(b,c)); insert into t1(b,c) values(0,0), (1,1), (2,2), (3,3); diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store.test b/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store.test index 6100d9aeec230..8b6df4966f498 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store.test @@ -12,33 +12,11 @@ let $MYSQLD_DATADIR= `SELECT @@datadir`; create table foo (a int, b int); create table bar (a int, key(a)); -# Write file to make mysql-test-run.pl expect the "crash", but don't start -# it until it's told to ---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -wait -EOF - -# Send shutdown to the connected server and give -# it 10 seconds to die before zapping it -shutdown_server 10; - +--source include/shutdown_mysqld.inc remove_file $MYSQLD_DATADIR/test/foo.frm; copy_file $MYSQLD_DATADIR/test/bar.frm $MYSQLD_DATADIR/test/foo.frm; remove_file $MYSQLD_DATADIR/test/bar.frm; - -# Write file to make mysql-test-run.pl start up the server again ---append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart -EOF - -# Turn on reconnect ---enable_reconnect - -# Call script that will poll the server waiting for it to be back online again ---source include/wait_until_connected_again.inc - -# Turn off reconnect again ---disable_reconnect +--source include/start_mysqld.inc show create table foo; show create table bar; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store2.test b/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store2.test index e1acea13ed703..53c1037b051d8 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store2.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store2.test @@ -15,33 +15,11 @@ create table bar (a int); alter table foo drop column a; alter table bar add column b int, add column c int; -# Write file to make mysql-test-run.pl expect the "crash", but don't start -# it until it's told to ---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -wait -EOF - -# Send shutdown to the connected server and give -# it 10 seconds to die before zapping it -shutdown_server 10; - +--source include/shutdown_mysqld.inc remove_file $MYSQLD_DATADIR/test/foo.frm; copy_file $MYSQLD_DATADIR/test/bar.frm $MYSQLD_DATADIR/test/foo.frm; remove_file $MYSQLD_DATADIR/test/bar.frm; - -# Write file to make mysql-test-run.pl start up the server again ---append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart -EOF - -# Turn on reconnect ---enable_reconnect - -# Call script that will poll the server waiting for it to be back online again ---source include/wait_until_connected_again.inc - -# Turn off reconnect again ---disable_reconnect +--source include/start_mysqld.inc show create table foo; show create table bar; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store3.test b/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store3.test index 17a124249da26..0421b8e9d269f 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store3.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/frm_store3.test @@ -14,33 +14,11 @@ create table bar (a bigint)engine=TokuDB; alter table foo drop index b; alter table bar add index (a); -# Write file to make mysql-test-run.pl expect the "crash", but don't start -# it until it's told to ---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -wait -EOF - -# Send shutdown to the connected server and give -# it 10 seconds to die before zapping it -shutdown_server 10; - +--source include/shutdown_mysqld.inc remove_file $MYSQLD_DATADIR/test/foo.frm; copy_file $MYSQLD_DATADIR/test/bar.frm $MYSQLD_DATADIR/test/foo.frm; remove_file $MYSQLD_DATADIR/test/bar.frm; - -# Write file to make mysql-test-run.pl start up the server again ---append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart -EOF - -# Turn on reconnect ---enable_reconnect - -# Call script that will poll the server waiting for it to be back online again ---source include/wait_until_connected_again.inc - -# Turn off reconnect again ---disable_reconnect +--source include/start_mysqld.inc show create table foo; show create table bar; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_part_table_668.test b/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_part_table_668.test index 42dbb30058a74..4c40339be5a2f 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_part_table_668.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_part_table_668.test @@ -7,17 +7,7 @@ set default_storage_engine='tokudb'; # capture the datadir let $MYSQLD_DATADIR= `SELECT @@datadir`; -# shutdown mysqld (code stolen from mysql_plugin.test) -let $expect_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; -# MTR will remove this file later, but this might be too late. ---error 0,1 ---remove_file $expect_file ---write_file $expect_file -wait -EOF ---shutdown_server 10 ---source include/wait_until_disconnected.inc - +--source include/shutdown_mysqld.inc # remove all tokudb file in the datadir system mkdir $MYSQLD_DATADIR/save; system mv $MYSQLD_DATADIR/*toku* $MYSQLD_DATADIR/test $MYSQLD_DATADIR/save; @@ -25,13 +15,7 @@ system mkdir $MYSQLD_DATADIR/test; # install 6.6.8 tokudb test files system cp -r std_data/tokudb_drop_part_table_668/data/* $MYSQLD_DATADIR; - -# restart mysqld ---append_file $expect_file -restart -EOF ---enable_reconnect ---source include/wait_until_connected_again.inc +--source include/start_mysqld.inc create table tc (a int, b int, c int, primary key(a), key(b)) engine=tokudb partition by hash(a) partitions 2; @@ -45,26 +29,9 @@ select dictionary_name from information_schema.tokudb_file_map; # check that the test dir is empty list_files $MYSQLD_DATADIR/test *.frm; -# shutdown mysqld (code stolen from mysql_plugin.test) -let $expect_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; -# MTR will remove this file later, but this might be too late. ---error 0,1 ---remove_file $expect_file ---write_file $expect_file -wait -EOF ---shutdown_server 10 ---source include/wait_until_disconnected.inc - +--source include/shutdown_mysqld.inc # restore saved datadir system rm -rf $MYSQLD_DATADIR/*toku* $MYSQLD_DATADIR/test; system mv $MYSQLD_DATADIR/save/* $MYSQLD_DATADIR; system rmdir $MYSQLD_DATADIR/save; - -# restart mysqld ---append_file $expect_file -restart -EOF ---enable_reconnect ---source include/wait_until_connected_again.inc - +--source include/start_mysqld.inc diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_simple_table_668.test b/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_simple_table_668.test index 3903c2cef9f28..0340b960fa5ab 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_simple_table_668.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/tokudb_drop_simple_table_668.test @@ -6,17 +6,7 @@ set default_storage_engine='tokudb'; # capture the datadir let $MYSQLD_DATADIR= `SELECT @@datadir`; -# shutdown mysqld (code stolen from mysql_plugin.test) -let $expect_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; -# MTR will remove this file later, but this might be too late. ---error 0,1 ---remove_file $expect_file ---write_file $expect_file -wait -EOF ---shutdown_server 10 ---source include/wait_until_disconnected.inc - +--source include/shutdown_mysqld.inc # remove all tokudb file in the datadir system mkdir $MYSQLD_DATADIR/save; system mv $MYSQLD_DATADIR/*toku* $MYSQLD_DATADIR/test $MYSQLD_DATADIR/save; @@ -24,13 +14,7 @@ system mkdir $MYSQLD_DATADIR/test; # install 6.6.8 tokudb test files system cp -r std_data/tokudb_drop_simple_table_668/data/* $MYSQLD_DATADIR; - -# restart mysqld ---append_file $expect_file -restart -EOF ---enable_reconnect ---source include/wait_until_connected_again.inc +--source include/start_mysqld.inc create table tc (id int, x int, primary key(id), key(x)); @@ -46,26 +30,9 @@ select dictionary_name from information_schema.tokudb_file_map; # check that the test dir is empty list_files $MYSQLD_DATADIR/test *.frm; -# shutdown mysqld (code stolen from mysql_plugin.test) -let $expect_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; -# MTR will remove this file later, but this might be too late. ---error 0,1 ---remove_file $expect_file ---write_file $expect_file -wait -EOF ---shutdown_server 10 ---source include/wait_until_disconnected.inc - +--source include/shutdown_mysqld.inc # restore saved datadir system rm -rf $MYSQLD_DATADIR/*toku* $MYSQLD_DATADIR/test; system mv $MYSQLD_DATADIR/save/* $MYSQLD_DATADIR; system rmdir $MYSQLD_DATADIR/save; - -# restart mysqld ---append_file $expect_file -restart -EOF ---enable_reconnect ---source include/wait_until_connected_again.inc - +--source include/start_mysqld.inc diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_foreign_key_tokudb.result b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_foreign_key_tokudb.result deleted file mode 100644 index cdab171d4134e..0000000000000 --- a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_foreign_key_tokudb.result +++ /dev/null @@ -1,58 +0,0 @@ -include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. -[connection master] -CREATE TABLE t1 (a INT AUTO_INCREMENT KEY) ENGINE=TokuDB; -CREATE TABLE t2 (b INT AUTO_INCREMENT KEY, c INT, FOREIGN KEY(b) REFERENCES t1(a)) ENGINE=TokuDB; -SET FOREIGN_KEY_CHECKS=0; -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (NULL),(NULL),(NULL); -INSERT INTO t2 VALUES (5,0); -INSERT INTO t2 VALUES (NULL,LAST_INSERT_ID()); -SET FOREIGN_KEY_CHECKS=1; -SELECT * FROM t1 ORDER BY a; -a -10 -11 -12 -13 -SELECT * FROM t2 ORDER BY b; -b c -5 0 -6 11 -include/sync_slave_sql_with_master.inc -SELECT * FROM t1 ORDER BY a; -a -10 -11 -12 -13 -SELECT * FROM t2 ORDER BY b; -b c -5 0 -6 11 -SET TIMESTAMP=1000000000; -CREATE TABLE t3 ( a INT UNIQUE ); -SET FOREIGN_KEY_CHECKS=0; -INSERT INTO t3 VALUES (1),(1); -Got one of the listed errors -include/sync_slave_sql_with_master.inc -SET FOREIGN_KEY_CHECKS=0; -DROP TABLE IF EXISTS t1,t2,t3; -SET FOREIGN_KEY_CHECKS=1; -include/sync_slave_sql_with_master.inc -create table t1 (b int primary key) engine = TokuDB; -create table t2 (a int primary key, b int, foreign key (b) references t1(b)) -engine = TokuDB; -insert into t1 set b=1; -insert into t2 set a=1, b=1; -set foreign_key_checks=0; -delete from t1; -must sync w/o a problem (could not with the buggy code) -include/sync_slave_sql_with_master.inc -select count(*) from t1 /* must be zero */; -count(*) -0 -drop table t2,t1; -include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_foreign_key_tokudb.test b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_foreign_key_tokudb.test deleted file mode 100644 index d798cfd4a6232..0000000000000 --- a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_foreign_key_tokudb.test +++ /dev/null @@ -1,4 +0,0 @@ --- source include/not_ndb_default.inc --- source include/have_tokudb.inc -let $engine_type=TokuDB; --- source extra/rpl_tests/rpl_foreign_key.test diff --git a/storage/tokudb/tokudb_background.cc b/storage/tokudb/tokudb_background.cc index d8ef54a59729e..e019e41c78824 100644 --- a/storage/tokudb/tokudb_background.cc +++ b/storage/tokudb/tokudb_background.cc @@ -8,7 +8,7 @@ This file is part of TokuDB Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - TokuDBis is free software: you can redistribute it and/or modify + TokuDB is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2, as published by the Free Software Foundation. @@ -68,7 +68,8 @@ void job_manager_t::destroy() { while (_background_jobs.size()) { _mutex.lock(); job_t* job = _background_jobs.front(); - cancel(job); + if (!job->cancelled()) + cancel(job); _background_jobs.pop_front(); delete job; _mutex.unlock(); @@ -148,11 +149,8 @@ bool job_manager_t::cancel_job(const char* key) { it != _background_jobs.end(); it++) { job_t* job = *it; - if (!job->cancelled() && - strcmp(job->key(), key) == 0) { - + if (!job->cancelled() && strcmp(job->key(), key) == 0) { cancel(job); - ret = true; } } @@ -162,8 +160,6 @@ bool job_manager_t::cancel_job(const char* key) { } void job_manager_t::iterate_jobs(pfn_iterate_t callback, void* extra) const { - char database[256], table[256], type[256], params[256], status[256]; - _mutex.lock(); for (jobs_t::const_iterator it = _background_jobs.begin(); @@ -171,19 +167,7 @@ void job_manager_t::iterate_jobs(pfn_iterate_t callback, void* extra) const { it++) { job_t* job = *it; if (!job->cancelled()) { - database[0] = table[0] = type[0] = params[0] = status[0] = '\0'; - job->status(database, table, type, params, status); - callback( - job->id(), - database, - table, - type, - params, - status, - job->user_scheduled(), - job->scheduled_time(), - job->started_time(), - extra); + callback(job, extra); } } @@ -233,6 +217,7 @@ void job_manager_t::run(job_t* job) { } void job_manager_t::cancel(job_t* job) { assert_debug(_mutex.is_owned_by_me()); + assert_always(!job->cancelled()); job->cancel(); } job_manager_t* _job_manager = NULL; diff --git a/storage/tokudb/tokudb_background.h b/storage/tokudb/tokudb_background.h index 3786701fd0fa0..29991ab325d54 100644 --- a/storage/tokudb/tokudb_background.h +++ b/storage/tokudb/tokudb_background.h @@ -7,7 +7,7 @@ This file is part of TokuDB Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. - TokuDBis is free software: you can redistribute it and/or modify + TokuDB is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2, as published by the Free Software Foundation. @@ -58,13 +58,20 @@ class job_manager_t { // (or jobs) usually used to find jobs to cancel virtual const char* key() = 0; - // method to get info for information schema, 255 chars per buffer - virtual void status( - char* database, - char* table, - char* type, - char* params, - char* status) = 0; + // method to obtain the database name the job is scheduled on + virtual const char* database() = 0; + + // method to obtain the table name the job is scheduled on + virtual const char* table() = 0; + + // method to obtain the type of job + virtual const char* type() = 0; + + // method to obtain a stringized list of job parameters + virtual const char* parameters() = 0; + + // method to obtain a sting identifying the current status of the job + virtual const char* status() = 0; inline bool running() const; @@ -99,17 +106,7 @@ class job_manager_t { }; // pfn for iterate callback - typedef void (*pfn_iterate_t)( - uint64_t, - const char*, - const char*, - const char*, - const char*, - const char*, - bool, - time_t, - time_t, - void*); + typedef void (*pfn_iterate_t)(class job_t*, void*); public: void* operator new(size_t sz); @@ -144,6 +141,11 @@ class job_manager_t { // data passed when the job was scheduled void iterate_jobs(pfn_iterate_t callback, void* extra) const; + // lock the bjm, this prevents anyone from running, cancelling or iterating + // jobs in the bjm. + inline void lock(); + inline void unlock(); + private: static void* thread_func(void* v); @@ -170,6 +172,15 @@ extern job_manager_t* _job_manager; bool initialize(); bool destroy(); +inline void job_manager_t::lock() { + assert_debug(!_mutex.is_owned_by_me()); + _mutex.lock(); +} +inline void job_manager_t::unlock() { + assert_debug(_mutex.is_owned_by_me()); + _mutex.unlock(); +} + inline void job_manager_t::job_t::run() { if (!_cancelled) { _running = true; diff --git a/storage/tokudb/tokudb_information_schema.cc b/storage/tokudb/tokudb_information_schema.cc index 1d4ca2e018132..6cdd9b275fbd1 100644 --- a/storage/tokudb/tokudb_information_schema.cc +++ b/storage/tokudb/tokudb_information_schema.cc @@ -1083,7 +1083,7 @@ ST_FIELD_INFO background_job_status_field_info[] = { {"scheduler", 32, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, {"scheduled_time", 0, MYSQL_TYPE_DATETIME, 0, 0, NULL, SKIP_OPEN_TABLE }, {"started_time", 0, MYSQL_TYPE_DATETIME, 0, MY_I_S_MAYBE_NULL, NULL, SKIP_OPEN_TABLE }, - {"status", 256, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, SKIP_OPEN_TABLE }, + {"status", 1024, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, SKIP_OPEN_TABLE }, {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} }; @@ -1093,15 +1093,7 @@ struct background_job_status_extra { }; void background_job_status_callback( - uint64_t id, - const char* database_name, - const char* table_name, - const char* type, - const char* params, - const char* status, - bool user_scheduled, - time_t scheduled_time, - time_t started_time, + tokudb::background::job_manager_t::job_t* job, void* extra) { background_job_status_extra* e = @@ -1109,24 +1101,33 @@ void background_job_status_callback( THD* thd = e->thd; TABLE* table = e->table; + const char* tmp = NULL; - table->field[0]->store(id, false); - table->field[1]->store( - database_name, - strlen(database_name), - system_charset_info); - table->field[2]->store(table_name, strlen(table_name), system_charset_info); - table->field[3]->store(type, strlen(type), system_charset_info); - table->field[4]->store(params, strlen(params), system_charset_info); - if (user_scheduled) + table->field[0]->store(job->id(), false); + + tmp = job->database(); + table->field[1]->store(tmp, strlen(tmp), system_charset_info); + + tmp = job->table(); + table->field[2]->store(tmp, strlen(tmp), system_charset_info); + + tmp = job->type(); + table->field[3]->store(tmp, strlen(tmp), system_charset_info); + + tmp = job->parameters(); + table->field[4]->store(tmp, strlen(tmp), system_charset_info); + + if (job->user_scheduled()) table->field[5]->store("USER", strlen("USER"), system_charset_info); else table->field[5]->store("AUTO", strlen("AUTO"), system_charset_info); - field_store_time_t(table->field[6], scheduled_time); - field_store_time_t(table->field[7], started_time); - if (status[0] != '\0') { - table->field[8]->store(status, strlen(status), system_charset_info); + field_store_time_t(table->field[6], job->scheduled_time()); + field_store_time_t(table->field[7], job->started_time()); + + tmp = job->status(); + if (tmp && tmp[0] != '\0') { + table->field[8]->store(tmp, strlen(tmp), system_charset_info); table->field[8]->set_notnull(); } else { table->field[8]->store(NULL, 0, system_charset_info); From 93ab3093cb6646834844139ef51c51e2c84b73a6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Sep 2016 18:00:59 +0200 Subject: [PATCH 076/295] 5.6.32-78.1 --- storage/xtradb/btr/btr0btr.cc | 16 ++-- storage/xtradb/btr/btr0cur.cc | 4 +- storage/xtradb/buf/buf0flu.cc | 2 + storage/xtradb/fil/fil0fil.cc | 6 ++ storage/xtradb/fts/fts0fts.cc | 67 ++++++++++++--- storage/xtradb/fts/fts0opt.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 121 ++++++++++++++++++++++------ storage/xtradb/handler/i_s.cc | 39 ++++++++- storage/xtradb/ibuf/ibuf0ibuf.cc | 4 +- storage/xtradb/include/buf0buf.h | 12 +++ storage/xtradb/include/buf0buf.ic | 14 ++++ storage/xtradb/include/fts0fts.h | 4 +- storage/xtradb/include/srv0srv.h | 10 ++- storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0log.cc | 4 +- storage/xtradb/log/log0online.cc | 12 ++- storage/xtradb/log/log0recv.cc | 17 ++-- storage/xtradb/row/row0merge.cc | 2 +- storage/xtradb/srv/srv0mon.cc | 7 +- storage/xtradb/srv/srv0srv.cc | 15 ++-- 20 files changed, 280 insertions(+), 80 deletions(-) diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index 95b8892415c1f..ecea98fccfebb 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -78,7 +78,7 @@ btr_corruption_report( buf_block_get_zip_size(block), BUF_PAGE_PRINT_NO_CRASH); } - buf_page_print(buf_block_get_frame(block), 0, 0); + buf_page_print(buf_nonnull_block_get_frame(block), 0, 0); } #ifndef UNIV_HOTBACKUP @@ -804,8 +804,10 @@ btr_height_get( /* S latches the page */ root_block = btr_root_block_get(index, RW_S_LATCH, mtr); + ut_ad(root_block); // The index must not be corrupted - height = btr_page_get_level(buf_block_get_frame(root_block), mtr); + height = btr_page_get_level(buf_nonnull_block_get_frame(root_block), + mtr); /* Release the S latch on the root page. */ mtr_memo_release(mtr, root_block, MTR_MEMO_PAGE_S_FIX); @@ -1231,7 +1233,7 @@ btr_get_size( SRV_CORRUPT_TABLE_CHECK(root, { mtr_commit(mtr); - return(0); + return(ULINT_UNDEFINED); }); if (flag == BTR_N_LEAF_PAGES) { @@ -2756,7 +2758,7 @@ btr_attach_half_pages( } /* Get the level of the split pages */ - level = btr_page_get_level(buf_block_get_frame(block), mtr); + level = btr_page_get_level(buf_nonnull_block_get_frame(block), mtr); ut_ad(level == btr_page_get_level(buf_block_get_frame(new_block), mtr)); @@ -4133,8 +4135,10 @@ btr_discard_page( /* Decide the page which will inherit the locks */ - left_page_no = btr_page_get_prev(buf_block_get_frame(block), mtr); - right_page_no = btr_page_get_next(buf_block_get_frame(block), mtr); + left_page_no = btr_page_get_prev(buf_nonnull_block_get_frame(block), + mtr); + right_page_no = btr_page_get_next(buf_nonnull_block_get_frame(block), + mtr); if (left_page_no != FIL_NULL) { merge_block = btr_block_get(space, zip_size, left_page_no, diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index c1efe9ca91c75..753309bdeeceb 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -1751,7 +1751,7 @@ btr_cur_pessimistic_insert( } if (!page_rec_is_infimum(btr_cur_get_rec(cursor)) || btr_page_get_prev( - buf_block_get_frame( + buf_nonnull_block_get_frame( btr_cur_get_block(cursor)), mtr) == FIL_NULL) { /* split and inserted need to call @@ -2220,7 +2220,7 @@ btr_cur_update_in_place( if (page_zip && !(flags & BTR_KEEP_IBUF_BITMAP) && !dict_index_is_clust(index) - && page_is_leaf(buf_block_get_frame(block))) { + && page_is_leaf(buf_nonnull_block_get_frame(block))) { /* Update the free bits in the insert buffer. */ ibuf_update_free_bits_zip(block, mtr); } diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 03504b15599dd..14a5fbde7e820 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -309,6 +309,8 @@ buf_flush_init_flush_rbt(void) buf_flush_list_mutex_enter(buf_pool); + ut_ad(buf_pool->flush_rbt == NULL); + /* Create red black tree for speedy insertions in flush list. */ buf_pool->flush_rbt = rbt_create( sizeof(buf_page_t*), buf_flush_block_cmp); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 75bb811198ac9..c1dbb5f91b996 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1822,6 +1822,9 @@ fil_close_all_files(void) { fil_space_t* space; + // Must check both flags as it's possible for this to be called during + // server startup with srv_track_changed_pages == true but + // srv_redo_log_thread_started == false if (srv_track_changed_pages && srv_redo_log_thread_started) os_event_wait(srv_redo_log_tracked_event); @@ -1861,6 +1864,9 @@ fil_close_log_files( { fil_space_t* space; + // Must check both flags as it's possible for this to be called during + // server startup with srv_track_changed_pages == true but + // srv_redo_log_thread_started == false if (srv_track_changed_pages && srv_redo_log_thread_started) os_event_wait(srv_redo_log_tracked_event); diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index 22278338072fa..25059db96b078 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -265,13 +265,15 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state @param[in] unlock_cache whether unlock cache lock when write node @param[in] wait whether wait when a sync is in progress +@param[in] has_dict whether has dict operation lock @return DB_SUCCESS if all OK */ static dberr_t fts_sync( fts_sync_t* sync, bool unlock_cache, - bool wait); + bool wait, + bool has_dict); /****************************************************************//** Release all resources help by the words rb tree e.g., the node ilist. */ @@ -3566,7 +3568,7 @@ fts_add_doc_by_id( DBUG_EXECUTE_IF( "fts_instrument_sync_debug", - fts_sync(cache->sync, true, true); + fts_sync(cache->sync, true, true, false); ); DEBUG_SYNC_C("fts_instrument_sync_request"); @@ -4378,13 +4380,11 @@ fts_sync_index( } /** Check if index cache has been synced completely -@param[in,out] sync sync state @param[in,out] index_cache index cache @return true if index is synced, otherwise false. */ static bool fts_sync_index_check( - fts_sync_t* sync, fts_index_cache_t* index_cache) { const ib_rbt_node_t* rbt_node; @@ -4407,14 +4407,36 @@ fts_sync_index_check( return(true); } -/*********************************************************************//** -Commit the SYNC, change state of processed doc ids etc. +/** Reset synced flag in index cache when rollback +@param[in,out] index_cache index cache */ +static +void +fts_sync_index_reset( + fts_index_cache_t* index_cache) +{ + const ib_rbt_node_t* rbt_node; + + for (rbt_node = rbt_first(index_cache->words); + rbt_node != NULL; + rbt_node = rbt_next(index_cache->words, rbt_node)) { + + fts_tokenizer_word_t* word; + word = rbt_value(fts_tokenizer_word_t, rbt_node); + + fts_node_t* fts_node; + fts_node = static_cast(ib_vector_last(word->nodes)); + + fts_node->synced = false; + } +} + +/** Commit the SYNC, change state of processed doc ids etc. +@param[in,out] sync sync state @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_sync_commit( -/*============*/ - fts_sync_t* sync) /*!< in: sync state */ + fts_sync_t* sync) { dberr_t error; trx_t* trx = sync->trx; @@ -4467,6 +4489,8 @@ fts_sync_commit( (double) n_nodes/ (double) elapsed_time); } + /* Avoid assertion in trx_free(). */ + trx->dict_operation_lock_mode = 0; trx_free_for_background(trx); return(error); @@ -4489,6 +4513,10 @@ fts_sync_rollback( index_cache = static_cast( ib_vector_get(cache->indexes, i)); + /* Reset synced flag so nodes will not be skipped + in the next sync, see fts_sync_write_words(). */ + fts_sync_index_reset(index_cache); + for (j = 0; fts_index_selector[j].value; ++j) { if (index_cache->ins_graph[j] != NULL) { @@ -4514,6 +4542,9 @@ fts_sync_rollback( rw_lock_x_unlock(&cache->lock); fts_sql_rollback(trx); + + /* Avoid assertion in trx_free(). */ + trx->dict_operation_lock_mode = 0; trx_free_for_background(trx); } @@ -4522,13 +4553,15 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state @param[in] unlock_cache whether unlock cache lock when write node @param[in] wait whether wait when a sync is in progress +@param[in] has_dict whether has dict operation lock @return DB_SUCCESS if all OK */ static dberr_t fts_sync( fts_sync_t* sync, bool unlock_cache, - bool wait) + bool wait, + bool has_dict) { ulint i; dberr_t error = DB_SUCCESS; @@ -4557,6 +4590,12 @@ fts_sync( DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); + /* When sync in background, we hold dict operation lock + to prevent DDL like DROP INDEX, etc. */ + if (has_dict) { + sync->trx->dict_operation_lock_mode = RW_S_LATCH; + } + begin_sync: if (cache->total_size > fts_max_cache_size) { /* Avoid the case: sync never finish when @@ -4597,7 +4636,7 @@ fts_sync( ib_vector_get(cache->indexes, i)); if (index_cache->index->to_be_dropped - || fts_sync_index_check(sync, index_cache)) { + || fts_sync_index_check(index_cache)) { continue; } @@ -4612,6 +4651,7 @@ fts_sync( } rw_lock_x_lock(&cache->lock); + sync->interrupted = false; sync->in_progress = false; os_event_set(sync->event); rw_lock_x_unlock(&cache->lock); @@ -4635,20 +4675,23 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table @param[in] unlock_cache whether unlock cache when write node @param[in] wait whether wait for existing sync to finish +@param[in] has_dict whether has dict operation lock @return DB_SUCCESS on success, error code on failure. */ UNIV_INTERN dberr_t fts_sync_table( dict_table_t* table, bool unlock_cache, - bool wait) + bool wait, + bool has_dict) { dberr_t err = DB_SUCCESS; ut_ad(table->fts); if (!dict_table_is_discarded(table) && table->fts->cache) { - err = fts_sync(table->fts->cache->sync, unlock_cache, wait); + err = fts_sync(table->fts->cache->sync, + unlock_cache, wait, has_dict); } return(err); diff --git a/storage/xtradb/fts/fts0opt.cc b/storage/xtradb/fts/fts0opt.cc index 1cf45961ae22b..0d45a195c95c8 100644 --- a/storage/xtradb/fts/fts0opt.cc +++ b/storage/xtradb/fts/fts0opt.cc @@ -2986,7 +2986,7 @@ fts_optimize_sync_table( if (table) { if (dict_table_has_fts_index(table) && table->fts->cache) { - fts_sync_table(table, true, false); + fts_sync_table(table, true, false, true); } dict_table_close(table, FALSE, FALSE); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index c492bfcdd1f9b..f00d11bd870c4 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -701,6 +701,19 @@ innobase_is_fake_change( THD* thd); /*!< in: MySQL thread handle of the user for whom the transaction is being committed */ +/** Get the list of foreign keys referencing a specified table +table. +@param thd The thread handle +@param path Path to the table +@param f_key_list[out] The list of foreign keys + +@return error code or zero for success */ +static +int +innobase_get_parent_fk_list( + THD* thd, + const char* path, + List* f_key_list); /******************************************************************//** Maps a MySQL trx isolation level code to the InnoDB isolation level code @@ -3405,6 +3418,7 @@ innobase_init( innobase_hton->purge_changed_page_bitmaps = innobase_purge_changed_page_bitmaps; innobase_hton->is_fake_change = innobase_is_fake_change; + innobase_hton->get_parent_fk_list = innobase_get_parent_fk_list; innobase_hton->kill_connection = innobase_kill_connection; @@ -7294,6 +7308,7 @@ dberr_t ha_innobase::innobase_lock_autoinc(void) /*====================================*/ { + DBUG_ENTER("ha_innobase::innobase_lock_autoinc"); dberr_t error = DB_SUCCESS; ut_ad(!srv_read_only_mode); @@ -7328,6 +7343,8 @@ ha_innobase::innobase_lock_autoinc(void) /* Fall through to old style locking. */ case AUTOINC_OLD_STYLE_LOCKING: + DBUG_EXECUTE_IF("die_if_autoinc_old_lock_style_used", + ut_ad(0);); error = row_lock_table_autoinc_for_mysql(prebuilt); if (error == DB_SUCCESS) { @@ -7341,7 +7358,7 @@ ha_innobase::innobase_lock_autoinc(void) ut_error; } - return(error); + DBUG_RETURN(error); } /********************************************************************//** @@ -12287,7 +12304,7 @@ ha_innobase::optimize( if (innodb_optimize_fulltext_only) { if (prebuilt->table->fts && prebuilt->table->fts->cache && !dict_table_is_discarded(prebuilt->table)) { - fts_sync_table(prebuilt->table, false, true); + fts_sync_table(prebuilt->table, false, true, false); fts_optimize_table(prebuilt->table); } return(HA_ADMIN_OK); @@ -12454,7 +12471,14 @@ ha_innobase::check( prebuilt->select_lock_type = LOCK_NONE; - if (!row_check_index_for_mysql(prebuilt, index, &n_rows)) { + bool check_result + = row_check_index_for_mysql(prebuilt, index, &n_rows); + DBUG_EXECUTE_IF( + "dict_set_index_corrupted", + if (!(index->type & DICT_CLUSTERED)) { + check_result = false; + }); + if (!check_result) { innobase_format_name( index_name, sizeof index_name, index->name, TRUE); @@ -12807,6 +12831,75 @@ get_foreign_key_info( return(pf_key_info); } +/** Get the list of foreign keys referencing a specified table +table. +@param thd The thread handle +@param path Path to the table +@param f_key_list[out] The list of foreign keys */ +static +void +fill_foreign_key_list(THD* thd, + const dict_table_t* table, + List* f_key_list) +{ + ut_ad(mutex_own(&dict_sys->mutex)); + + for (dict_foreign_set::iterator it = table->referenced_set.begin(); + it != table->referenced_set.end(); ++it) { + + dict_foreign_t* foreign = *it; + + FOREIGN_KEY_INFO* pf_key_info + = get_foreign_key_info(thd, foreign); + if (pf_key_info) { + f_key_list->push_back(pf_key_info); + } + } +} + +/** Get the list of foreign keys referencing a specified table +table. +@param thd The thread handle +@param path Path to the table +@param f_key_list[out] The list of foreign keys + +@return error code or zero for success */ +static +int +innobase_get_parent_fk_list( + THD* thd, + const char* path, + List* f_key_list) +{ + ut_a(strlen(path) <= FN_REFLEN); + char norm_name[FN_REFLEN + 1]; + normalize_table_name(norm_name, path); + + trx_t* parent_trx = check_trx_exists(thd); + parent_trx->op_info = "getting list of referencing foreign keys"; + trx_search_latch_release_if_reserved(parent_trx); + + mutex_enter(&dict_sys->mutex); + + dict_table_t* table + = dict_table_open_on_name(norm_name, TRUE, FALSE, + static_cast( + DICT_ERR_IGNORE_INDEX_ROOT + | DICT_ERR_IGNORE_CORRUPT)); + if (!table) { + mutex_exit(&dict_sys->mutex); + return(HA_ERR_NO_SUCH_TABLE); + } + + fill_foreign_key_list(thd, table, f_key_list); + + dict_table_close(table, TRUE, FALSE); + + mutex_exit(&dict_sys->mutex); + parent_trx->op_info = ""; + return(0); +} + /*******************************************************************//** Gets the list of foreign keys in this table. @return always 0, that is, always succeeds */ @@ -12859,9 +12952,6 @@ ha_innobase::get_parent_foreign_key_list( THD* thd, /*!< in: user thread handle */ List* f_key_list) /*!< out: foreign key list */ { - FOREIGN_KEY_INFO* pf_key_info; - dict_foreign_t* foreign; - ut_a(prebuilt != NULL); update_thd(ha_thd()); @@ -12870,20 +12960,7 @@ ha_innobase::get_parent_foreign_key_list( trx_search_latch_release_if_reserved(prebuilt->trx); mutex_enter(&(dict_sys->mutex)); - - for (dict_foreign_set::iterator it - = prebuilt->table->referenced_set.begin(); - it != prebuilt->table->referenced_set.end(); - ++it) { - - foreign = *it; - - pf_key_info = get_foreign_key_info(thd, foreign); - if (pf_key_info) { - f_key_list->push_back(pf_key_info); - } - } - + fill_foreign_key_list(thd, prebuilt->table, f_key_list); mutex_exit(&(dict_sys->mutex)); prebuilt->trx->op_info = ""; @@ -16597,7 +16674,6 @@ innodb_track_changed_pages_validate( for update function */ struct st_mysql_value* value) /*!< in: incoming bool */ { - static bool enabled_on_startup = false; long long intbuf = 0; if (value->val_int(value, &intbuf)) { @@ -16605,8 +16681,7 @@ innodb_track_changed_pages_validate( return 1; } - if (srv_track_changed_pages || enabled_on_startup) { - enabled_on_startup = true; + if (srv_redo_log_thread_started) { *reinterpret_cast(save) = static_cast(intbuf); return 0; diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc index 4bc834479fe9a..dfdad55ec3bef 100644 --- a/storage/xtradb/handler/i_s.cc +++ b/storage/xtradb/handler/i_s.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3016,15 +3016,26 @@ i_s_fts_deleted_generic_fill( DBUG_RETURN(0); } - deleted = fts_doc_ids_create(); + /* Prevent DDL to drop fts aux tables. */ + rw_lock_s_lock(&dict_operation_lock); user_table = dict_table_open_on_name( fts_internal_tbl_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); if (!user_table) { + rw_lock_s_unlock(&dict_operation_lock); + + DBUG_RETURN(0); + } else if (!dict_table_has_fts_index(user_table)) { + dict_table_close(user_table, FALSE, FALSE); + + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } + deleted = fts_doc_ids_create(); + trx = trx_allocate_for_background(); trx->op_info = "Select for FTS DELETE TABLE"; @@ -3052,6 +3063,8 @@ i_s_fts_deleted_generic_fill( dict_table_close(user_table, FALSE, FALSE); + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } @@ -3433,6 +3446,12 @@ i_s_fts_index_cache_fill( DBUG_RETURN(0); } + if (user_table->fts == NULL || user_table->fts->cache == NULL) { + dict_table_close(user_table, FALSE, FALSE); + + DBUG_RETURN(0); + } + cache = user_table->fts->cache; ut_a(cache); @@ -3871,10 +3890,15 @@ i_s_fts_index_table_fill( DBUG_RETURN(0); } + /* Prevent DDL to drop fts aux tables. */ + rw_lock_s_lock(&dict_operation_lock); + user_table = dict_table_open_on_name( fts_internal_tbl_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); if (!user_table) { + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } @@ -3887,6 +3911,8 @@ i_s_fts_index_table_fill( dict_table_close(user_table, FALSE, FALSE); + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } @@ -4026,14 +4052,21 @@ i_s_fts_config_fill( fields = table->field; + /* Prevent DDL to drop fts aux tables. */ + rw_lock_s_lock(&dict_operation_lock); + user_table = dict_table_open_on_name( fts_internal_tbl_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); if (!user_table) { + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } else if (!dict_table_has_fts_index(user_table)) { dict_table_close(user_table, FALSE, FALSE); + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } @@ -4089,6 +4122,8 @@ i_s_fts_config_fill( dict_table_close(user_table, FALSE, FALSE); + rw_lock_s_unlock(&dict_operation_lock); + DBUG_RETURN(0); } diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index 4334fd8c6dde8..d0d47c3b87afd 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -938,7 +938,7 @@ ibuf_set_free_bits_low( ulint space; ulint page_no; - if (!page_is_leaf(buf_block_get_frame(block))) { + if (!page_is_leaf(buf_nonnull_block_get_frame(block))) { return; } @@ -1113,7 +1113,7 @@ ibuf_update_free_bits_zip( page_no = buf_block_get_page_no(block); zip_size = buf_block_get_zip_size(block); - ut_a(page_is_leaf(buf_block_get_frame(block))); + ut_a(page_is_leaf(buf_nonnull_block_get_frame(block))); ut_a(zip_size); bitmap_page = ibuf_bitmap_get_map_page(space, page_no, zip_size, mtr); diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index b265b8661c8f0..9aadd7b12fd69 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1060,8 +1060,20 @@ buf_block_get_frame( /*================*/ const buf_block_t* block) /*!< in: pointer to the control block */ MY_ATTRIBUTE((pure)); + +/*********************************************************************//** +Gets a pointer to the memory frame of a block, where block is known not to be +NULL. +@return pointer to the frame */ +UNIV_INLINE +buf_frame_t* +buf_nonnull_block_get_frame( + const buf_block_t* block) /*!< in: pointer to the control block */ + MY_ATTRIBUTE((pure)); + #else /* UNIV_DEBUG */ # define buf_block_get_frame(block) (block ? (block)->frame : 0) +# define buf_nonnull_block_get_frame(block) ((block)->frame) #endif /* UNIV_DEBUG */ /*********************************************************************//** Gets the space id of a block. diff --git a/storage/xtradb/include/buf0buf.ic b/storage/xtradb/include/buf0buf.ic index b40285ae3f008..8a21f44a2eec4 100644 --- a/storage/xtradb/include/buf0buf.ic +++ b/storage/xtradb/include/buf0buf.ic @@ -690,6 +690,19 @@ buf_block_get_frame( { SRV_CORRUPT_TABLE_CHECK(block, return(0);); + return(buf_nonnull_block_get_frame(block)); +} + +/*********************************************************************//** +Gets a pointer to the memory frame of a block, where block is known not to be +NULL. +@return pointer to the frame */ +UNIV_INLINE +buf_frame_t* +buf_nonnull_block_get_frame( +/*========================*/ + const buf_block_t* block) /*!< in: pointer to the control block */ +{ switch (buf_block_get_state(block)) { case BUF_BLOCK_POOL_WATCH: case BUF_BLOCK_ZIP_PAGE: @@ -711,6 +724,7 @@ buf_block_get_frame( ok: return((buf_frame_t*) block->frame); } + #endif /* UNIV_DEBUG */ /*********************************************************************//** diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h index 68d4d333245d9..87b5787d416be 100644 --- a/storage/xtradb/include/fts0fts.h +++ b/storage/xtradb/include/fts0fts.h @@ -840,13 +840,15 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table @param[in] unlock_cache whether unlock cache when write node @param[in] wait whether wait for existing sync to finish +@param[in] has_dict whether has dict operation lock @return DB_SUCCESS on success, error code on failure. */ UNIV_INTERN dberr_t fts_sync_table( dict_table_t* table, bool unlock_cache, - bool wait); + bool wait, + bool has_dict); /****************************************************************//** Free the query graph but check whether dict_sys->mutex is already diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 95065d6997480..692d339608a2e 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -165,8 +165,10 @@ extern os_event_t srv_checkpoint_completed_event; log tracking iteration */ extern os_event_t srv_redo_log_tracked_event; -/** srv_redo_log_follow_thread spawn flag */ -extern bool srv_redo_log_thread_started; +/** Whether the redo log tracker thread has been started. Does not take into +account whether the tracking is currently enabled (see srv_track_changed_pages +for that) */ +extern bool srv_redo_log_thread_started; /* If the last data file is auto-extended, we add this many pages to it at a time */ @@ -262,6 +264,10 @@ extern char** srv_data_file_names; extern ulint* srv_data_file_sizes; extern ulint* srv_data_file_is_raw_partition; + +/** Whether the redo log tracking is currently enabled. Note that it is +possible for the log tracker thread to be running and the tracking to be +disabled */ extern my_bool srv_track_changed_pages; extern ulonglong srv_max_bitmap_file_size; diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index ffae4af4c5611..4d64e3249c0b4 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -47,7 +47,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_BUGFIX MYSQL_VERSION_PATCH #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 77.0 +#define PERCONA_INNODB_VERSION 78.1 #endif /* Enable UNIV_LOG_ARCHIVE in XtraDB */ diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index e6e5762b1e913..0768bb6bb003a 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -3572,7 +3572,7 @@ logs_empty_and_mark_files_at_shutdown(void) /* Wake the log tracking thread which will then immediatelly quit because of srv_shutdown_state value */ - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { os_event_reset(srv_redo_log_tracked_event); os_event_set(srv_checkpoint_completed_event); } @@ -3651,7 +3651,7 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; /* Signal the log following thread to quit */ - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { os_event_reset(srv_redo_log_tracked_event); os_event_set(srv_checkpoint_completed_event); } diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 4e58755e1f275..d80cb2ad4471d 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -1796,20 +1796,20 @@ log_online_purge_changed_page_bitmaps( lsn = LSN_MAX; } - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { /* User requests might happen with both enabled and disabled tracking */ mutex_enter(&log_bmp_sys->mutex); } if (!log_online_setup_bitmap_file_range(&bitmap_files, 0, LSN_MAX)) { - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { mutex_exit(&log_bmp_sys->mutex); } return TRUE; } - if (srv_track_changed_pages && lsn > log_bmp_sys->end_lsn) { + if (srv_redo_log_thread_started && lsn > log_bmp_sys->end_lsn) { /* If we have to delete the current output file, close it first. */ os_file_close(log_bmp_sys->out.file); @@ -1842,7 +1842,7 @@ log_online_purge_changed_page_bitmaps( } } - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { if (lsn > log_bmp_sys->end_lsn) { lsn_t new_file_lsn; if (lsn == LSN_MAX) { @@ -1853,9 +1853,7 @@ log_online_purge_changed_page_bitmaps( new_file_lsn = log_bmp_sys->end_lsn; } if (!log_online_rotate_bitmap_file(new_file_lsn)) { - /* If file create failed, signal the log - tracking thread to quit next time it wakes - up. */ + /* If file create failed, stop log tracking */ srv_track_changed_pages = FALSE; } } diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 23fadfb0bf2c9..b80f1f8597e79 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -379,12 +379,6 @@ recv_sys_init( } #ifndef UNIV_HOTBACKUP - /* Initialize red-black tree for fast insertions into the - flush_list during recovery process. - As this initialization is done while holding the buffer pool - mutex we perform it before acquiring recv_sys->mutex. */ - buf_flush_init_flush_rbt(); - mutex_enter(&(recv_sys->mutex)); recv_sys->heap = mem_heap_create_typed(256, @@ -474,9 +468,6 @@ recv_sys_debug_free(void) recv_sys->last_block_buf_start = NULL; mutex_exit(&(recv_sys->mutex)); - - /* Free up the flush_rbt. */ - buf_flush_free_flush_rbt(); } # endif /* UNIV_LOG_DEBUG */ @@ -3118,6 +3109,11 @@ recv_recovery_from_checkpoint_start_func( byte* log_hdr_buf_base = static_cast (alloca(LOG_FILE_HDR_SIZE + OS_FILE_LOG_BLOCK_SIZE)); dberr_t err; + + /* Initialize red-black tree for fast insertions into the + flush_list during recovery process. */ + buf_flush_init_flush_rbt(); + ut_when_dtor tmp(recv_sys->dblwr); log_hdr_buf = static_cast @@ -3537,6 +3533,9 @@ recv_recovery_from_checkpoint_finish(void) #ifndef UNIV_LOG_DEBUG recv_sys_debug_free(); #endif + /* Free up the flush_rbt. */ + buf_flush_free_flush_rbt(); + /* Roll back any recovered data dictionary transactions, so that the data dictionary tables will be free of any locks. The data dictionary latch should guarantee that there is at diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 0bba529d16713..feb18c82ab62a 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1993,7 +1993,7 @@ row_merge_read_clustered_index( /* Sync fts cache for other fts indexes to keep all fts indexes consistent in sync_doc_id. */ err = fts_sync_table(const_cast(new_table), - false, true); + false, true, false); if (err == DB_SUCCESS) { fts_update_next_doc_id( diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index 80c8f7fadbceb..1aab949564463 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2010, 2014, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. This program is free software; you can redistribute it and/or modify it under @@ -1347,7 +1347,10 @@ srv_mon_set_module_control( module */ set_current_module = FALSE; } else if (module_id == MONITOR_ALL_COUNTER) { - continue; + if (!(innodb_counter_info[ix].monitor_type + & MONITOR_GROUP_MODULE)) { + continue; + } } else { /* Hitting the next module, stop */ break; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index cba87df32b046..1012d0ccb08da 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -184,6 +184,9 @@ UNIV_INTERN char** srv_data_file_names = NULL; /* size in database pages */ UNIV_INTERN ulint* srv_data_file_sizes = NULL; +/** Whether the redo log tracking is currently enabled. Note that it is +possible for the log tracker thread to be running and the tracking to be +disabled */ UNIV_INTERN my_bool srv_track_changed_pages = FALSE; UNIV_INTERN ulonglong srv_max_bitmap_file_size = 100 * 1024 * 1024; @@ -749,6 +752,9 @@ UNIV_INTERN os_event_t srv_checkpoint_completed_event; UNIV_INTERN os_event_t srv_redo_log_tracked_event; +/** Whether the redo log tracker thread has been started. Does not take into +account whether the tracking is currently enabled (see srv_track_changed_pages +for that) */ UNIV_INTERN bool srv_redo_log_thread_started = false; /*********************************************************************//** @@ -2324,13 +2330,8 @@ DECLARE_THREAD(srv_redo_log_follow_thread)( os_event_wait(srv_checkpoint_completed_event); os_event_reset(srv_checkpoint_completed_event); -#ifdef UNIV_DEBUG - if (!srv_track_changed_pages) { - continue; - } -#endif - - if (srv_shutdown_state < SRV_SHUTDOWN_LAST_PHASE) { + if (srv_track_changed_pages + && srv_shutdown_state < SRV_SHUTDOWN_LAST_PHASE) { if (!log_online_follow_redo_log()) { /* TODO: sync with I_S log tracking status? */ ib_logf(IB_LOG_LEVEL_ERROR, From 0e76054b7b5f09246f31f8927194e9782f82634b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 28 Sep 2016 12:52:01 +0000 Subject: [PATCH 077/295] Feedback plugin : add support for Windows 10 / Server 2016. Also add fallback version string for unknown future versions. --- plugin/feedback/utils.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugin/feedback/utils.cc b/plugin/feedback/utils.cc index f6fcb3d208233..cff19c93ed231 100644 --- a/plugin/feedback/utils.cc +++ b/plugin/feedback/utils.cc @@ -43,7 +43,11 @@ static const char *get_os_version_name(OSVERSIONINFOEX *ver) { DWORD major = ver->dwMajorVersion; DWORD minor = ver->dwMinorVersion; - + if (major == 10 && minor == 0) + { + return (ver->wProductType == VER_NT_WORKSTATION) ? + "Windows 10" : "Windows Server 2016"; + } if (major == 6 && minor == 3) { return (ver->wProductType == VER_NT_WORKSTATION)? @@ -102,7 +106,12 @@ static int uname(struct utsname *buf) if(version_str && version_str[0]) sprintf(buf->version, "%s %s",version_str, ver.szCSDVersion); else - sprintf(buf->version, "%s", ver.szCSDVersion); + { + /* Fallback for unknown versions, e.g "Windows ." */ + sprintf(buf->version, "Windows %d.%d%s", + ver.dwMajorVersion, ver.dwMinorVersion, + (ver.wProductType == VER_NT_WORKSTATION ? "" : " Server")); + } #ifdef _WIN64 strcpy(buf->machine, "x64"); From a53f3c6d3cfa50b15b1aff26bc9479eb582d8611 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 28 Sep 2016 16:12:58 +0300 Subject: [PATCH 078/295] MDEV-10649: Optimizer sometimes use "index" instead of "range" access for UPDATE (Fixing both InnoDB and XtraDB) Re-opening a TABLE object (after e.g. FLUSH TABLES or open table cache eviction) causes ha_innobase to call dict_stats_update(DICT_STATS_FETCH_ONLY_IF_NOT_IN_MEMORY). Inside this call, the following is done: dict_stats_empty_table(table); dict_stats_copy(table, t); On the other hand, commands like UPDATE make this call to get the "rows in table" statistics in table->stats.records: ha_innobase->info(HA_STATUS_VARIABLE|HA_STATUS_NO_LOCK) note the HA_STATUS_NO_LOCK parameter. It means, no locks are taken by ::info() If the ::info() call happens between dict_stats_empty_table and dict_stats_copy calls, the UPDATE's optimizer will get an estimate of table->stats.records=1, which causes it to pick a full table scan, which in turn will take a lot of row locks and cause other bad consequences. --- storage/innobase/dict/dict0stats.cc | 29 +++++++++++++++++++---------- storage/xtradb/dict/dict0stats.cc | 29 +++++++++++++++++++---------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index b073398f8ecf7..a4aa43651f8a5 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -673,7 +673,10 @@ void dict_stats_copy( /*============*/ dict_table_t* dst, /*!< in/out: destination table */ - const dict_table_t* src) /*!< in: source table */ + const dict_table_t* src, /*!< in: source table */ + bool reset_ignored_indexes) /*!< in: if true, set ignored indexes + to have the same statistics as if + the table was empty */ { dst->stats_last_recalc = src->stats_last_recalc; dst->stat_n_rows = src->stat_n_rows; @@ -692,7 +695,16 @@ dict_stats_copy( && (src_idx = dict_table_get_next_index(src_idx)))) { if (dict_stats_should_ignore_index(dst_idx)) { - continue; + if (reset_ignored_indexes) { + /* Reset index statistics for all ignored indexes, + unless they are FT indexes (these have no statistics)*/ + if (dst_idx->type & DICT_FTS) { + continue; + } + dict_stats_empty_index(dst_idx); + } else { + continue; + } } ut_ad(!dict_index_is_univ(dst_idx)); @@ -782,7 +794,7 @@ dict_stats_snapshot_create( t = dict_stats_table_clone_create(table); - dict_stats_copy(t, table); + dict_stats_copy(t, table, false); t->stat_persistent = table->stat_persistent; t->stats_auto_recalc = table->stats_auto_recalc; @@ -3240,13 +3252,10 @@ dict_stats_update( dict_table_stats_lock(table, RW_X_LATCH); - /* Initialize all stats to dummy values before - copying because dict_stats_table_clone_create() does - skip corrupted indexes so our dummy object 't' may - have less indexes than the real object 'table'. */ - dict_stats_empty_table(table); - - dict_stats_copy(table, t); + /* Pass reset_ignored_indexes=true as parameter + to dict_stats_copy. This will cause statictics + for corrupted indexes to be set to empty values */ + dict_stats_copy(table, t, true); dict_stats_assert_initialized(table); diff --git a/storage/xtradb/dict/dict0stats.cc b/storage/xtradb/dict/dict0stats.cc index b073398f8ecf7..a4aa43651f8a5 100644 --- a/storage/xtradb/dict/dict0stats.cc +++ b/storage/xtradb/dict/dict0stats.cc @@ -673,7 +673,10 @@ void dict_stats_copy( /*============*/ dict_table_t* dst, /*!< in/out: destination table */ - const dict_table_t* src) /*!< in: source table */ + const dict_table_t* src, /*!< in: source table */ + bool reset_ignored_indexes) /*!< in: if true, set ignored indexes + to have the same statistics as if + the table was empty */ { dst->stats_last_recalc = src->stats_last_recalc; dst->stat_n_rows = src->stat_n_rows; @@ -692,7 +695,16 @@ dict_stats_copy( && (src_idx = dict_table_get_next_index(src_idx)))) { if (dict_stats_should_ignore_index(dst_idx)) { - continue; + if (reset_ignored_indexes) { + /* Reset index statistics for all ignored indexes, + unless they are FT indexes (these have no statistics)*/ + if (dst_idx->type & DICT_FTS) { + continue; + } + dict_stats_empty_index(dst_idx); + } else { + continue; + } } ut_ad(!dict_index_is_univ(dst_idx)); @@ -782,7 +794,7 @@ dict_stats_snapshot_create( t = dict_stats_table_clone_create(table); - dict_stats_copy(t, table); + dict_stats_copy(t, table, false); t->stat_persistent = table->stat_persistent; t->stats_auto_recalc = table->stats_auto_recalc; @@ -3240,13 +3252,10 @@ dict_stats_update( dict_table_stats_lock(table, RW_X_LATCH); - /* Initialize all stats to dummy values before - copying because dict_stats_table_clone_create() does - skip corrupted indexes so our dummy object 't' may - have less indexes than the real object 'table'. */ - dict_stats_empty_table(table); - - dict_stats_copy(table, t); + /* Pass reset_ignored_indexes=true as parameter + to dict_stats_copy. This will cause statictics + for corrupted indexes to be set to empty values */ + dict_stats_copy(table, t, true); dict_stats_assert_initialized(table); From 7cb79a65ba6286ac66d5ebbebea3243ef97f5c41 Mon Sep 17 00:00:00 2001 From: Arun Kuruvila Date: Wed, 28 Sep 2016 15:52:05 +0530 Subject: [PATCH 079/295] Bug#24707666: DEFAULT SETTING FOR SECURE-FILE-PRIV SHOULD BE RESTRICTED IN ALL GA RELEASES Back port of WL#6782 to 5.5 and 5.6. This also includes back port of Bug#20771331, Bug#20741572 and Bug#20770671. Bug#24695274 and Bug#24679907 are also handled along with this. --- cmake/install_layout.cmake | 256 +++++++++++++++++- config.h.cmake | 4 + mysql-test/include/mtr_warnings.sql | 7 +- mysql-test/include/mysqld--help.inc | 3 +- mysql-test/mysql-test-run.pl | 4 +- mysql-test/r/mysqld--help-notwin.result | 1 - mysql-test/r/mysqld--help-win.result | 1 - .../auth_sec/r/secure_file_priv_error.result | 7 + .../auth_sec/r/secure_file_priv_null.result | 21 ++ .../r/secure_file_priv_warnings.result | 17 ++ .../secure_file_priv_warnings_not_win.result | 9 + .../r/secure_file_priv_warnings_win.result | 8 + .../auth_sec/t/secure_file_priv_error.test | 39 +++ .../t/secure_file_priv_null-master.opt | 1 + .../auth_sec/t/secure_file_priv_null.test | 42 +++ .../t/secure_file_priv_warnings-master.opt | 1 + .../auth_sec/t/secure_file_priv_warnings.test | 47 ++++ .../t/secure_file_priv_warnings_not_win.test | 24 ++ .../t/secure_file_priv_warnings_win.test | 35 +++ packaging/rpm-oel/mysql-systemd-start | 6 + packaging/rpm-oel/mysql.init | 10 +- packaging/rpm-oel/mysql.spec.in | 5 + packaging/rpm-sles/mysql.spec.in | 5 + packaging/solaris/postinstall-solaris.sh | 8 +- sql/mysqld.cc | 244 +++++++++++++++-- sql/sql_class.cc | 2 + sql/sql_class.h | 1 + sql/sys_vars.cc | 8 +- support-files/mysql.spec.sh | 7 +- 29 files changed, 786 insertions(+), 37 deletions(-) create mode 100644 mysql-test/suite/auth_sec/r/secure_file_priv_error.result create mode 100644 mysql-test/suite/auth_sec/r/secure_file_priv_null.result create mode 100644 mysql-test/suite/auth_sec/r/secure_file_priv_warnings.result create mode 100644 mysql-test/suite/auth_sec/r/secure_file_priv_warnings_not_win.result create mode 100644 mysql-test/suite/auth_sec/r/secure_file_priv_warnings_win.result create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_error.test create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_null-master.opt create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_null.test create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_warnings-master.opt create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_warnings.test create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_warnings_not_win.test create mode 100644 mysql-test/suite/auth_sec/t/secure_file_priv_warnings_win.test diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index 4adda0b6eacb9..4fd18b049f238 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,7 +22,7 @@ # and relative links. Windows zip uses the same tarball layout but without # the build prefix. # -# RPM +# RPM, SLES # Build as per default RPM layout, with prefix=/usr # Note: The layout for ULN RPMs differs, see the "RPM" section. # @@ -32,10 +32,22 @@ # SVR4 # Solaris package layout suitable for pkg* tools, prefix=/opt/mysql/mysql # +# FREEBSD, GLIBC, OSX, TARGZ +# Build with prefix=/usr/local/mysql, create tarball with install prefix="." +# and relative links. +# +# WIN +# Windows zip : same as tarball layout but without the build prefix +# # To force a directory layout, use -DINSTALL_LAYOUT=. # # The default is STANDALONE. # +# Note : At present, RPM and SLES layouts are similar. This is also true +# for layouts like FREEBSD, GLIBC, OSX, TARGZ. However, they provide +# opportunity to fine-tune deployment for each platform without +# affecting all other types of deployment. +# # There is the possibility to further fine-tune installation directories. # Several variables can be overwritten: # @@ -60,6 +72,7 @@ # - INSTALL_SUPPORTFILESDIR (various extra support files) # # - INSTALL_MYSQLDATADIR (data directory) +# - INSTALL_SECURE_FILE_PRIVDIR (--secure-file-priv directory) # # When changing this page, _please_ do not forget to update public Wiki # http://forge.mysql.com/wiki/CMake#Fine-tuning_installation_paths @@ -69,10 +82,11 @@ IF(NOT INSTALL_LAYOUT) ENDIF() SET(INSTALL_LAYOUT "${DEFAULT_INSTALL_LAYOUT}" -CACHE STRING "Installation directory layout. Options are: STANDALONE (as in zip or tar.gz installer), RPM, DEB, SVR4") +CACHE STRING "Installation directory layout. Options are: TARGZ (as in tar.gz installer), WIN (as in zip installer), STANDALONE, RPM, DEB, SVR4, FREEBSD, GLIBC, OSX, SLES") IF(UNIX) - IF(INSTALL_LAYOUT MATCHES "RPM") + IF(INSTALL_LAYOUT MATCHES "RPM" OR + INSTALL_LAYOUT MATCHES "SLES") SET(default_prefix "/usr") ELSEIF(INSTALL_LAYOUT MATCHES "DEB") SET(default_prefix "/opt/mysql/server-${MYSQL_BASE_VERSION}") @@ -87,7 +101,7 @@ IF(UNIX) SET(CMAKE_INSTALL_PREFIX ${default_prefix} CACHE PATH "install prefix" FORCE) ENDIF() - SET(VALID_INSTALL_LAYOUTS "RPM" "STANDALONE" "DEB" "SVR4") + SET(VALID_INSTALL_LAYOUTS "RPM" "DEB" "SVR4" "FREEBSD" "GLIBC" "OSX" "TARGZ" "SLES" "STANDALONE") LIST(FIND VALID_INSTALL_LAYOUTS "${INSTALL_LAYOUT}" ind) IF(ind EQUAL -1) MESSAGE(FATAL_ERROR "Invalid INSTALL_LAYOUT parameter:${INSTALL_LAYOUT}." @@ -99,6 +113,15 @@ IF(UNIX) MARK_AS_ADVANCED(SYSCONFDIR) ENDIF() +IF(WIN32) + SET(VALID_INSTALL_LAYOUTS "TARGZ" "STANDALONE" "WIN") + LIST(FIND VALID_INSTALL_LAYOUTS "${INSTALL_LAYOUT}" ind) + IF(ind EQUAL -1) + MESSAGE(FATAL_ERROR "Invalid INSTALL_LAYOUT parameter:${INSTALL_LAYOUT}." + " Choose between ${VALID_INSTALL_LAYOUTS}" ) + ENDIF() +ENDIF() + # # plugin_tests's value should not be used by imported plugins, # just use if(INSTALL_PLUGINTESTDIR). @@ -109,6 +132,22 @@ FILE(GLOB plugin_tests ${CMAKE_SOURCE_DIR}/internal/plugin/*/tests ) +# +# DEFAULT_SECURE_FILE_PRIV_DIR/DEFAULT_SECURE_FILE_PRIV_EMBEDDED_DIR +# +IF(INSTALL_LAYOUT MATCHES "STANDALONE" OR + INSTALL_LAYOUT MATCHES "WIN") + SET(secure_file_priv_path "NULL") +ELSEIF(INSTALL_LAYOUT MATCHES "RPM" OR + INSTALL_LAYOUT MATCHES "SLES" OR + INSTALL_LAYOUT MATCHES "SVR4" OR + INSTALL_LAYOUT MATCHES "DEB") + SET(secure_file_priv_path "/var/lib/mysql-files") +ELSE() + SET(secure_file_priv_path "${default_prefix}/mysql-files") +ENDIF() +SET(secure_file_priv_embedded_path "NULL") + # # STANDALONE layout # @@ -134,6 +173,148 @@ SET(INSTALL_SUPPORTFILESDIR_STANDALONE "support-files") # SET(INSTALL_MYSQLDATADIR_STANDALONE "data") SET(INSTALL_PLUGINTESTDIR_STANDALONE ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_STANDALONE ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_STANDALONE ${secure_file_priv_embedded_path}) + +# +# WIN layout +# +SET(INSTALL_BINDIR_WIN "bin") +SET(INSTALL_SBINDIR_WIN "bin") +SET(INSTALL_SCRIPTDIR_WIN "scripts") +# +SET(INSTALL_LIBDIR_WIN "lib") +SET(INSTALL_PLUGINDIR_WIN "lib/plugin") +# +SET(INSTALL_INCLUDEDIR_WIN "include") +# +SET(INSTALL_DOCDIR_WIN "docs") +SET(INSTALL_DOCREADMEDIR_WIN ".") +SET(INSTALL_MANDIR_WIN "man") +SET(INSTALL_INFODIR_WIN "docs") +# +SET(INSTALL_SHAREDIR_WIN "share") +SET(INSTALL_MYSQLSHAREDIR_WIN "share") +SET(INSTALL_MYSQLTESTDIR_WIN "mysql-test") +SET(INSTALL_SQLBENCHDIR_WIN ".") +SET(INSTALL_SUPPORTFILESDIR_WIN "support-files") +# +SET(INSTALL_MYSQLDATADIR_WIN "data") +SET(INSTALL_PLUGINTESTDIR_WIN ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_WIN ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_WIN ${secure_file_priv_embedded_path}) + +# +# FREEBSD layout +# +SET(INSTALL_BINDIR_FREEBSD "bin") +SET(INSTALL_SBINDIR_FREEBSD "bin") +SET(INSTALL_SCRIPTDIR_FREEBSD "scripts") +# +SET(INSTALL_LIBDIR_FREEBSD "lib") +SET(INSTALL_PLUGINDIR_FREEBSD "lib/plugin") +# +SET(INSTALL_INCLUDEDIR_FREEBSD "include") +# +SET(INSTALL_DOCDIR_FREEBSD "docs") +SET(INSTALL_DOCREADMEDIR_FREEBSD ".") +SET(INSTALL_MANDIR_FREEBSD "man") +SET(INSTALL_INFODIR_FREEBSD "docs") +# +SET(INSTALL_SHAREDIR_FREEBSD "share") +SET(INSTALL_MYSQLSHAREDIR_FREEBSD "share") +SET(INSTALL_MYSQLTESTDIR_FREEBSD "mysql-test") +SET(INSTALL_SQLBENCHDIR_FREEBSD ".") +SET(INSTALL_SUPPORTFILESDIR_FREEBSD "support-files") +# +SET(INSTALL_MYSQLDATADIR_FREEBSD "data") +SET(INSTALL_PLUGINTESTDIR_FREEBSD ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_FREEBSD ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_FREEBSD ${secure_file_priv_embedded_path}) + +# +# GLIBC layout +# +SET(INSTALL_BINDIR_GLIBC "bin") +SET(INSTALL_SBINDIR_GLIBC "bin") +SET(INSTALL_SCRIPTDIR_GLIBC "scripts") +# +SET(INSTALL_LIBDIR_GLIBC "lib") +SET(INSTALL_PLUGINDIR_GLIBC "lib/plugin") +# +SET(INSTALL_INCLUDEDIR_GLIBC "include") +# +SET(INSTALL_DOCDIR_GLIBC "docs") +SET(INSTALL_DOCREADMEDIR_GLIBC ".") +SET(INSTALL_MANDIR_GLIBC "man") +SET(INSTALL_INFODIR_GLIBC "docs") +# +SET(INSTALL_SHAREDIR_GLIBC "share") +SET(INSTALL_MYSQLSHAREDIR_GLIBC "share") +SET(INSTALL_MYSQLTESTDIR_GLIBC "mysql-test") +SET(INSTALL_SQLBENCHDIR_GLIBC ".") +SET(INSTALL_SUPPORTFILESDIR_GLIBC "support-files") +# +SET(INSTALL_MYSQLDATADIR_GLIBC "data") +SET(INSTALL_PLUGINTESTDIR_GLIBC ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_GLIBC ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_GLIBC ${secure_file_priv_embedded_path}) + +# +# OSX layout +# +SET(INSTALL_BINDIR_OSX "bin") +SET(INSTALL_SBINDIR_OSX "bin") +SET(INSTALL_SCRIPTDIR_OSX "scripts") +# +SET(INSTALL_LIBDIR_OSX "lib") +SET(INSTALL_PLUGINDIR_OSX "lib/plugin") +# +SET(INSTALL_INCLUDEDIR_OSX "include") +# +SET(INSTALL_DOCDIR_OSX "docs") +SET(INSTALL_DOCREADMEDIR_OSX ".") +SET(INSTALL_MANDIR_OSX "man") +SET(INSTALL_INFODIR_OSX "docs") +# +SET(INSTALL_SHAREDIR_OSX "share") +SET(INSTALL_MYSQLSHAREDIR_OSX "share") +SET(INSTALL_MYSQLTESTDIR_OSX "mysql-test") +SET(INSTALL_SQLBENCHDIR_OSX ".") +SET(INSTALL_SUPPORTFILESDIR_OSX "support-files") +# +SET(INSTALL_MYSQLDATADIR_OSX "data") +SET(INSTALL_PLUGINTESTDIR_OSX ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_OSX ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_OSX ${secure_file_priv_embedded_path}) + +# +# TARGZ layout +# +SET(INSTALL_BINDIR_TARGZ "bin") +SET(INSTALL_SBINDIR_TARGZ "bin") +SET(INSTALL_SCRIPTDIR_TARGZ "scripts") +# +SET(INSTALL_LIBDIR_TARGZ "lib") +SET(INSTALL_PLUGINDIR_TARGZ "lib/plugin") +# +SET(INSTALL_INCLUDEDIR_TARGZ "include") +# +SET(INSTALL_DOCDIR_TARGZ "docs") +SET(INSTALL_DOCREADMEDIR_TARGZ ".") +SET(INSTALL_MANDIR_TARGZ "man") +SET(INSTALL_INFODIR_TARGZ "docs") +# +SET(INSTALL_SHAREDIR_TARGZ "share") +SET(INSTALL_MYSQLSHAREDIR_TARGZ "share") +SET(INSTALL_MYSQLTESTDIR_TARGZ "mysql-test") +SET(INSTALL_SQLBENCHDIR_TARGZ ".") +SET(INSTALL_SUPPORTFILESDIR_TARGZ "support-files") +# +SET(INSTALL_MYSQLDATADIR_TARGZ "data") +SET(INSTALL_PLUGINTESTDIR_TARGZ ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_TARGZ ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_TARGZ ${secure_file_priv_embedded_path}) # # RPM layout @@ -169,6 +350,41 @@ SET(INSTALL_SUPPORTFILESDIR_RPM "share/mysql") # SET(INSTALL_MYSQLDATADIR_RPM "/var/lib/mysql") SET(INSTALL_PLUGINTESTDIR_RPM ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_RPM ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_RPM ${secure_file_priv_embedded_path}) + +# +# SLES layout +# +SET(INSTALL_BINDIR_SLES "bin") +SET(INSTALL_SBINDIR_SLES "sbin") +SET(INSTALL_SCRIPTDIR_SLES "bin") +# +IF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") + SET(INSTALL_LIBDIR_SLES "lib64") + SET(INSTALL_PLUGINDIR_SLES "lib64/mysql/plugin") +ELSE() + SET(INSTALL_LIBDIR_SLES "lib") + SET(INSTALL_PLUGINDIR_SLES "lib/mysql/plugin") +ENDIF() +# +SET(INSTALL_INCLUDEDIR_SLES "include/mysql") +# +#SET(INSTALL_DOCDIR_SLES unset - installed directly by SLES) +#SET(INSTALL_DOCREADMEDIR_SLES unset - installed directly by SLES) +SET(INSTALL_INFODIR_SLES "share/info") +SET(INSTALL_MANDIR_SLES "share/man") +# +SET(INSTALL_SHAREDIR_SLES "share") +SET(INSTALL_MYSQLSHAREDIR_SLES "share/mysql") +SET(INSTALL_MYSQLTESTDIR_SLES "share/mysql-test") +SET(INSTALL_SQLBENCHDIR_SLES "") +SET(INSTALL_SUPPORTFILESDIR_SLES "share/mysql") +# +SET(INSTALL_MYSQLDATADIR_SLES "/var/lib/mysql") +SET(INSTALL_PLUGINTESTDIR_SLES ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_SLES ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_SLES ${secure_file_priv_embedded_path}) # # DEB layout @@ -193,8 +409,10 @@ SET(INSTALL_MYSQLTESTDIR_DEB "mysql-test") SET(INSTALL_SQLBENCHDIR_DEB ".") SET(INSTALL_SUPPORTFILESDIR_DEB "support-files") # -SET(INSTALL_MYSQLDATADIR_DEB "data") +SET(INSTALL_MYSQLDATADIR_DEB "/var/lib/mysql") SET(INSTALL_PLUGINTESTDIR_DEB ${plugin_tests}) +SET(INSTALL_SECURE_FILE_PRIVDIR_DEB ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_DEB ${secure_file_priv_embedded_path}) # # SVR4 layout @@ -221,7 +439,8 @@ SET(INSTALL_SUPPORTFILESDIR_SVR4 "support-files") # SET(INSTALL_MYSQLDATADIR_SVR4 "/var/lib/mysql") SET(INSTALL_PLUGINTESTDIR_SVR4 ${plugin_tests}) - +SET(INSTALL_SECURE_FILE_PRIVDIR_SVR4 ${secure_file_priv_path}) +SET(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR_SVR4 ${secure_file_priv_embedded_path}) # Clear cached variables if install layout was changed IF(OLD_INSTALL_LAYOUT) @@ -235,8 +454,29 @@ SET(OLD_INSTALL_LAYOUT ${INSTALL_LAYOUT} CACHE INTERNAL "") # will be defined as ${INSTALL_BINDIR_STANDALONE} by default if STANDALONE # layout is chosen) FOREACH(var BIN SBIN LIB MYSQLSHARE SHARE PLUGIN INCLUDE SCRIPT DOC MAN - INFO MYSQLTEST SQLBENCH DOCREADME SUPPORTFILES MYSQLDATA PLUGINTEST) + INFO MYSQLTEST SQLBENCH DOCREADME SUPPORTFILES MYSQLDATA PLUGINTEST + SECURE_FILE_PRIV SECURE_FILE_PRIV_EMBEDDED) SET(INSTALL_${var}DIR ${INSTALL_${var}DIR_${INSTALL_LAYOUT}} CACHE STRING "${var} installation directory" ${FORCE}) MARK_AS_ADVANCED(INSTALL_${var}DIR) ENDFOREACH() + +# +# Set DEFAULT_SECURE_FILE_PRIV_DIR +# This is used as default value for --secure-file-priv +# +IF(INSTALL_SECURE_FILE_PRIVDIR) + SET(DEFAULT_SECURE_FILE_PRIV_DIR "\"${INSTALL_SECURE_FILE_PRIVDIR}\"" + CACHE INTERNAL "default --secure-file-priv directory" FORCE) +ELSE() + SET(DEFAULT_SECURE_FILE_PRIV_DIR \"\" + CACHE INTERNAL "default --secure-file-priv directory" FORCE) +ENDIF() + +IF(INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR) + SET(DEFAULT_SECURE_FILE_PRIV_EMBEDDED_DIR "\"${INSTALL_SECURE_FILE_PRIV_EMBEDDEDDIR}\"" + CACHE INTERNAL "default --secure-file-priv directory (for embedded library)" FORCE) +ELSE() + SET(DEFAULT_SECURE_FILE_PRIV_EMBEDDED_DIR "NULL" + CACHE INTERNAL "default --secure-file-priv directory (for embedded library)" FORCE) +ENDIF() diff --git a/config.h.cmake b/config.h.cmake index 4548d0a221f24..c7ed127379a32 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -624,4 +624,8 @@ #cmakedefine SIZEOF_TIME_T @SIZEOF_TIME_T@ #cmakedefine TIME_T_UNSIGNED @TIME_T_UNSIGNED@ +/* For --secure-file-priv */ +#cmakedefine DEFAULT_SECURE_FILE_PRIV_DIR @DEFAULT_SECURE_FILE_PRIV_DIR@ +#cmakedefine DEFAULT_SECURE_FILE_PRIV_EMBEDDED_DIR @DEFAULT_SECURE_FILE_PRIV_EMBEDDED_DIR@ + #endif diff --git a/mysql-test/include/mtr_warnings.sql b/mysql-test/include/mtr_warnings.sql index 45acbc03b7e84..0a3c3bc60b3e6 100644 --- a/mysql-test/include/mtr_warnings.sql +++ b/mysql-test/include/mtr_warnings.sql @@ -1,4 +1,4 @@ --- Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. +-- Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. -- -- This program is free software; you can redistribute it and/or modify -- it under the terms of the GNU General Public License as published by @@ -204,6 +204,11 @@ INSERT INTO global_suppressions VALUES */ ("Found lock of type 6 that is write and read locked"), + /* + Warnings related to --secure-file-priv + */ + ("Insecure configuration for --secure-file-priv:*"), + ("THE_LAST_SUPPRESSION")|| diff --git a/mysql-test/include/mysqld--help.inc b/mysql-test/include/mysqld--help.inc index 380a7f6c8cfa8..7fa57abbe1ed5 100644 --- a/mysql-test/include/mysqld--help.inc +++ b/mysql-test/include/mysqld--help.inc @@ -18,7 +18,8 @@ perl; # their paths may vary: @skipvars=qw/basedir open-files-limit general-log-file log plugin-dir log-slow-queries pid-file slow-query-log-file - datadir slave-load-tmpdir tmpdir socket/; + datadir slave-load-tmpdir tmpdir socket + secure-file-priv/; # Plugins which may or may not be there: @plugins=qw/innodb ndb archive blackhole federated partition ndbcluster debug temp-pool ssl des-key-file diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 684d262f41094..3eb70c1bdb9b4 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl # -*- cperl -*- -# Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2004, 2016, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1823,6 +1823,7 @@ sub collect_mysqld_features { mtr_init_args(\$args); mtr_add_arg($args, "--no-defaults"); mtr_add_arg($args, "--datadir=%s", mixed_path($tmpdir)); + mtr_add_arg($args, "--secure-file-priv=\"\""); mtr_add_arg($args, "--lc-messages-dir=%s", $path_language); mtr_add_arg($args, "--skip-grant-tables"); mtr_add_arg($args, "--verbose"); @@ -3297,6 +3298,7 @@ sub mysql_install_db { mtr_add_arg($args, "--loose-skip-falcon"); mtr_add_arg($args, "--loose-skip-ndbcluster"); mtr_add_arg($args, "--tmpdir=%s", "$opt_vardir/tmp/"); + mtr_add_arg($args, "--secure-file-priv=%s", "$opt_vardir"); mtr_add_arg($args, "--core-file"); if ( $opt_debug ) diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index d527d6cb70279..78dc9ab4d880a 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -923,7 +923,6 @@ report-user (No default value) rpl-recovery-rank 0 safe-user-create FALSE secure-auth FALSE -secure-file-priv (No default value) server-id 0 show-slave-auth-info FALSE skip-grant-tables TRUE diff --git a/mysql-test/r/mysqld--help-win.result b/mysql-test/r/mysqld--help-win.result index 2ce9e763b14ad..1d56da7aa5e6e 100644 --- a/mysql-test/r/mysqld--help-win.result +++ b/mysql-test/r/mysqld--help-win.result @@ -931,7 +931,6 @@ report-user (No default value) rpl-recovery-rank 0 safe-user-create FALSE secure-auth FALSE -secure-file-priv (No default value) server-id 0 shared-memory FALSE shared-memory-base-name MYSQL diff --git a/mysql-test/suite/auth_sec/r/secure_file_priv_error.result b/mysql-test/suite/auth_sec/r/secure_file_priv_error.result new file mode 100644 index 0000000000000..4bb4d87c5f068 --- /dev/null +++ b/mysql-test/suite/auth_sec/r/secure_file_priv_error.result @@ -0,0 +1,7 @@ +#----------------------------------------------------------------------- +# Setup +# Try to restart server with invalid value for --secure-file-priv +# Search for : Failed to access directory for --secure-file-priv. +# Restart completed. +# Restart +#----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/r/secure_file_priv_null.result b/mysql-test/suite/auth_sec/r/secure_file_priv_null.result new file mode 100644 index 0000000000000..e2a5102c627e3 --- /dev/null +++ b/mysql-test/suite/auth_sec/r/secure_file_priv_null.result @@ -0,0 +1,21 @@ +#----------------------------------------------------------------------- +# Setup +#----------------------------------------------------------------------- +# Search for : --secure-file-priv is set to NULL. Operations +# related to importing and exporting data are +# disabled +show variables like 'secure_file_priv'; +Variable_name Value +secure_file_priv null +use test; +drop table if exists secure_file_priv_test_null; +create table secure_file_priv_test_null(c1 int); +insert into secure_file_priv_test_null values (1), (2), (3), (4); +select * from secure_file_priv_test_null into outfile 'blah'; +ERROR HY000: The MySQL server is running with the --secure-file-priv option so it cannot execute this statement +select * from secure_file_priv_test_null into outfile 'null/blah'; +ERROR HY000: The MySQL server is running with the --secure-file-priv option so it cannot execute this statement +drop table secure_file_priv_test_null; +#----------------------------------------------------------------------- +# Clean-up +#----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/r/secure_file_priv_warnings.result b/mysql-test/suite/auth_sec/r/secure_file_priv_warnings.result new file mode 100644 index 0000000000000..3b80cbe8d6fcf --- /dev/null +++ b/mysql-test/suite/auth_sec/r/secure_file_priv_warnings.result @@ -0,0 +1,17 @@ +#----------------------------------------------------------------------- +# Setup +#----------------------------------------------------------------------- +# Search for : Insecure configuration for --secure-file-priv: Current +# value does not restrict location of generated files. +# Consider setting it to a valid, non-empty path. +SHOW VARIABLES LIKE 'secure_file_priv'; +Variable_name Value +secure_file_priv +#----------------------------------------------------------------------- +# Restart completed. +# Search for : Insecure configuration for --secure-file-priv: Plugin +# directory is accessible through --secure-file-priv. +# Consider choosing a different directory. +#----------------------------------------------------------------------- +# Clean-up +#----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/r/secure_file_priv_warnings_not_win.result b/mysql-test/suite/auth_sec/r/secure_file_priv_warnings_not_win.result new file mode 100644 index 0000000000000..84e2f8ac3c214 --- /dev/null +++ b/mysql-test/suite/auth_sec/r/secure_file_priv_warnings_not_win.result @@ -0,0 +1,9 @@ +#----------------------------------------------------------------------- +# Search for : Insecure configuration for --secure-file-priv: Data +# directory is accessible through --secure-file-priv. +# Consider choosing a different directory. +#----------------------------------------------------------------------- +# Search for : Insecure configuration for --secure-file-priv: Location +# is accessible to all OS users. Consider choosing a +# different directory. +#----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/r/secure_file_priv_warnings_win.result b/mysql-test/suite/auth_sec/r/secure_file_priv_warnings_win.result new file mode 100644 index 0000000000000..3beff6c4747fe --- /dev/null +++ b/mysql-test/suite/auth_sec/r/secure_file_priv_warnings_win.result @@ -0,0 +1,8 @@ +#----------------------------------------------------------------------- +# Test 2 : Restarting mysqld with : +# --secure-file-priv=MYSQLTEST_VARDIR/mysqld.1/Data +# Restart completed. +# Search for : Insecure configuration for --secure-file-priv: Data +# directory is accessible through --secure-file-priv. +# Consider choosing a different directory. +#----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_error.test b/mysql-test/suite/auth_sec/t/secure_file_priv_error.test new file mode 100644 index 0000000000000..9f8d185d8f51c --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_error.test @@ -0,0 +1,39 @@ +--source include/no_valgrind_without_big.inc +--source include/not_embedded.inc + +--echo #----------------------------------------------------------------------- +--echo # Setup +let restart_log= $MYSQLTEST_VARDIR/log/my_restart.err; +let SEARCH_FILE= $restart_log; +let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; + +--echo # Try to restart server with invalid value for --secure-file-priv +--exec echo "wait" > $restart_file +--shutdown_server +--source include/wait_until_disconnected.inc + +--error 0,1 +--remove_file $restart_log +# Following should fail +--error 1 +--exec $MYSQLD_CMD --secure-file-priv=blahblahblah --loose-console > $restart_log 2>&1 + +--echo # Search for : Failed to access directory for --secure-file-priv. +let SEARCH_PATTERN= Failed to access directory for --secure-file-priv; +--source include/search_pattern_in_file.inc + +--remove_file $restart_log + +--source include/wait_until_disconnected.inc +# Dummy argument for restart +--exec echo "restart:" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect +--echo # Restart completed. + +--echo # Restart +--disable_warnings +--source include/force_restart.inc +--enable_warnings +--echo #----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_null-master.opt b/mysql-test/suite/auth_sec/t/secure_file_priv_null-master.opt new file mode 100644 index 0000000000000..80d7f3cd46905 --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_null-master.opt @@ -0,0 +1 @@ +--secure-file-priv=null diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_null.test b/mysql-test/suite/auth_sec/t/secure_file_priv_null.test new file mode 100644 index 0000000000000..8d394a135895a --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_null.test @@ -0,0 +1,42 @@ +--source include/no_valgrind_without_big.inc +--source include/not_embedded.inc + +--echo #----------------------------------------------------------------------- +--echo # Setup +let server_log= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_FILE= $server_log; +let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; +--echo #----------------------------------------------------------------------- + +--echo # Search for : --secure-file-priv is set to NULL. Operations +--echo # related to importing and exporting data are +--echo # disabled +let SEARCH_PATTERN= --secure-file-priv is set to NULL. Operations related to importing and exporting data are disabled; +--source include/search_pattern_in_file.inc + +connect(test4_con,localhost,root,,,,,); +show variables like 'secure_file_priv'; + +use test; +--disable_warnings +drop table if exists secure_file_priv_test_null; +--enable_warnings +create table secure_file_priv_test_null(c1 int); +insert into secure_file_priv_test_null values (1), (2), (3), (4); +--error 1290 +select * from secure_file_priv_test_null into outfile 'blah'; +--error 1290 +select * from secure_file_priv_test_null into outfile 'null/blah'; +drop table secure_file_priv_test_null; + +connection default; +disconnect test4_con; + +--echo #----------------------------------------------------------------------- + +--echo # Clean-up +--disable_warnings +--source include/force_restart.inc +--enable_warnings + +--echo #----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_warnings-master.opt b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings-master.opt new file mode 100644 index 0000000000000..22520f0aa9901 --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings-master.opt @@ -0,0 +1 @@ +--secure-file-priv="" diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_warnings.test b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings.test new file mode 100644 index 0000000000000..cc7a79d5b3c18 --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings.test @@ -0,0 +1,47 @@ +--source include/no_valgrind_without_big.inc +--source include/not_embedded.inc + +--echo #----------------------------------------------------------------------- +--echo # Setup +let server_log= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_FILE= $server_log; +let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; +let PLUGIN_DIR= $MYSQLTEST_VARDIR/tmp; +--echo #----------------------------------------------------------------------- + +--echo # Search for : Insecure configuration for --secure-file-priv: Current +--echo # value does not restrict location of generated files. +--echo # Consider setting it to a valid, non-empty path. +let SEARCH_PATTERN= Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path.; +--source include/search_pattern_in_file.inc + +# Must show empty string +SHOW VARIABLES LIKE 'secure_file_priv'; + +--echo #----------------------------------------------------------------------- + +let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; +--exec echo "wait" > $restart_file +--shutdown_server +--source include/wait_until_disconnected.inc +--remove_file $server_log +--exec echo "restart:--plugin-dir=$PLUGIN_DIR --secure-file-priv=$PLUGIN_DIR" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect +--echo # Restart completed. + +--echo # Search for : Insecure configuration for --secure-file-priv: Plugin +--echo # directory is accessible through --secure-file-priv. +--echo # Consider choosing a different directory. +let SEARCH_PATTERN= Insecure configuration for --secure-file-priv: Plugin directory is accessible through --secure-file-priv. Consider choosing a different directory.; +--source include/search_pattern_in_file.inc + +--echo #----------------------------------------------------------------------- + +--echo # Clean-up +--disable_warnings +--source include/force_restart.inc +--enable_warnings + +--echo #----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_warnings_not_win.test b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings_not_win.test new file mode 100644 index 0000000000000..ec027d4a743f5 --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings_not_win.test @@ -0,0 +1,24 @@ +--source include/no_valgrind_without_big.inc +--source include/not_windows.inc +--source include/not_embedded.inc + +let server_log= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_FILE= $server_log; + +--echo #----------------------------------------------------------------------- + +--echo # Search for : Insecure configuration for --secure-file-priv: Data +--echo # directory is accessible through --secure-file-priv. +--echo # Consider choosing a different directory. +let SEARCH_PATTERN= Insecure configuration for --secure-file-priv: Data directory is accessible through --secure-file-priv. Consider choosing a different directory.; +--source include/search_pattern_in_file.inc + +--echo #----------------------------------------------------------------------- + +--echo # Search for : Insecure configuration for --secure-file-priv: Location +--echo # is accessible to all OS users. Consider choosing a +--echo # different directory. +let SEARCH_PATTERN= Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.; +--source include/search_pattern_in_file.inc + +--echo #----------------------------------------------------------------------- diff --git a/mysql-test/suite/auth_sec/t/secure_file_priv_warnings_win.test b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings_win.test new file mode 100644 index 0000000000000..bb175fb40ea3f --- /dev/null +++ b/mysql-test/suite/auth_sec/t/secure_file_priv_warnings_win.test @@ -0,0 +1,35 @@ +--source include/no_valgrind_without_big.inc +--source include/windows.inc +--source include/not_embedded.inc + +let server_log= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_FILE= $server_log; + +--echo #----------------------------------------------------------------------- + +--echo # Test 2 : Restarting mysqld with : +--echo # --secure-file-priv=MYSQLTEST_VARDIR/mysqld.1/Data + +let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; +--exec echo "wait" > $restart_file +--shutdown_server +--source include/wait_until_disconnected.inc +--error 0,1 +--remove_file $server_log +--exec echo "restart: --secure-file-priv=$MYSQLTEST_VARDIR/mysqld.1/Data" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect +--echo # Restart completed. + +--echo # Search for : Insecure configuration for --secure-file-priv: Data +--echo # directory is accessible through --secure-file-priv. +--echo # Consider choosing a different directory. +let SEARCH_PATTERN= Insecure configuration for --secure-file-priv: Data directory is accessible through --secure-file-priv. Consider choosing a different directory.; +--source include/search_pattern_in_file.inc + +--disable_warnings +--source include/force_restart.inc +--enable_warnings + +--echo #----------------------------------------------------------------------- diff --git a/packaging/rpm-oel/mysql-systemd-start b/packaging/rpm-oel/mysql-systemd-start index fab7b3627b31a..231a76087ac3e 100644 --- a/packaging/rpm-oel/mysql-systemd-start +++ b/packaging/rpm-oel/mysql-systemd-start @@ -30,6 +30,12 @@ install_db () { if [ -x /usr/sbin/restorecon ]; then /usr/sbin/restorecon "$datadir" /usr/sbin/restorecon $log + for dir in /var/lib/mysql-files ; do + if [ -x /usr/sbin/semanage -a -d /var/lib/mysql -a -d $dir ] ; then + /usr/sbin/semanage fcontext -a -e /var/lib/mysql $dir >/dev/null 2>&1 + /sbin/restorecon $dir + fi + done fi # If special mysql dir is in place, skip db install diff --git a/packaging/rpm-oel/mysql.init b/packaging/rpm-oel/mysql.init index aaea498d15339..75ae672801b77 100644 --- a/packaging/rpm-oel/mysql.init +++ b/packaging/rpm-oel/mysql.init @@ -82,7 +82,15 @@ start(){ fi chown mysql:mysql "$datadir" chmod 0755 "$datadir" - [ -x /sbin/restorecon ] && /sbin/restorecon "$datadir" + if [ -x /sbin/restorecon ]; then + /sbin/restorecon "$datadir" + for dir in /var/lib/mysql-files ; do + if [ -x /usr/sbin/semanage -a -d /var/lib/mysql -a -d $dir ] ; then + /usr/sbin/semanage fcontext -a -e /var/lib/mysql $dir >/dev/null 2>&1 + /sbin/restorecon $dir + fi + done + fi # Now create the database action $"Initializing MySQL database: " /usr/bin/mysql_install_db --rpm --datadir="$datadir" --user=mysql ret=$? diff --git a/packaging/rpm-oel/mysql.spec.in b/packaging/rpm-oel/mysql.spec.in index 409c325b6759f..7ef294ffa8492 100644 --- a/packaging/rpm-oel/mysql.spec.in +++ b/packaging/rpm-oel/mysql.spec.in @@ -560,6 +560,7 @@ MBD=$RPM_BUILD_DIR/%{src_dir} install -d -m 0755 %{buildroot}%{_datadir}/mysql/SELinux/RHEL4 install -d -m 0755 %{buildroot}/var/lib/mysql install -d -m 0755 %{buildroot}/var/run/mysqld +install -d -m 0750 %{buildroot}/var/lib/mysql-files # Install all binaries cd $MBD/release @@ -790,6 +791,7 @@ fi %attr(644, root, root) %config(noreplace,missingok) %{_sysconfdir}/logrotate.d/mysql %dir %attr(755, mysql, mysql) /var/lib/mysql %dir %attr(755, mysql, mysql) /var/run/mysqld +%dir %attr(750, mysql, mysql) /var/lib/mysql-files %files common %defattr(-, root, root, -) @@ -916,6 +918,9 @@ fi %endif %changelog +* Mon Sep 26 2016 Balasubramanian Kandasamy - 5.5.53-1 +- Include mysql-files directory + * Tue Jul 05 2016 Balasubramanian Kandasamy - 5.5.51-1 - Remove mysql_config from client subpackage diff --git a/packaging/rpm-sles/mysql.spec.in b/packaging/rpm-sles/mysql.spec.in index a11dfff7b70e0..6652cdcccb614 100644 --- a/packaging/rpm-sles/mysql.spec.in +++ b/packaging/rpm-sles/mysql.spec.in @@ -425,6 +425,7 @@ MBD=$RPM_BUILD_DIR/%{src_dir} install -d -m 0755 %{buildroot}/var/lib/mysql install -d -m 0755 %{buildroot}/var/run/mysql install -d -m 0750 %{buildroot}/var/log/mysql +install -d -m 0750 %{buildroot}/var/lib/mysql-files # Install all binaries cd $MBD/release @@ -638,6 +639,7 @@ fi %dir %attr(755, mysql, mysql) /var/lib/mysql %dir %attr(755, mysql, mysql) /var/run/mysql %dir %attr(750, mysql, mysql) /var/log/mysql +%dir %attr(750, mysql, mysql) /var/lib/mysql-files %files common %defattr(-, root, root, -) @@ -783,6 +785,9 @@ fi %attr(755, root, root) %{_libdir}/mysql/libmysqld.so %changelog +* Mon Sep 26 2016 Balasubramanian Kandasamy - 5.5.53-1 +- Include mysql-files directory + * Tue Sep 29 2015 Balasubramanian Kandasamy - 5.5.47-1 - Added conflicts to mysql-connector-c-shared dependencies diff --git a/packaging/solaris/postinstall-solaris.sh b/packaging/solaris/postinstall-solaris.sh index b024d94f15854..a31e151e1bb32 100644 --- a/packaging/solaris/postinstall-solaris.sh +++ b/packaging/solaris/postinstall-solaris.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,7 @@ mygroup=mysql myuser=mysql mydatadir=/var/lib/mysql basedir=@@basedir@@ +mysecurefiledir=/var/lib/mysql-files if [ -n "$BASEDIR" ] ; then basedir="$BASEDIR" @@ -58,6 +59,11 @@ fi chown -R $myuser:$mygroup $mydatadir +# Create securefile directory +[ -d "$mysecurefiledir" ] || mkdir -p -m 770 "$mysecurefiledir" || exit 1 +chown -R $myuser:$mygroup $mysecurefiledir + + # Solaris patch 119255 (somewhere around revision 42) changes the behaviour # of pkgadd to set TMPDIR internally to a root-owned install directory. This # has the unfortunate side effect of breaking running mysql_install_db with diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e979ea1b731a9..2429db0774b4a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -570,6 +570,7 @@ uint mysql_real_data_home_len, mysql_data_home_len= 1; uint reg_ext_length; const key_map key_map_empty(0); key_map key_map_full(0); // Will be initialized later +char secure_file_real_path[FN_REFLEN]; DATE_TIME_FORMAT global_date_format, global_datetime_format, global_time_format; Time_zone *default_tz; @@ -7598,9 +7599,9 @@ bool is_secure_file_path(char *path) char buff1[FN_REFLEN], buff2[FN_REFLEN]; size_t opt_secure_file_priv_len; /* - All paths are secure if opt_secure_file_path is 0 + All paths are secure if opt_secure_file_priv is 0 */ - if (!opt_secure_file_priv) + if (!opt_secure_file_priv[0]) return TRUE; opt_secure_file_priv_len= strlen(opt_secure_file_priv); @@ -7608,6 +7609,9 @@ bool is_secure_file_path(char *path) if (strlen(path) >= FN_REFLEN) return FALSE; + if (!my_strcasecmp(system_charset_info, opt_secure_file_priv, "NULL")) + return FALSE; + if (my_realpath(buff1, path, 0)) { /* @@ -7640,9 +7644,184 @@ bool is_secure_file_path(char *path) } +/** + check_secure_file_priv_path : Checks path specified through + --secure-file-priv and raises warning in following cases: + 1. If path is empty string or NULL and mysqld is not running + with --bootstrap mode. + 2. If path can access data directory + 3. If path points to a directory which is accessible by + all OS users (non-Windows build only) + + It throws error in following cases: + + 1. If path normalization fails + 2. If it can not get stats of the directory + + @params NONE + + Assumptions : + 1. Data directory path has been normalized + 2. opt_secure_file_priv has been normalized unless it is set + to "NULL". + + @returns Status of validation + @retval true : Validation is successful with/without warnings + @retval false : Validation failed. Error is raised. +*/ + +bool check_secure_file_priv_path() +{ + char datadir_buffer[FN_REFLEN+1]={0}; + char plugindir_buffer[FN_REFLEN+1]={0}; + char whichdir[20]= {0}; + size_t opt_plugindir_len= 0; + size_t opt_datadir_len= 0; + size_t opt_secure_file_priv_len= 0; + bool warn= false; + bool case_insensitive_fs; +#ifndef _WIN32 + MY_STAT dir_stat; +#endif + + if (!opt_secure_file_priv[0]) + { + if (opt_bootstrap) + { + /* + Do not impose --secure-file-priv restriction + in --bootstrap mode + */ + sql_print_information("Ignoring --secure-file-priv value as server is " + "running with --bootstrap."); + } + else + { + sql_print_warning("Insecure configuration for --secure-file-priv: " + "Current value does not restrict location of generated " + "files. Consider setting it to a valid, " + "non-empty path."); + } + return true; + } + + /* + Setting --secure-file-priv to NULL would disable + reading/writing from/to file + */ + if(!my_strcasecmp(system_charset_info, opt_secure_file_priv, "NULL")) + { + sql_print_information("--secure-file-priv is set to NULL. " + "Operations related to importing and exporting " + "data are disabled"); + return true; + } + + /* + Check if --secure-file-priv can access data directory + */ + opt_secure_file_priv_len= strlen(opt_secure_file_priv); + + /* + Adds dir seperator at the end. + This is required in subsequent comparison + */ + convert_dirname(datadir_buffer, mysql_unpacked_real_data_home, NullS); + opt_datadir_len= strlen(datadir_buffer); + + case_insensitive_fs= + (test_if_case_insensitive(datadir_buffer) == 1); + + if (!case_insensitive_fs) + { + if (!strncmp(datadir_buffer, opt_secure_file_priv, + opt_datadir_len < opt_secure_file_priv_len ? + opt_datadir_len : opt_secure_file_priv_len)) + { + warn= true; + strcpy(whichdir, "Data directory"); + } + } + else + { + if (!files_charset_info->coll->strnncoll(files_charset_info, + (uchar *) datadir_buffer, + opt_datadir_len, + (uchar *) opt_secure_file_priv, + opt_secure_file_priv_len, + TRUE)) + { + warn= true; + strcpy(whichdir, "Data directory"); + } + } + + /* + Don't bother comparing --secure-file-priv with --plugin-dir + if we already have a match against --datadir or + --plugin-dir is not pointing to a valid directory. + */ + if (!warn && !my_realpath(plugindir_buffer, opt_plugin_dir, 0)) + { + convert_dirname(plugindir_buffer, plugindir_buffer, NullS); + opt_plugindir_len= strlen(plugindir_buffer); + + if (!case_insensitive_fs) + { + if (!strncmp(plugindir_buffer, opt_secure_file_priv, + opt_plugindir_len < opt_secure_file_priv_len ? + opt_plugindir_len : opt_secure_file_priv_len)) + { + warn= true; + strcpy(whichdir, "Plugin directory"); + } + } + else + { + if (!files_charset_info->coll->strnncoll(files_charset_info, + (uchar *) plugindir_buffer, + opt_plugindir_len, + (uchar *) opt_secure_file_priv, + opt_secure_file_priv_len, + TRUE)) + { + warn= true; + strcpy(whichdir, "Plugin directory"); + } + } + } + + + if (warn) + sql_print_warning("Insecure configuration for --secure-file-priv: " + "%s is accessible through " + "--secure-file-priv. Consider choosing a different " + "directory.", whichdir); + +#ifndef _WIN32 + /* + Check for --secure-file-priv directory's permission + */ + if (!(my_stat(opt_secure_file_priv, &dir_stat, MYF(0)))) + { + sql_print_error("Failed to get stat for directory pointed out " + "by --secure-file-priv"); + return false; + } + + if (dir_stat.st_mode & S_IRWXO) + sql_print_warning("Insecure configuration for --secure-file-priv: " + "Location is accessible to all OS users. " + "Consider choosing a different directory."); +#endif + return true; +} + + static int fix_paths(void) { char buff[FN_REFLEN],*pos; + bool secure_file_priv_nonempty= false; convert_dirname(mysql_home,mysql_home,NullS); /* Resolve symlinks to allow 'mysql_home' to be a relative symlink */ my_realpath(mysql_home,mysql_home,MYF(0)); @@ -7700,29 +7879,56 @@ static int fix_paths(void) Convert the secure-file-priv option to system format, allowing a quick strcmp to check if read or write is in an allowed dir */ - if (opt_secure_file_priv) + if (opt_bootstrap) + opt_secure_file_priv= EMPTY_STR.str; + secure_file_priv_nonempty= opt_secure_file_priv[0] ? true : false; + + if (secure_file_priv_nonempty && strlen(opt_secure_file_priv) > FN_REFLEN) { - if (*opt_secure_file_priv == 0) - { - my_free(opt_secure_file_priv); - opt_secure_file_priv= 0; - } - else + sql_print_warning("Value for --secure-file-priv is longer than maximum " + "limit of %d", FN_REFLEN-1); + return 1; + } + + memset(buff, 0, sizeof(buff)); + if (secure_file_priv_nonempty && + my_strcasecmp(system_charset_info, opt_secure_file_priv, "NULL")) + { + int retval= my_realpath(buff, opt_secure_file_priv, MYF(MY_WME)); + if (!retval) { - if (strlen(opt_secure_file_priv) >= FN_REFLEN) - opt_secure_file_priv[FN_REFLEN-1]= '\0'; - if (my_realpath(buff, opt_secure_file_priv, 0)) + convert_dirname(secure_file_real_path, buff, NullS); +#ifdef WIN32 + MY_DIR *dir= my_dir(secure_file_real_path, MYF(MY_DONT_SORT+MY_WME)); + if (!dir) { - sql_print_warning("Failed to normalize the argument for --secure-file-priv."); - return 1; + retval= 1; } - char *secure_file_real_path= (char *)my_malloc(FN_REFLEN, MYF(MY_FAE)); - convert_dirname(secure_file_real_path, buff, NullS); - my_free(opt_secure_file_priv); - opt_secure_file_priv= secure_file_real_path; + else + { + my_dirend(dir); + } +#endif + } + + if (retval) + { + char err_buffer[FN_REFLEN]; + my_snprintf(err_buffer, FN_REFLEN-1, + "Failed to access directory for --secure-file-priv." + " Please make sure that directory exists and is " + "accessible by MySQL Server. Supplied value : %s", + opt_secure_file_priv); + err_buffer[FN_REFLEN-1]='\0'; + sql_print_error("%s", err_buffer); + return 1; } + opt_secure_file_priv= secure_file_real_path; } - + + if (!check_secure_file_priv_path()) + return 1; + return 0; } diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 0696021cfc09b..d9fda85d8f669 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -68,6 +68,8 @@ char internal_table_name[2]= "*"; char empty_c_string[1]= {0}; /* used for not defined db */ +LEX_STRING EMPTY_STR= { (char *) "", 0 }; + const char * const THD::DEFAULT_WHERE= "field list"; diff --git a/sql/sql_class.h b/sql/sql_class.h index dcc7458ee5043..aa6745e4564ea 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -105,6 +105,7 @@ enum enum_filetype { FILETYPE_CSV, FILETYPE_XML }; extern char internal_table_name[2]; extern char empty_c_string[1]; +extern LEX_STRING EMPTY_STR; extern MYSQL_PLUGIN_IMPORT const char **errmesg; extern bool volatile shutdown_in_progress; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index d08cb4f8ca838..6fd728d638dea 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1941,8 +1941,12 @@ static Sys_var_charptr Sys_secure_file_priv( "secure_file_priv", "Limit LOAD DATA, SELECT ... OUTFILE, and LOAD_FILE() to files " "within specified directory", - PREALLOCATED READ_ONLY GLOBAL_VAR(opt_secure_file_priv), - CMD_LINE(REQUIRED_ARG), IN_FS_CHARSET, DEFAULT(0)); + READ_ONLY GLOBAL_VAR(opt_secure_file_priv), +#ifndef EMBEDDED_LIBRARY + CMD_LINE(REQUIRED_ARG), IN_FS_CHARSET, DEFAULT(DEFAULT_SECURE_FILE_PRIV_DIR)); +#else + CMD_LINE(REQUIRED_ARG), IN_FS_CHARSET, DEFAULT(DEFAULT_SECURE_FILE_PRIV_EMBEDDED_DIR)); +#endif static bool fix_server_id(sys_var *self, THD *thd, enum_var_type type) { diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh index 5af4783f9195a..211ed4f38881d 100644 --- a/support-files/mysql.spec.sh +++ b/support-files/mysql.spec.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -562,6 +562,7 @@ install -d $RBR%{_includedir} install -d $RBR%{_libdir} install -d $RBR%{_mandir} install -d $RBR%{_sbindir} +install -d $RBR/var/lib/mysql-files mkdir -p $RBR%{_sysconfdir}/my.cnf.d @@ -1141,6 +1142,7 @@ echo "=====" >> $STATUS_HISTORY %attr(755, root, root) %{_sysconfdir}/init.d/mysql %attr(755, root, root) %{_datadir}/mysql/ +%dir %attr(750, mysql, mysql) /var/lib/mysql-files # ---------------------------------------------------------------------------- %files -n MySQL-client%{product_suffix} @@ -1226,6 +1228,9 @@ echo "=====" >> $STATUS_HISTORY # merging BK trees) ############################################################################## %changelog +* Mon Sep 26 2016 Balasubramanian Kandasamy +- Include mysql-files directory + * Wed Jul 02 2014 Bjorn Munch - Disable dtrace unconditionally, breaks after we install Oracle dtrace From 5c6169fb309981b564a17bee31b367a18866d674 Mon Sep 17 00:00:00 2001 From: Robert Golebiowski Date: Tue, 27 Sep 2016 11:17:38 +0200 Subject: [PATCH 080/295] Bug #24740291: YASSL UPDATE TO 2.4.2 --- extra/yassl/README | 18 +++ extra/yassl/certs/dsa-cert.pem | 38 ++--- extra/yassl/include/openssl/ssl.h | 2 +- extra/yassl/src/ssl.cpp | 60 +++++--- extra/yassl/taocrypt/include/aes.hpp | 58 ++++++++ extra/yassl/taocrypt/include/integer.hpp | 3 + extra/yassl/taocrypt/src/aes.cpp | 172 ++++++++++++++--------- extra/yassl/taocrypt/src/asn.cpp | 24 ++-- extra/yassl/taocrypt/src/dsa.cpp | 16 ++- extra/yassl/taocrypt/test/test.cpp | 3 + extra/yassl/testsuite/test.hpp | 2 +- 11 files changed, 274 insertions(+), 122 deletions(-) diff --git a/extra/yassl/README b/extra/yassl/README index b5eb88824fb0d..a3d4f60f56128 100644 --- a/extra/yassl/README +++ b/extra/yassl/README @@ -12,6 +12,24 @@ before calling SSL_new(); *** end Note *** +yaSSL Release notes, version 2.4.2 (9/22/2016) + This release of yaSSL fixes a medium security vulnerability. A fix for + potential AES side channel leaks is included that a local user monitoring + the same CPU core cache could exploit. VM users, hyper-threading users, + and users where potential attackers have access to the CPU cache will need + to update if they utilize AES. + + DSA padding fixes for unusual sizes is included as well. Users with DSA + certficiates should update. + +yaSSL Release notes, version 2.4.0 (5/20/2016) + This release of yaSSL fixes the OpenSSL compatibility function + SSL_CTX_load_verify_locations() when using the path directory to allow + unlimited path sizes. Minor Windows build fixes are included. + No high level security fixes in this version but we always recommend + updating. + + yaSSL Release notes, version 2.3.9b (2/03/2016) This release of yaSSL fixes the OpenSSL compatibility function X509_NAME_get_index_by_NID() to use the actual index of the common name diff --git a/extra/yassl/certs/dsa-cert.pem b/extra/yassl/certs/dsa-cert.pem index 10d533edc88b0..10794cbee7313 100644 --- a/extra/yassl/certs/dsa-cert.pem +++ b/extra/yassl/certs/dsa-cert.pem @@ -1,22 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDqzCCA2ugAwIBAgIJAMGqrgDU6DyhMAkGByqGSM44BAMwgY4xCzAJBgNVBAYT +MIIDrzCCA2+gAwIBAgIJAK1zRM7YFcNjMAkGByqGSM44BAMwgZAxCzAJBgNVBAYT AlVTMQ8wDQYDVQQIDAZPcmVnb24xETAPBgNVBAcMCFBvcnRsYW5kMRAwDgYDVQQK -DAd3b2xmU1NMMRAwDgYDVQQLDAd0ZXN0aW5nMRYwFAYDVQQDDA13d3cueWFzc2wu -Y29tMR8wHQYJKoZIhvcNAQkBFhBpbmZvQHdvbGZzc2wuY29tMB4XDTEzMDQyMjIw -MDk0NFoXDTE2MDExNzIwMDk0NFowgY4xCzAJBgNVBAYTAlVTMQ8wDQYDVQQIDAZP -cmVnb24xETAPBgNVBAcMCFBvcnRsYW5kMRAwDgYDVQQKDAd3b2xmU1NMMRAwDgYD -VQQLDAd0ZXN0aW5nMRYwFAYDVQQDDA13d3cueWFzc2wuY29tMR8wHQYJKoZIhvcN -AQkBFhBpbmZvQHdvbGZzc2wuY29tMIIBuDCCASwGByqGSM44BAEwggEfAoGBAL1R -7koy4IrH6sbh6nDEUUPPKgfhxxLCWCVexF2+qzANEr+hC9M002haJXFOfeS9DyoO -WFbL0qMZOuqv+22CaHnoUWl7q3PjJOAI3JH0P54ZyUPuU1909RzgTdIDp5+ikbr7 -KYjnltL73FQVMbjTZQKthIpPn3MjYcF+4jp2W2zFAhUAkcntYND6MGf+eYzIJDN2 -L7SonHUCgYEAklpxErfqznIZjVvqqHFaq+mgAL5J8QrKVmdhYZh/Y8z4jCjoCA8o -TDoFKxf7s2ZzgaPKvglaEKiYqLqic9qY78DYJswzQMLFvjsF4sFZ+pYCBdWPQI4N -PgxCiznK6Ce+JH9ikSBvMvG+tevjr2UpawDIHX3+AWYaZBZwKADAaboDgYUAAoGB -AJ3LY89yHyvQ/TsQ6zlYbovjbk/ogndsMqPdNUvL4RuPTgJP/caaDDa0XJ7ak6A7 -TJ+QheLNwOXoZPYJC4EGFSDAXpYniGhbWIrVTCGe6lmZDfnx40WXS0kk3m/DHaC0 -3ElLAiybxVGxyqoUfbT3Zv1JwftWMuiqHH5uADhdXuXVo1AwTjAdBgNVHQ4EFgQU -IJjk416o4v8qpH9LBtXlR9v8gccwHwYDVR0jBBgwFoAUIJjk416o4v8qpH9LBtXl -R9v8gccwDAYDVR0TBAUwAwEB/zAJBgcqhkjOOAQDAy8AMCwCFCjGKIdOSV12LcTu -k08owGM6YkO1AhQe+K173VuaO/OsDNsxZlKpyH8+1g== +DAd3b2xmU1NMMRAwDgYDVQQLDAd0ZXN0aW5nMRgwFgYDVQQDDA93d3cud29sZnNz +bC5jb20xHzAdBgkqhkiG9w0BCQEWEGluZm9Ad29sZnNzbC5jb20wHhcNMTYwOTIy +MjEyMzA0WhcNMjIwMzE1MjEyMzA0WjCBkDELMAkGA1UEBhMCVVMxDzANBgNVBAgM +Bk9yZWdvbjERMA8GA1UEBwwIUG9ydGxhbmQxEDAOBgNVBAoMB3dvbGZTU0wxEDAO +BgNVBAsMB3Rlc3RpbmcxGDAWBgNVBAMMD3d3dy53b2xmc3NsLmNvbTEfMB0GCSqG +SIb3DQEJARYQaW5mb0B3b2xmc3NsLmNvbTCCAbgwggEsBgcqhkjOOAQBMIIBHwKB +gQC9Ue5KMuCKx+rG4epwxFFDzyoH4ccSwlglXsRdvqswDRK/oQvTNNNoWiVxTn3k +vQ8qDlhWy9KjGTrqr/ttgmh56FFpe6tz4yTgCNyR9D+eGclD7lNfdPUc4E3SA6ef +opG6+ymI55bS+9xUFTG402UCrYSKT59zI2HBfuI6dltsxQIVAJHJ7WDQ+jBn/nmM +yCQzdi+0qJx1AoGBAJJacRK36s5yGY1b6qhxWqvpoAC+SfEKylZnYWGYf2PM+Iwo +6AgPKEw6BSsX+7Nmc4Gjyr4JWhComKi6onPamO/A2CbMM0DCxb47BeLBWfqWAgXV +j0CODT4MQos5yugnviR/YpEgbzLxvrXr469lKWsAyB19/gFmGmQWcCgAwGm6A4GF +AAKBgQCdy2PPch8r0P07EOs5WG6L425P6IJ3bDKj3TVLy+Ebj04CT/3Gmgw2tFye +2pOgO0yfkIXizcDl6GT2CQuBBhUgwF6WJ4hoW1iK1UwhnupZmQ358eNFl0tJJN5v +wx2gtNxJSwIsm8VRscqqFH2092b9ScH7VjLoqhx+bgA4XV7l1aNQME4wHQYDVR0O +BBYEFCCY5ONeqOL/KqR/SwbV5Ufb/IHHMB8GA1UdIwQYMBaAFCCY5ONeqOL/KqR/ +SwbV5Ufb/IHHMAwGA1UdEwQFMAMBAf8wCQYHKoZIzjgEAwMvADAsAhQRYSCVN/Ge +agV3mffU3qNZ92fI0QIUPH7Jp+iASI7U1ocaYDc10qXGaGY= -----END CERTIFICATE----- diff --git a/extra/yassl/include/openssl/ssl.h b/extra/yassl/include/openssl/ssl.h index 83daf3cc81f9e..0609dfc0592f5 100644 --- a/extra/yassl/include/openssl/ssl.h +++ b/extra/yassl/include/openssl/ssl.h @@ -35,7 +35,7 @@ #include "rsa.h" -#define YASSL_VERSION "2.3.9b" +#define YASSL_VERSION "2.4.2" #if defined(__cplusplus) diff --git a/extra/yassl/src/ssl.cpp b/extra/yassl/src/ssl.cpp index cde32df4f43f4..1925e2f759267 100644 --- a/extra/yassl/src/ssl.cpp +++ b/extra/yassl/src/ssl.cpp @@ -161,7 +161,7 @@ int read_file(SSL_CTX* ctx, const char* file, int format, CertType type) TaoCrypt::DSA_PrivateKey dsaKey; dsaKey.Initialize(dsaSource); - if (rsaSource.GetError().What()) { + if (dsaSource.GetError().What()) { // neither worked ret = SSL_FAILURE; } @@ -784,40 +784,67 @@ int SSL_CTX_load_verify_locations(SSL_CTX* ctx, const char* file, WIN32_FIND_DATA FindFileData; HANDLE hFind; - char name[MAX_PATH + 1]; // directory specification - strncpy(name, path, MAX_PATH - 3); - strncat(name, "\\*", 3); + const int DELIMITER_SZ = 2; + const int DELIMITER_STAR_SZ = 3; + int pathSz = (int)strlen(path); + int nameSz = pathSz + DELIMITER_STAR_SZ + 1; // plus 1 for terminator + char* name = NEW_YS char[nameSz]; // directory specification + memset(name, 0, nameSz); + strncpy(name, path, nameSz - DELIMITER_STAR_SZ - 1); + strncat(name, "\\*", DELIMITER_STAR_SZ); hFind = FindFirstFile(name, &FindFileData); - if (hFind == INVALID_HANDLE_VALUE) return SSL_BAD_PATH; + if (hFind == INVALID_HANDLE_VALUE) { + ysArrayDelete(name); + return SSL_BAD_PATH; + } do { - if (FindFileData.dwFileAttributes != FILE_ATTRIBUTE_DIRECTORY) { - strncpy(name, path, MAX_PATH - 2 - HALF_PATH); - strncat(name, "\\", 2); - strncat(name, FindFileData.cFileName, HALF_PATH); + if (!(FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { + int curSz = (int)strlen(FindFileData.cFileName); + if (pathSz + curSz + DELIMITER_SZ + 1 > nameSz) { + ysArrayDelete(name); + // plus 1 for terminator + nameSz = pathSz + curSz + DELIMITER_SZ + 1; + name = NEW_YS char[nameSz]; + } + memset(name, 0, nameSz); + strncpy(name, path, nameSz - curSz - DELIMITER_SZ - 1); + strncat(name, "\\", DELIMITER_SZ); + strncat(name, FindFileData.cFileName, + nameSz - pathSz - DELIMITER_SZ - 1); ret = read_file(ctx, name, SSL_FILETYPE_PEM, CA); } } while (ret == SSL_SUCCESS && FindNextFile(hFind, &FindFileData)); + ysArrayDelete(name); FindClose(hFind); #else // _WIN32 - - const int MAX_PATH = 260; - DIR* dir = opendir(path); if (!dir) return SSL_BAD_PATH; struct dirent* entry; struct stat buf; - char name[MAX_PATH + 1]; + const int DELIMITER_SZ = 1; + int pathSz = (int)strlen(path); + int nameSz = pathSz + DELIMITER_SZ + 1; //plus 1 for null terminator + char* name = NEW_YS char[nameSz]; // directory specification while (ret == SSL_SUCCESS && (entry = readdir(dir))) { - strncpy(name, path, MAX_PATH - 1 - HALF_PATH); - strncat(name, "/", 1); - strncat(name, entry->d_name, HALF_PATH); + int curSz = (int)strlen(entry->d_name); + if (pathSz + curSz + DELIMITER_SZ + 1 > nameSz) { + ysArrayDelete(name); + nameSz = pathSz + DELIMITER_SZ + curSz + 1; + name = NEW_YS char[nameSz]; + } + memset(name, 0, nameSz); + strncpy(name, path, nameSz - curSz - 1); + strncat(name, "/", DELIMITER_SZ); + strncat(name, entry->d_name, nameSz - pathSz - DELIMITER_SZ - 1); + if (stat(name, &buf) < 0) { + ysArrayDelete(name); closedir(dir); return SSL_BAD_STAT; } @@ -826,6 +853,7 @@ int SSL_CTX_load_verify_locations(SSL_CTX* ctx, const char* file, ret = read_file(ctx, name, SSL_FILETYPE_PEM, CA); } + ysArrayDelete(name); closedir(dir); #endif diff --git a/extra/yassl/taocrypt/include/aes.hpp b/extra/yassl/taocrypt/include/aes.hpp index 017630331560b..bccf6e73fc720 100644 --- a/extra/yassl/taocrypt/include/aes.hpp +++ b/extra/yassl/taocrypt/include/aes.hpp @@ -60,6 +60,7 @@ class AES : public Mode_BASE { static const word32 Te[5][256]; static const word32 Td[5][256]; + static const byte CTd4[256]; static const word32* Te0; static const word32* Te1; @@ -80,11 +81,68 @@ class AES : public Mode_BASE { void ProcessAndXorBlock(const byte*, const byte*, byte*) const; + word32 PreFetchTe() const; + word32 PreFetchTd() const; + word32 PreFetchCTd4() const; + AES(const AES&); // hide copy AES& operator=(const AES&); // and assign }; +#if defined(__x86_64__) || defined(_M_X64) || \ + (defined(__ILP32__) && (__ILP32__ >= 1)) + #define TC_CACHE_LINE_SZ 64 +#else + /* default cache line size */ + #define TC_CACHE_LINE_SZ 32 +#endif + +inline word32 AES::PreFetchTe() const +{ + word32 x = 0; + + /* 4 tables of 256 entries */ + for (int i = 0; i < 4; i++) { + /* each entry is 4 bytes */ + for (int j = 0; j < 256; j += TC_CACHE_LINE_SZ/4) { + x &= Te[i][j]; + } + } + + return x; +} + + +inline word32 AES::PreFetchTd() const +{ + word32 x = 0; + + /* 4 tables of 256 entries */ + for (int i = 0; i < 4; i++) { + /* each entry is 4 bytes */ + for (int j = 0; j < 256; j += TC_CACHE_LINE_SZ/4) { + x &= Td[i][j]; + } + } + + return x; +} + + +inline word32 AES::PreFetchCTd4() const +{ + word32 x = 0; + int i; + + for (i = 0; i < 256; i += TC_CACHE_LINE_SZ) { + x &= CTd4[i]; + } + + return x; +} + + typedef BlockCipher AES_ECB_Encryption; typedef BlockCipher AES_ECB_Decryption; diff --git a/extra/yassl/taocrypt/include/integer.hpp b/extra/yassl/taocrypt/include/integer.hpp index 75a3ee3d3df80..05fe189fd585f 100644 --- a/extra/yassl/taocrypt/include/integer.hpp +++ b/extra/yassl/taocrypt/include/integer.hpp @@ -119,6 +119,9 @@ namespace TaoCrypt { +#ifdef _WIN32 + #undef max // avoid name clash +#endif // general MAX template inline const T& max(const T& a, const T& b) diff --git a/extra/yassl/taocrypt/src/aes.cpp b/extra/yassl/taocrypt/src/aes.cpp index ee4c7a6e8a1d3..3fcf80ac20207 100644 --- a/extra/yassl/taocrypt/src/aes.cpp +++ b/extra/yassl/taocrypt/src/aes.cpp @@ -109,10 +109,10 @@ void AES::SetKey(const byte* userKey, word32 keylen, CipherDir /*dummy*/) { temp = rk[3]; rk[4] = rk[0] ^ - (Te4[GETBYTE(temp, 2)] & 0xff000000) ^ - (Te4[GETBYTE(temp, 1)] & 0x00ff0000) ^ - (Te4[GETBYTE(temp, 0)] & 0x0000ff00) ^ - (Te4[GETBYTE(temp, 3)] & 0x000000ff) ^ + (Te2[GETBYTE(temp, 2)] & 0xff000000) ^ + (Te3[GETBYTE(temp, 1)] & 0x00ff0000) ^ + (Te0[GETBYTE(temp, 0)] & 0x0000ff00) ^ + (Te1[GETBYTE(temp, 3)] & 0x000000ff) ^ rcon_[i]; rk[5] = rk[1] ^ rk[4]; rk[6] = rk[2] ^ rk[5]; @@ -128,10 +128,10 @@ void AES::SetKey(const byte* userKey, word32 keylen, CipherDir /*dummy*/) { temp = rk[ 5]; rk[ 6] = rk[ 0] ^ - (Te4[GETBYTE(temp, 2)] & 0xff000000) ^ - (Te4[GETBYTE(temp, 1)] & 0x00ff0000) ^ - (Te4[GETBYTE(temp, 0)] & 0x0000ff00) ^ - (Te4[GETBYTE(temp, 3)] & 0x000000ff) ^ + (Te2[GETBYTE(temp, 2)] & 0xff000000) ^ + (Te3[GETBYTE(temp, 1)] & 0x00ff0000) ^ + (Te0[GETBYTE(temp, 0)] & 0x0000ff00) ^ + (Te1[GETBYTE(temp, 3)] & 0x000000ff) ^ rcon_[i]; rk[ 7] = rk[ 1] ^ rk[ 6]; rk[ 8] = rk[ 2] ^ rk[ 7]; @@ -149,10 +149,10 @@ void AES::SetKey(const byte* userKey, word32 keylen, CipherDir /*dummy*/) { temp = rk[ 7]; rk[ 8] = rk[ 0] ^ - (Te4[GETBYTE(temp, 2)] & 0xff000000) ^ - (Te4[GETBYTE(temp, 1)] & 0x00ff0000) ^ - (Te4[GETBYTE(temp, 0)] & 0x0000ff00) ^ - (Te4[GETBYTE(temp, 3)] & 0x000000ff) ^ + (Te2[GETBYTE(temp, 2)] & 0xff000000) ^ + (Te3[GETBYTE(temp, 1)] & 0x00ff0000) ^ + (Te0[GETBYTE(temp, 0)] & 0x0000ff00) ^ + (Te1[GETBYTE(temp, 3)] & 0x000000ff) ^ rcon_[i]; rk[ 9] = rk[ 1] ^ rk[ 8]; rk[10] = rk[ 2] ^ rk[ 9]; @@ -161,10 +161,10 @@ void AES::SetKey(const byte* userKey, word32 keylen, CipherDir /*dummy*/) break; temp = rk[11]; rk[12] = rk[ 4] ^ - (Te4[GETBYTE(temp, 3)] & 0xff000000) ^ - (Te4[GETBYTE(temp, 2)] & 0x00ff0000) ^ - (Te4[GETBYTE(temp, 1)] & 0x0000ff00) ^ - (Te4[GETBYTE(temp, 0)] & 0x000000ff); + (Te2[GETBYTE(temp, 3)] & 0xff000000) ^ + (Te3[GETBYTE(temp, 2)] & 0x00ff0000) ^ + (Te0[GETBYTE(temp, 1)] & 0x0000ff00) ^ + (Te1[GETBYTE(temp, 0)] & 0x000000ff); rk[13] = rk[ 5] ^ rk[12]; rk[14] = rk[ 6] ^ rk[13]; rk[15] = rk[ 7] ^ rk[14]; @@ -191,25 +191,25 @@ void AES::SetKey(const byte* userKey, word32 keylen, CipherDir /*dummy*/) for (i = 1; i < rounds_; i++) { rk += 4; rk[0] = - Td0[Te4[GETBYTE(rk[0], 3)] & 0xff] ^ - Td1[Te4[GETBYTE(rk[0], 2)] & 0xff] ^ - Td2[Te4[GETBYTE(rk[0], 1)] & 0xff] ^ - Td3[Te4[GETBYTE(rk[0], 0)] & 0xff]; + Td0[Te1[GETBYTE(rk[0], 3)] & 0xff] ^ + Td1[Te1[GETBYTE(rk[0], 2)] & 0xff] ^ + Td2[Te1[GETBYTE(rk[0], 1)] & 0xff] ^ + Td3[Te1[GETBYTE(rk[0], 0)] & 0xff]; rk[1] = - Td0[Te4[GETBYTE(rk[1], 3)] & 0xff] ^ - Td1[Te4[GETBYTE(rk[1], 2)] & 0xff] ^ - Td2[Te4[GETBYTE(rk[1], 1)] & 0xff] ^ - Td3[Te4[GETBYTE(rk[1], 0)] & 0xff]; + Td0[Te1[GETBYTE(rk[1], 3)] & 0xff] ^ + Td1[Te1[GETBYTE(rk[1], 2)] & 0xff] ^ + Td2[Te1[GETBYTE(rk[1], 1)] & 0xff] ^ + Td3[Te1[GETBYTE(rk[1], 0)] & 0xff]; rk[2] = - Td0[Te4[GETBYTE(rk[2], 3)] & 0xff] ^ - Td1[Te4[GETBYTE(rk[2], 2)] & 0xff] ^ - Td2[Te4[GETBYTE(rk[2], 1)] & 0xff] ^ - Td3[Te4[GETBYTE(rk[2], 0)] & 0xff]; + Td0[Te1[GETBYTE(rk[2], 3)] & 0xff] ^ + Td1[Te1[GETBYTE(rk[2], 2)] & 0xff] ^ + Td2[Te1[GETBYTE(rk[2], 1)] & 0xff] ^ + Td3[Te1[GETBYTE(rk[2], 0)] & 0xff]; rk[3] = - Td0[Te4[GETBYTE(rk[3], 3)] & 0xff] ^ - Td1[Te4[GETBYTE(rk[3], 2)] & 0xff] ^ - Td2[Te4[GETBYTE(rk[3], 1)] & 0xff] ^ - Td3[Te4[GETBYTE(rk[3], 0)] & 0xff]; + Td0[Te1[GETBYTE(rk[3], 3)] & 0xff] ^ + Td1[Te1[GETBYTE(rk[3], 2)] & 0xff] ^ + Td2[Te1[GETBYTE(rk[3], 1)] & 0xff] ^ + Td3[Te1[GETBYTE(rk[3], 0)] & 0xff]; } } } @@ -244,6 +244,7 @@ void AES::encrypt(const byte* inBlock, const byte* xorBlock, s2 ^= rk[2]; s3 ^= rk[3]; + s0 |= PreFetchTe(); /* * Nr - 1 full rounds: */ @@ -312,28 +313,28 @@ void AES::encrypt(const byte* inBlock, const byte* xorBlock, */ s0 = - (Te4[GETBYTE(t0, 3)] & 0xff000000) ^ - (Te4[GETBYTE(t1, 2)] & 0x00ff0000) ^ - (Te4[GETBYTE(t2, 1)] & 0x0000ff00) ^ - (Te4[GETBYTE(t3, 0)] & 0x000000ff) ^ + (Te2[GETBYTE(t0, 3)] & 0xff000000) ^ + (Te3[GETBYTE(t1, 2)] & 0x00ff0000) ^ + (Te0[GETBYTE(t2, 1)] & 0x0000ff00) ^ + (Te1[GETBYTE(t3, 0)] & 0x000000ff) ^ rk[0]; s1 = - (Te4[GETBYTE(t1, 3)] & 0xff000000) ^ - (Te4[GETBYTE(t2, 2)] & 0x00ff0000) ^ - (Te4[GETBYTE(t3, 1)] & 0x0000ff00) ^ - (Te4[GETBYTE(t0, 0)] & 0x000000ff) ^ + (Te2[GETBYTE(t1, 3)] & 0xff000000) ^ + (Te3[GETBYTE(t2, 2)] & 0x00ff0000) ^ + (Te0[GETBYTE(t3, 1)] & 0x0000ff00) ^ + (Te1[GETBYTE(t0, 0)] & 0x000000ff) ^ rk[1]; s2 = - (Te4[GETBYTE(t2, 3)] & 0xff000000) ^ - (Te4[GETBYTE(t3, 2)] & 0x00ff0000) ^ - (Te4[GETBYTE(t0, 1)] & 0x0000ff00) ^ - (Te4[GETBYTE(t1, 0)] & 0x000000ff) ^ + (Te2[GETBYTE(t2, 3)] & 0xff000000) ^ + (Te3[GETBYTE(t3, 2)] & 0x00ff0000) ^ + (Te0[GETBYTE(t0, 1)] & 0x0000ff00) ^ + (Te1[GETBYTE(t1, 0)] & 0x000000ff) ^ rk[2]; s3 = - (Te4[GETBYTE(t3, 3)] & 0xff000000) ^ - (Te4[GETBYTE(t0, 2)] & 0x00ff0000) ^ - (Te4[GETBYTE(t1, 1)] & 0x0000ff00) ^ - (Te4[GETBYTE(t2, 0)] & 0x000000ff) ^ + (Te2[GETBYTE(t3, 3)] & 0xff000000) ^ + (Te3[GETBYTE(t0, 2)] & 0x00ff0000) ^ + (Te0[GETBYTE(t1, 1)] & 0x0000ff00) ^ + (Te1[GETBYTE(t2, 0)] & 0x000000ff) ^ rk[3]; @@ -358,6 +359,8 @@ void AES::decrypt(const byte* inBlock, const byte* xorBlock, s2 ^= rk[2]; s3 ^= rk[3]; + s0 |= PreFetchTd(); + /* * Nr - 1 full rounds: */ @@ -423,29 +426,32 @@ void AES::decrypt(const byte* inBlock, const byte* xorBlock, * apply last round and * map cipher state to byte array block: */ + + t0 |= PreFetchCTd4(); + s0 = - (Td4[GETBYTE(t0, 3)] & 0xff000000) ^ - (Td4[GETBYTE(t3, 2)] & 0x00ff0000) ^ - (Td4[GETBYTE(t2, 1)] & 0x0000ff00) ^ - (Td4[GETBYTE(t1, 0)] & 0x000000ff) ^ + ((word32)CTd4[GETBYTE(t0, 3)] << 24) ^ + ((word32)CTd4[GETBYTE(t3, 2)] << 16) ^ + ((word32)CTd4[GETBYTE(t2, 1)] << 8) ^ + ((word32)CTd4[GETBYTE(t1, 0)]) ^ rk[0]; s1 = - (Td4[GETBYTE(t1, 3)] & 0xff000000) ^ - (Td4[GETBYTE(t0, 2)] & 0x00ff0000) ^ - (Td4[GETBYTE(t3, 1)] & 0x0000ff00) ^ - (Td4[GETBYTE(t2, 0)] & 0x000000ff) ^ + ((word32)CTd4[GETBYTE(t1, 3)] << 24) ^ + ((word32)CTd4[GETBYTE(t0, 2)] << 16) ^ + ((word32)CTd4[GETBYTE(t3, 1)] << 8) ^ + ((word32)CTd4[GETBYTE(t2, 0)]) ^ rk[1]; s2 = - (Td4[GETBYTE(t2, 3)] & 0xff000000) ^ - (Td4[GETBYTE(t1, 2)] & 0x00ff0000) ^ - (Td4[GETBYTE(t0, 1)] & 0x0000ff00) ^ - (Td4[GETBYTE(t3, 0)] & 0x000000ff) ^ + ((word32)CTd4[GETBYTE(t2, 3)] << 24 ) ^ + ((word32)CTd4[GETBYTE(t1, 2)] << 16 ) ^ + ((word32)CTd4[GETBYTE(t0, 1)] << 8 ) ^ + ((word32)CTd4[GETBYTE(t3, 0)]) ^ rk[2]; s3 = - (Td4[GETBYTE(t3, 3)] & 0xff000000) ^ - (Td4[GETBYTE(t2, 2)] & 0x00ff0000) ^ - (Td4[GETBYTE(t1, 1)] & 0x0000ff00) ^ - (Td4[GETBYTE(t0, 0)] & 0x000000ff) ^ + ((word32)CTd4[GETBYTE(t3, 3)] << 24) ^ + ((word32)CTd4[GETBYTE(t2, 2)] << 16) ^ + ((word32)CTd4[GETBYTE(t1, 1)] << 8) ^ + ((word32)CTd4[GETBYTE(t0, 0)]) ^ rk[3]; gpBlock::Put(xorBlock, outBlock)(s0)(s1)(s2)(s3); @@ -1826,18 +1832,52 @@ const word32 AES::Td[5][256] = { } }; +const byte AES::CTd4[256] = +{ + 0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U, + 0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU, + 0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U, + 0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU, + 0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU, + 0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU, + 0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U, + 0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U, + 0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U, + 0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U, + 0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU, + 0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U, + 0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU, + 0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U, + 0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U, + 0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU, + 0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU, + 0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U, + 0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U, + 0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU, + 0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U, + 0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU, + 0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U, + 0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U, + 0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U, + 0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU, + 0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU, + 0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU, + 0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U, + 0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U, + 0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U, + 0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU, +}; + const word32* AES::Te0 = AES::Te[0]; const word32* AES::Te1 = AES::Te[1]; const word32* AES::Te2 = AES::Te[2]; const word32* AES::Te3 = AES::Te[3]; -const word32* AES::Te4 = AES::Te[4]; const word32* AES::Td0 = AES::Td[0]; const word32* AES::Td1 = AES::Td[1]; const word32* AES::Td2 = AES::Td[2]; const word32* AES::Td3 = AES::Td[3]; -const word32* AES::Td4 = AES::Td[4]; diff --git a/extra/yassl/taocrypt/src/asn.cpp b/extra/yassl/taocrypt/src/asn.cpp index a210d805452f3..7ff3c7167d23b 100644 --- a/extra/yassl/taocrypt/src/asn.cpp +++ b/extra/yassl/taocrypt/src/asn.cpp @@ -1209,17 +1209,17 @@ word32 DecodeDSA_Signature(byte* decoded, const byte* encoded, word32 sz) } word32 rLen = GetLength(source); if (rLen != 20) { - if (rLen == 21) { // zero at front, eat + while (rLen > 20 && source.remaining() > 0) { // zero's at front, eat source.next(); --rLen; } - else if (rLen == 19) { // add zero to front so 20 bytes + if (rLen < 20) { // add zero's to front so 20 bytes + word32 tmpLen = rLen; + while (tmpLen < 20) { decoded[0] = 0; decoded++; + tmpLen++; } - else { - source.SetError(DSA_SZ_E); - return 0; } } memcpy(decoded, source.get_buffer() + source.get_index(), rLen); @@ -1232,17 +1232,17 @@ word32 DecodeDSA_Signature(byte* decoded, const byte* encoded, word32 sz) } word32 sLen = GetLength(source); if (sLen != 20) { - if (sLen == 21) { - source.next(); // zero at front, eat + while (sLen > 20 && source.remaining() > 0) { + source.next(); // zero's at front, eat --sLen; } - else if (sLen == 19) { - decoded[rLen] = 0; // add zero to front so 20 bytes + if (sLen < 20) { // add zero's to front so 20 bytes + word32 tmpLen = sLen; + while (tmpLen < 20) { + decoded[rLen] = 0; decoded++; + tmpLen++; } - else { - source.SetError(DSA_SZ_E); - return 0; } } memcpy(decoded + rLen, source.get_buffer() + source.get_index(), sLen); diff --git a/extra/yassl/taocrypt/src/dsa.cpp b/extra/yassl/taocrypt/src/dsa.cpp index bf116d3e48d12..b19fed9235b24 100644 --- a/extra/yassl/taocrypt/src/dsa.cpp +++ b/extra/yassl/taocrypt/src/dsa.cpp @@ -172,6 +172,7 @@ word32 DSA_Signer::Sign(const byte* sha_digest, byte* sig, const Integer& q = key_.GetSubGroupOrder(); const Integer& g = key_.GetSubGroupGenerator(); const Integer& x = key_.GetPrivatePart(); + byte* tmpPtr = sig; // initial signature output Integer k(rng, 1, q - 1); @@ -187,22 +188,23 @@ word32 DSA_Signer::Sign(const byte* sha_digest, byte* sig, return -1; int rSz = r_.ByteCount(); + int tmpSz = rSz; - if (rSz == 19) { - sig[0] = 0; - sig++; + while (tmpSz++ < SHA::DIGEST_SIZE) { + *sig++ = 0; } r_.Encode(sig, rSz); + sig = tmpPtr + SHA::DIGEST_SIZE; // advance sig output to s int sSz = s_.ByteCount(); + tmpSz = sSz; - if (sSz == 19) { - sig[rSz] = 0; - sig++; + while (tmpSz++ < SHA::DIGEST_SIZE) { + *sig++ = 0; } - s_.Encode(sig + rSz, sSz); + s_.Encode(sig, sSz); return 40; } diff --git a/extra/yassl/taocrypt/test/test.cpp b/extra/yassl/taocrypt/test/test.cpp index a7d5cb3e8af35..fc1f0e8762ddc 100644 --- a/extra/yassl/taocrypt/test/test.cpp +++ b/extra/yassl/taocrypt/test/test.cpp @@ -1277,6 +1277,9 @@ int dsa_test() if (!verifier.Verify(digest, decoded)) return -90; + if (!verifier.Verify(digest, signature)) + return -91; + return 0; } diff --git a/extra/yassl/testsuite/test.hpp b/extra/yassl/testsuite/test.hpp index 5c9dc7ce117ef..e2e44c24027df 100644 --- a/extra/yassl/testsuite/test.hpp +++ b/extra/yassl/testsuite/test.hpp @@ -22,7 +22,6 @@ #define yaSSL_TEST_HPP #include "runtime.hpp" -#include "openssl/ssl.h" /* openssl compatibility test */ #include "error.hpp" #include #include @@ -56,6 +55,7 @@ #endif #define SOCKET_T int #endif /* _WIN32 */ +#include "openssl/ssl.h" /* openssl compatibility test */ #ifdef _MSC_VER From 794c826244c4d8b082600afa4999130fe688aa49 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Sep 2016 14:29:23 +0200 Subject: [PATCH 081/295] MDEV-10890 plugins.pam fails in buildbot with valgrind initialize uninitialized value --- plugin/auth_pam/auth_pam.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/auth_pam/auth_pam.c b/plugin/auth_pam/auth_pam.c index ac1b3b2da0948..8810a418cd380 100644 --- a/plugin/auth_pam/auth_pam.c +++ b/plugin/auth_pam/auth_pam.c @@ -131,7 +131,7 @@ static int pam_auth(MYSQL_PLUGIN_VIO *vio, MYSQL_SERVER_AUTH_INFO *info) { pam_handle_t *pamh = NULL; int status; - const char *new_username; + const char *new_username= NULL; struct param param; /* The following is written in such a way to make also solaris happy */ struct pam_conv pam_start_arg = { &conv, (char*) ¶m }; @@ -139,7 +139,7 @@ static int pam_auth(MYSQL_PLUGIN_VIO *vio, MYSQL_SERVER_AUTH_INFO *info) /* get the service name, as specified in - CREATE USER ... IDENTIFIED WITH pam_auth AS "service" + CREATE USER ... IDENTIFIED WITH pam AS "service" */ const char *service = info->auth_string && info->auth_string[0] ? info->auth_string : "mysql"; From 735a4a17c2b83a214dcd5279c66cd3577ef76eb1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 28 Sep 2016 17:59:11 +0200 Subject: [PATCH 082/295] MDEV-10508 Mariadb crash on out of disk space during dump import update info->write_end and info->write_pos together, with no "return on error" in between, otherwise write_end might end up being smaller than write_pos --- mysql-test/r/slowlog_enospace-10508.result | 60 ++++++++++++++++++++++ mysql-test/t/slowlog_enospace-10508.test | 24 +++++++++ mysys/mf_iocache.c | 4 +- 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 mysql-test/r/slowlog_enospace-10508.result create mode 100644 mysql-test/t/slowlog_enospace-10508.test diff --git a/mysql-test/r/slowlog_enospace-10508.result b/mysql-test/r/slowlog_enospace-10508.result new file mode 100644 index 0000000000000..f39bfa2f00e7e --- /dev/null +++ b/mysql-test/r/slowlog_enospace-10508.result @@ -0,0 +1,60 @@ +call mtr.add_suppression('Error writing file.*errno: 28 '); +create table t1 (a int, b int) engine=memory; +insert t1 select seq, seq+1 from seq_1_to_1000; +set global general_log=0; +set global log_queries_not_using_indexes=1; +set debug_dbug='+d,simulate_file_write_error'; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +select * from t1 where a>10; +set debug_dbug=''; +set global general_log=1; +set global log_queries_not_using_indexes=default; +drop table t1; diff --git a/mysql-test/t/slowlog_enospace-10508.test b/mysql-test/t/slowlog_enospace-10508.test new file mode 100644 index 0000000000000..b2c26a5984da6 --- /dev/null +++ b/mysql-test/t/slowlog_enospace-10508.test @@ -0,0 +1,24 @@ +# +# MDEV-10508 Mariadb crash on out of disk space during dump import +# +--source include/have_sequence.inc +--source include/have_debug.inc + +call mtr.add_suppression('Error writing file.*errno: 28 '); +create table t1 (a int, b int) engine=memory; +insert t1 select seq, seq+1 from seq_1_to_1000; +set global general_log=0; +set global log_queries_not_using_indexes=1; +set debug_dbug='+d,simulate_file_write_error'; +--disable_result_log +--let $run= 50 +while ($run) +{ + select * from t1 where a>10; + dec $run; +} +--enable_result_log +set debug_dbug=''; +set global general_log=1; +set global log_queries_not_using_indexes=default; +drop table t1; diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index daf99065eb08a..4afe74da0774f 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -1825,8 +1825,6 @@ int my_b_flush_io_cache(IO_CACHE *info, int need_append_buffer_lock) if ((length=(size_t) (info->write_pos - info->write_buffer))) { - info->write_end= (info->write_buffer + info->buffer_length - - ((info->pos_in_file + length) & (IO_SIZE - 1))); if (append_cache) { @@ -1848,6 +1846,8 @@ int my_b_flush_io_cache(IO_CACHE *info, int need_append_buffer_lock) set_if_bigger(info->end_of_file, info->pos_in_file); } + info->write_end= (info->write_buffer + info->buffer_length - + ((info->pos_in_file + length) & (IO_SIZE - 1))); info->write_pos= info->write_buffer; ++info->disk_writes; UNLOCK_APPEND_BUFFER; From c9ded859a6fad25fc513d3a0c7d17f826e567691 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Wed, 21 Sep 2016 10:51:37 +0200 Subject: [PATCH 083/295] MDEV-10853 netcat help output in error log when running xtrabackup SST --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- scripts/wsrep_sst_xtrabackup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index b5556c60c642d..6ce3120be4342 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -175,7 +175,7 @@ get_transfer() fi wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h | grep -q ncat;then + if nc -h 2>&1 | grep -q ncat;then tcmd="nc -l ${TSST_PORT}" else tcmd="nc -dl ${TSST_PORT}" diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index b40be208be759..287faf6b9def0 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -149,7 +149,7 @@ get_transfer() fi wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h | grep -q ncat;then + if nc -h 2>&1 | grep -q ncat;then tcmd="nc -l ${TSST_PORT}" else tcmd="nc -dl ${TSST_PORT}" From 88f2ec6f207be182d782d9176a66bf66b8fbf65f Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 28 Sep 2016 13:23:31 -0400 Subject: [PATCH 084/295] MDEV-10041: Server crashes sporadically during bootstrap while running wsrep tests The crash is caused due to a race condition where wsrep startup threads invoke ha_maria::implicit_commit() method while maria_hton is partially initialized. The fix is to skip this method if plugins are uninitialized. --- storage/maria/ha_maria.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 5fdd2a920a54b..c6d457eeea86f 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -2839,9 +2839,10 @@ int ha_maria::implicit_commit(THD *thd, bool new_trn) int error; uint locked_tables; DYNAMIC_ARRAY used_tables; + extern my_bool plugins_are_initialized; DBUG_ENTER("ha_maria::implicit_commit"); - if (!maria_hton || !(trn= THD_TRN)) + if (!maria_hton || !plugins_are_initialized || !(trn= THD_TRN)) DBUG_RETURN(0); if (!new_trn && (thd->locked_tables_mode == LTM_LOCK_TABLES || thd->locked_tables_mode == LTM_PRELOCKED_UNDER_LOCK_TABLES)) From 7c525ce36babf8cbceb05595a551dd140188e1ba Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 28 Sep 2016 13:26:13 -0400 Subject: [PATCH 085/295] MDEV-9312: storage engine not enforced during galera cluster replication Perform a post initialization of plugin-related variables of wsrep threads after their global counterparts have been initialized. --- .../galera/r/enforce_storage_engine2.result | 24 ++++++ .../galera/t/enforce_storage_engine2.opt | 2 + .../galera/t/enforce_storage_engine2.test | 20 +++++ sql/mysqld.cc | 15 ++++ sql/sql_plugin.cc | 86 +++++++++++++------ sql/sql_plugin.h | 7 ++ sql/wsrep_mysqld.cc | 9 +- 7 files changed, 137 insertions(+), 26 deletions(-) create mode 100644 mysql-test/suite/galera/r/enforce_storage_engine2.result create mode 100644 mysql-test/suite/galera/t/enforce_storage_engine2.opt create mode 100644 mysql-test/suite/galera/t/enforce_storage_engine2.test diff --git a/mysql-test/suite/galera/r/enforce_storage_engine2.result b/mysql-test/suite/galera/r/enforce_storage_engine2.result new file mode 100644 index 0000000000000..053c37d6854ae --- /dev/null +++ b/mysql-test/suite/galera/r/enforce_storage_engine2.result @@ -0,0 +1,24 @@ +# +# MDEV-9312: storage engine not enforced during galera cluster +# replication +# +CREATE TABLE t1(i INT) ENGINE=INNODB; +CREATE TABLE t2(i INT) ENGINE=MYISAM; +Warnings: +Note 1266 Using storage engine InnoDB for table 't2' +SHOW TABLES; +Tables_in_test +t1 +t2 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `i` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `i` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +DROP TABLE t1, t2; +# End of tests diff --git a/mysql-test/suite/galera/t/enforce_storage_engine2.opt b/mysql-test/suite/galera/t/enforce_storage_engine2.opt new file mode 100644 index 0000000000000..03f7dc5e527c6 --- /dev/null +++ b/mysql-test/suite/galera/t/enforce_storage_engine2.opt @@ -0,0 +1,2 @@ +--enforce_storage_engine=innodb --sql_mode='' + diff --git a/mysql-test/suite/galera/t/enforce_storage_engine2.test b/mysql-test/suite/galera/t/enforce_storage_engine2.test new file mode 100644 index 0000000000000..7a822bced595f --- /dev/null +++ b/mysql-test/suite/galera/t/enforce_storage_engine2.test @@ -0,0 +1,20 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--echo # +--echo # MDEV-9312: storage engine not enforced during galera cluster +--echo # replication +--echo # +--connection node_1 +CREATE TABLE t1(i INT) ENGINE=INNODB; +CREATE TABLE t2(i INT) ENGINE=MYISAM; + +--connection node_2 +SHOW TABLES; +SHOW CREATE TABLE t1; +SHOW CREATE TABLE t2; + +# Cleanup +DROP TABLE t1, t2; + +--echo # End of tests diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8da8273083c2b..f69416c34ea62 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5076,6 +5076,12 @@ static int init_server_components() opt_bin_logname= my_once_strdup(buf, MYF(MY_WME)); } + /* + Since some wsrep threads (THDs) are create before plugins are + initialized, LOCK_plugin mutex needs to be initialized here. + */ + plugin_mutex_init(); + /* Wsrep initialization must happen at this point, because: - opt_bin_logname must be known when starting replication @@ -5304,6 +5310,15 @@ static int init_server_components() #endif #ifdef WITH_WSREP + /* + Now is the right time to initialize members of wsrep startup threads + that rely on plugins and other related global system variables to be + initialized. This initialization was not possible before, as plugins + (and thus some global system variables) are initialized after wsrep + startup threads are created. + */ + wsrep_plugins_post_init(); + if (WSREP_ON && !opt_bin_log) { wsrep_emulate_bin_log= 1; diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index e72869605996d..d86876a973ba0 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1548,9 +1548,6 @@ int plugin_init(int *argc, char **argv, int flags) get_bookmark_hash_key, NULL, HASH_UNIQUE)) goto err; - - mysql_mutex_init(key_LOCK_plugin, &LOCK_plugin, MY_MUTEX_INIT_FAST); - if (my_init_dynamic_array(&plugin_dl_array, sizeof(struct st_plugin_dl *), 16, 16, MYF(0)) || my_init_dynamic_array(&plugin_array, @@ -3098,28 +3095,19 @@ void plugin_thdvar_init(THD *thd) thd->variables.dynamic_variables_size= 0; thd->variables.dynamic_variables_ptr= 0; - if (IF_WSREP((!WSREP(thd) || !thd->wsrep_applier),1)) - { - mysql_mutex_lock(&LOCK_plugin); - thd->variables.table_plugin= - intern_plugin_lock(NULL, global_system_variables.table_plugin); - if (global_system_variables.tmp_table_plugin) - thd->variables.tmp_table_plugin= - intern_plugin_lock(NULL, global_system_variables.tmp_table_plugin); - if (global_system_variables.enforced_table_plugin) - thd->variables.enforced_table_plugin= - intern_plugin_lock(NULL, global_system_variables.enforced_table_plugin); - intern_plugin_unlock(NULL, old_table_plugin); - intern_plugin_unlock(NULL, old_tmp_table_plugin); - intern_plugin_unlock(NULL, old_enforced_table_plugin); - mysql_mutex_unlock(&LOCK_plugin); - } - else - { - thd->variables.table_plugin= NULL; - thd->variables.tmp_table_plugin= NULL; - thd->variables.enforced_table_plugin= NULL; - } + mysql_mutex_lock(&LOCK_plugin); + thd->variables.table_plugin= + intern_plugin_lock(NULL, global_system_variables.table_plugin); + if (global_system_variables.tmp_table_plugin) + thd->variables.tmp_table_plugin= + intern_plugin_lock(NULL, global_system_variables.tmp_table_plugin); + if (global_system_variables.enforced_table_plugin) + thd->variables.enforced_table_plugin= + intern_plugin_lock(NULL, global_system_variables.enforced_table_plugin); + intern_plugin_unlock(NULL, old_table_plugin); + intern_plugin_unlock(NULL, old_tmp_table_plugin); + intern_plugin_unlock(NULL, old_enforced_table_plugin); + mysql_mutex_unlock(&LOCK_plugin); DBUG_VOID_RETURN; } @@ -4243,3 +4231,51 @@ int thd_setspecific(MYSQL_THD thd, MYSQL_THD_KEY_T key, void *value) return 0; } +void plugin_mutex_init() +{ + mysql_mutex_init(key_LOCK_plugin, &LOCK_plugin, MY_MUTEX_INIT_FAST); +} + +#ifdef WITH_WSREP + +/* + Placeholder for global_system_variables.table_plugin required during + initialization of startup wsrep threads. +*/ +static st_plugin_int wsrep_dummy_plugin; +static st_plugin_int *wsrep_dummy_plugin_ptr; + +/* + Initialize wsrep_dummy_plugin and assign it to + global_system_variables.table_plugin. +*/ +void wsrep_plugins_pre_init() +{ + wsrep_dummy_plugin_ptr= &wsrep_dummy_plugin; + wsrep_dummy_plugin.state= PLUGIN_IS_DISABLED; + global_system_variables.table_plugin= + plugin_int_to_ref(wsrep_dummy_plugin_ptr); +} + +/* + This function is intended to be called after the plugins and related + global system variables are initialized. It re-initializes some data + members of wsrep startup threads with correct values, as these value + were not available at the time these threads were created. +*/ +void wsrep_plugins_post_init() +{ + THD *thd; + I_List_iterator it(threads); + + while ((thd= it++)) + { + if (IF_WSREP(thd->wsrep_applier,1)) + { + plugin_thdvar_init(thd); + } + } + + return; +} +#endif /* WITH_WSREP */ diff --git a/sql/sql_plugin.h b/sql/sql_plugin.h index 77ec24bbe9c0a..d11c449962a87 100644 --- a/sql/sql_plugin.h +++ b/sql/sql_plugin.h @@ -180,6 +180,7 @@ sys_var *find_plugin_sysvar(st_plugin_int *plugin, st_mysql_sys_var *var); void plugin_opt_set_limits(struct my_option *, const struct st_mysql_sys_var *); extern SHOW_COMP_OPTION plugin_status(const char *name, size_t len, int type); extern bool check_valid_path(const char *path, size_t length); +extern void plugin_mutex_init(); typedef my_bool (plugin_foreach_func)(THD *thd, plugin_ref plugin, @@ -194,3 +195,9 @@ extern bool plugin_dl_foreach(THD *thd, const LEX_STRING *dl, extern void sync_dynamic_session_variables(THD* thd, bool global_lock); #endif + +#ifdef WITH_WSREP +extern void wsrep_plugins_pre_init(); +extern void wsrep_plugins_post_init(); +#endif /* WITH_WSREP */ + diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 776bf4a3ab266..5f25ea1ef380e 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -36,6 +36,7 @@ #include #include "log_event.h" #include +#include "sql_plugin.h" /* wsrep_plugins_pre_init() */ wsrep_t *wsrep = NULL; /* @@ -771,7 +772,6 @@ void wsrep_thr_init() mysql_mutex_init(key_LOCK_wsrep_config_state, &LOCK_wsrep_config_state, MY_MUTEX_INIT_FAST); } - void wsrep_init_startup (bool first) { if (wsrep_init()) unireg_abort(1); @@ -782,6 +782,13 @@ void wsrep_init_startup (bool first) wsrep_debug, wsrep_convert_LOCK_to_trx, (wsrep_on_fun)wsrep_on); + /* + Pre-initialize global_system_variables.table_plugin with a dummy engine + (placeholder) required during the initialization of wsrep threads (THDs). + (see: plugin_thdvar_init()) + */ + wsrep_plugins_pre_init(); + /* Skip replication start if dummy wsrep provider is loaded */ if (!strcmp(wsrep_provider, WSREP_NONE)) return; From d5dfa0f1c20614ca083a0c4f841534f2f68808dd Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 28 Sep 2016 13:27:34 -0400 Subject: [PATCH 086/295] MDEV-9416: MariaDB galera got signal 11 when altering table add unique index When a BF thread attempts to abort a victim thread's transaction, the victim thread is not locked and thus its not safe to rely on its data structures like htons registered for the trx. So, instead of getting the registered htons from victim, innodb's hton can be looked up directly from installed_htons[] and used to abort the transaction. (Same technique is used in older versions) --- sql/handler.cc | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 78e0c5b592379..802364cb6e5d3 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6096,6 +6096,13 @@ void handler::set_lock_type(enum thr_lock_type lock) @note Aborting the transaction does NOT end it, it still has to be rolled back with hton->rollback(). + @note It is safe to abort from one thread (bf_thd) the transaction, + running in another thread (victim_thd), because InnoDB's lock_sys and + trx_mutex guarantee the necessary protection. However, its not safe + to access victim_thd->transaction, because it's not protected from + concurrent accesses. And it's an overkill to take LOCK_plugin and + iterate the whole installed_htons[] array every time. + @param bf_thd brute force THD asking for the abort @param victim_thd victim THD to be aborted @@ -6112,29 +6119,16 @@ int ha_abort_transaction(THD *bf_thd, THD *victim_thd, my_bool signal) DBUG_RETURN(0); } - /* Try statement transaction if standard one is not set. */ - THD_TRANS *trans= (victim_thd->transaction.all.ha_list) ? - &victim_thd->transaction.all : &victim_thd->transaction.stmt; - - Ha_trx_info *ha_info= trans->ha_list, *ha_info_next; - - for (; ha_info; ha_info= ha_info_next) + handlerton *hton= installed_htons[DB_TYPE_INNODB]; + if (hton && hton->abort_transaction) { - handlerton *hton= ha_info->ht(); - if (!hton->abort_transaction) - { - /* Skip warning for binlog & wsrep. */ - if (hton->db_type != DB_TYPE_BINLOG && hton != wsrep_hton) - { - WSREP_WARN("Cannot abort transaction."); - } - } - else - { - hton->abort_transaction(hton, bf_thd, victim_thd, signal); - } - ha_info_next= ha_info->next(); + hton->abort_transaction(hton, bf_thd, victim_thd, signal); } + else + { + WSREP_WARN("Cannot abort InnoDB transaction"); + } + DBUG_RETURN(0); } From 9ff9acb3079b1c48f5be0f0a689cbbdda82b4c0e Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 22 Sep 2016 17:52:05 +0200 Subject: [PATCH 087/295] MDEV-10716: Assertion `real_type() != FIELD_ITEM' failed in Item_ref::build_equal_items(THD*, COND_EQUAL*, bool, COND_EQUAL**) Degenerated condition in AND should be treated in the same way as in WHERE/HAVING alone (i.e reference should be processed as well as fields) --- mysql-test/r/having.result | 20 ++++++++++++++++++++ mysql-test/t/having.test | 15 +++++++++++++++ sql/item_cmpfunc.cc | 3 ++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/having.result b/mysql-test/r/having.result index eda67460205e0..7fdec5a29cd29 100644 --- a/mysql-test/r/having.result +++ b/mysql-test/r/having.result @@ -709,3 +709,23 @@ c1 c2 x x DROP TABLE t1,t2; End of 10.0 tests +# +# MDEV-10716: Assertion `real_type() != FIELD_ITEM' failed in +# Item_ref::build_equal_items(THD*, COND_EQUAL*, bool, COND_EQUAL**) +# +CREATE TABLE t1 (i INT); +INSERT INTO t1 VALUES (1),(2); +SELECT i, COUNT(*) FROM t1 GROUP BY i HAVING i<>0 AND 1; +i COUNT(*) +1 1 +2 1 +SELECT i-1 A, COUNT(*) FROM t1 GROUP BY i HAVING A AND 1; +A COUNT(*) +1 1 +CREATE VIEW v1 as select i, i-1 as A from t1; +SELECT A, COUNT(*) FROM v1 GROUP BY i HAVING A AND 1; +A COUNT(*) +1 1 +DROP VIEW v1; +DROP TABLE t1; +End of 10.1 tests diff --git a/mysql-test/t/having.test b/mysql-test/t/having.test index 0f6be0b0ec640..f826feff5c0f6 100644 --- a/mysql-test/t/having.test +++ b/mysql-test/t/having.test @@ -743,3 +743,18 @@ SELECT * FROM t1 JOIN t2 ON c1 = c2 HAVING c2 > 'a' ORDER BY c2 LIMIT 1; DROP TABLE t1,t2; --echo End of 10.0 tests + +--echo # +--echo # MDEV-10716: Assertion `real_type() != FIELD_ITEM' failed in +--echo # Item_ref::build_equal_items(THD*, COND_EQUAL*, bool, COND_EQUAL**) +--echo # +CREATE TABLE t1 (i INT); +INSERT INTO t1 VALUES (1),(2); +SELECT i, COUNT(*) FROM t1 GROUP BY i HAVING i<>0 AND 1; +SELECT i-1 A, COUNT(*) FROM t1 GROUP BY i HAVING A AND 1; +CREATE VIEW v1 as select i, i-1 as A from t1; +SELECT A, COUNT(*) FROM v1 GROUP BY i HAVING A AND 1; +DROP VIEW v1; +DROP TABLE t1; + +--echo End of 10.1 tests diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index bd1e8b721570a..a222335cf972a 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4507,7 +4507,8 @@ Item_cond::fix_fields(THD *thd, Item **ref) was: become: = 1 */ - if (item->type() == FIELD_ITEM) + Item::Type type= item->type(); + if (type == Item::FIELD_ITEM || type == Item::REF_ITEM) { Query_arena backup, *arena; Item *new_item; From b34c813ae3531dbf851a9ef64ca816c1ccfc116d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 28 Sep 2016 22:12:03 +0200 Subject: [PATCH 088/295] MDEV-10201 SSL tests fail on fedora23 Fixed the remaining failing test case in 10.1 --- mysql-test/r/userstat.result | 14 +++++++------- mysql-test/t/userstat.test | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/userstat.result b/mysql-test/r/userstat.result index f818ab9c0a744..66bb4516c8582 100644 --- a/mysql-test/r/userstat.result +++ b/mysql-test/r/userstat.result @@ -80,9 +80,9 @@ delete from t1 where a=3; select * from t1 where a=999; a b drop table t1; -SHOW STATUS LIKE 'Ssl_cipher'; -Variable_name Value -Ssl_cipher DHE-RSA-AES256-SHA +SELECT (VARIABLE_VALUE <> '') AS have_ssl FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='Ssl_cipher'; +have_ssl +1 create table t1 (a int, primary key (a), b int default 0) engine=innodb; begin; insert into t1 values(1,1); @@ -147,11 +147,11 @@ TOTAL_CONNECTIONS 2 TOTAL_SSL_CONNECTIONS 1 CONCURRENT_CONNECTIONS 0 ROWS_READ 6 -ROWS_SENT 2 +ROWS_SENT 3 ROWS_DELETED 1 ROWS_INSERTED 7 ROWS_UPDATED 5 -SELECT_COMMANDS 3 +SELECT_COMMANDS 4 UPDATE_COMMANDS 11 OTHER_COMMANDS 7 COMMIT_TRANSACTIONS 19 @@ -165,11 +165,11 @@ TOTAL_CONNECTIONS 2 TOTAL_SSL_CONNECTIONS 1 CONCURRENT_CONNECTIONS 0 ROWS_READ 6 -ROWS_SENT 2 +ROWS_SENT 3 ROWS_DELETED 1 ROWS_INSERTED 7 ROWS_UPDATED 5 -SELECT_COMMANDS 3 +SELECT_COMMANDS 4 UPDATE_COMMANDS 11 OTHER_COMMANDS 7 COMMIT_TRANSACTIONS 19 diff --git a/mysql-test/t/userstat.test b/mysql-test/t/userstat.test index cb1250a13eab4..9ce3a32c44222 100644 --- a/mysql-test/t/userstat.test +++ b/mysql-test/t/userstat.test @@ -35,8 +35,7 @@ drop table t1; # test SSL connections --connect (ssl_con,localhost,root,,,,,SSL) ---replace_result DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-SHA -SHOW STATUS LIKE 'Ssl_cipher'; +SELECT (VARIABLE_VALUE <> '') AS have_ssl FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='Ssl_cipher'; --connection default # From 7497ebf8a49bfe30bb4110f2ac20a30f804b7946 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 29 Sep 2016 10:16:24 +0200 Subject: [PATCH 089/295] mysqld_safe: close stdout and stderr when they're not needed anymore. Helps when daemonizing it from mysql.init --- scripts/mysqld_safe.sh | 6 +++++- support-files/mysql.server.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 7f18abb3dc254..7cadce725d1fd 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -620,6 +620,10 @@ else logging=syslog fi +# close stdout and stderr, everything goes to $logging now +exec 1>&- +exec 2>&- + USER_OPTION="" if test -w / -o "$USER" = "root" then @@ -650,7 +654,7 @@ if [ ! -d $mysql_unix_port_dir ] then if ! `mkdir -p $mysql_unix_port_dir` then - echo "Fatal error Can't create database directory '$mysql_unix_port'" + log_error "Fatal error Can't create database directory '$mysql_unix_port'" exit 1 fi chown $user $mysql_unix_port_dir diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index e5c8814f9307f..d4fff33af13d3 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -308,7 +308,7 @@ case "$mode" in then # Give extra arguments to mysqld with the my.cnf file. This script # may be overwritten at next upgrade. - $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null & + $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args & wait_for_ready; return_value=$? # Make lock for RedHat / SuSE From ad2076945062f7f6d29752641048071229435391 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 29 Sep 2016 11:50:13 +0200 Subject: [PATCH 090/295] init plugin psi keys before LOCK_plugin --- sql/sql_plugin.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index d86876a973ba0..5b16482ae6608 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1536,10 +1536,6 @@ int plugin_init(int *argc, char **argv, int flags) dlopen_count =0; -#ifdef HAVE_PSI_INTERFACE - init_plugin_psi_keys(); -#endif - init_alloc_root(&plugin_mem_root, 4096, 4096, MYF(0)); init_alloc_root(&plugin_vars_mem_root, 4096, 4096, MYF(0)); init_alloc_root(&tmp_root, 4096, 4096, MYF(0)); @@ -4233,6 +4229,9 @@ int thd_setspecific(MYSQL_THD thd, MYSQL_THD_KEY_T key, void *value) void plugin_mutex_init() { +#ifdef HAVE_PSI_INTERFACE + init_plugin_psi_keys(); +#endif mysql_mutex_init(key_LOCK_plugin, &LOCK_plugin, MY_MUTEX_INIT_FAST); } From 235876d70e22b7d8c06768b3a4a9ecbf5c9bbc7a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 29 Sep 2016 14:21:14 +0200 Subject: [PATCH 091/295] update test results after merge --- mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff | 4 ++-- mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff | 4 ++-- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff index 930fa18541ff1..cc73fc52a253e 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff @@ -1227,8 +1227,8 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_VERSION SESSION_VALUE NULL --GLOBAL_VALUE 5.6.32 -+GLOBAL_VALUE 5.6.31-77.0 +-GLOBAL_VALUE 5.6.33 ++GLOBAL_VALUE 5.6.32-78.1 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE NULL VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff index aafc0a9e8063b..391e3acefdee0 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff @@ -661,8 +661,8 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_VERSION SESSION_VALUE NULL --GLOBAL_VALUE 5.6.32 -+GLOBAL_VALUE 5.6.31-77.0 +-GLOBAL_VALUE 5.6.33 ++GLOBAL_VALUE 5.6.32-78.1 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE NULL VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 8ef0d44904274..ee596b7f40e0c 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -2359,7 +2359,7 @@ READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_VERSION SESSION_VALUE NULL -GLOBAL_VALUE 5.6.32 +GLOBAL_VALUE 5.6.33 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE NULL VARIABLE_SCOPE GLOBAL From 6925689ca829901567e9503fd4fdce443f9a7d53 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Thu, 29 Sep 2016 14:58:32 -0400 Subject: [PATCH 092/295] MDEV-9312: storage engine not enforced during galera cluster replication Postfix: The pre-initialization of wsrep threads is not needed for mysqldump sst method. --- sql/mysqld.cc | 4 +++- sql/wsrep_mysqld.cc | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f69416c34ea62..466a54b21a8dd 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5316,8 +5316,10 @@ static int init_server_components() initialized. This initialization was not possible before, as plugins (and thus some global system variables) are initialized after wsrep startup threads are created. + Note: This only needs to be done for rsync, xtrabackup based SST methods. */ - wsrep_plugins_post_init(); + if (wsrep_before_SE()) + wsrep_plugins_post_init(); if (WSREP_ON && !opt_bin_log) { diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 5f25ea1ef380e..0deb19dfc7717 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -786,8 +786,12 @@ void wsrep_init_startup (bool first) Pre-initialize global_system_variables.table_plugin with a dummy engine (placeholder) required during the initialization of wsrep threads (THDs). (see: plugin_thdvar_init()) + Note: This only needs to be done for rsync & xtrabackup based SST methods. + In case of mysqldump SST method, the wsrep threads are created after the + server plugins & global system variables are initialized. */ - wsrep_plugins_pre_init(); + if (wsrep_before_SE()) + wsrep_plugins_pre_init(); /* Skip replication start if dummy wsrep provider is loaded */ if (!strcmp(wsrep_provider, WSREP_NONE)) return; From d83fd5fcde3a3feacab9006c63538541bd6e1c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 30 Sep 2016 09:13:39 +0300 Subject: [PATCH 093/295] MDEV-10685: innodb.xa_recovery failed in buildbot Test crashes server intentionally, need to be prepared for crash recovery and database page corruption. --- mysql-test/suite/innodb/r/xa_recovery.result | 1 + mysql-test/suite/innodb/t/xa_recovery.test | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/mysql-test/suite/innodb/r/xa_recovery.result b/mysql-test/suite/innodb/r/xa_recovery.result index 84cb37ef7c9eb..e65f702c4881c 100644 --- a/mysql-test/suite/innodb/r/xa_recovery.result +++ b/mysql-test/suite/innodb/r/xa_recovery.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); CREATE TABLE t1 (a INT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); XA START 'x'; diff --git a/mysql-test/suite/innodb/t/xa_recovery.test b/mysql-test/suite/innodb/t/xa_recovery.test index 2c1034f3c4d1d..32373d63d1432 100644 --- a/mysql-test/suite/innodb/t/xa_recovery.test +++ b/mysql-test/suite/innodb/t/xa_recovery.test @@ -12,6 +12,11 @@ if (`select plugin_auth_version <= "5.6.24" from information_schema.plugins wher FLUSH TABLES; --enable_query_log +# +# We kill server belown with timeout 0 that is not fully safe +# +call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); + CREATE TABLE t1 (a INT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); connect (con1,localhost,root); From f35e918989080d1810c2e1c7fc91f3130585607b Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Fri, 30 Sep 2016 12:11:09 -0400 Subject: [PATCH 094/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d921401f49512..e578c56c4e43d 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=18 +MYSQL_VERSION_PATCH=19 From 9b20d606fb1afd0327356e7c78c2aea774dec3d4 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Wed, 5 Oct 2016 23:44:54 +0200 Subject: [PATCH 095/295] - Fix MDEV-10948. Syntax error on quoted JDBC tables. Was because the quoting character was always '"' instead of being retrieve from the JDBC source. modified: storage/connect/JdbcInterface.java modified: storage/connect/jdbconn.cpp modified: storage/connect/tabjdbc.cpp --- storage/connect/JdbcInterface.java | 12 ++++++++++++ storage/connect/jdbconn.cpp | 15 +++++++++++++++ storage/connect/tabjdbc.cpp | 19 +++++++++++++++---- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index f765052915d72..e339c9891133a 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -340,6 +340,18 @@ public int GetMaxValue(int n) { return m; } // end of GetMaxValue + public String GetQuoteString() { + String qs = null; + + try { + qs = dbmd.getIdentifierQuoteString(); + } catch(SQLException se) { + SetErrmsg(se); + } // end try/catch + + return qs; + } // end of GetQuoteString + public int GetColumns(String[] parms) { int ncol = -1; diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index 952847507a0b3..229ade53ad1cf 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -1011,6 +1011,21 @@ int JDBConn::Open(PJPARM sop) return RC_FX; } // endif Msg + jmethodID qcid = nullptr; + + if (!gmID(g, qcid, "GetQuoteString", "()Ljava/lang/String;")) { + jstring s = (jstring)env->CallObjectMethod(job, qcid); + + if (s != nullptr) { + char *qch = (char*)env->GetStringUTFChars(s, (jboolean)false); + m_IDQuoteChar[0] = *qch; + } else { + s = (jstring)env->CallObjectMethod(job, errid); + Msg = (char*)env->GetStringUTFChars(s, (jboolean)false); + } // endif s + + } // endif qcid + if (gmID(g, typid, "ColumnType", "(ILjava/lang/String;)I")) return RC_FX; else diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index 86fd831b26201..e398523892f2e 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -686,6 +686,9 @@ bool TDBJDBC::MakeInsert(PGLOBAL g) else Prepared = true; + if (trace) + htrc("Insert=%s\n", Query->GetStr()); + return false; } // end of MakeInsert @@ -733,17 +736,18 @@ bool TDBJDBC::MakeCommand(PGLOBAL g) // If so, it must be quoted in the original query strlwr(strcat(strcat(strcpy(name, " "), Name), " ")); - if (!strstr(" update delete low_priority ignore quick from ", name)) - strlwr(strcpy(name, Name)); // Not a keyword - else + if (strstr(" update delete low_priority ignore quick from ", name)) { strlwr(strcat(strcat(strcpy(name, qc), Name), qc)); + k += 2; + } else + strlwr(strcpy(name, Name)); // Not a keyword if ((p = strstr(qrystr, name))) { for (i = 0; i < p - qrystr; i++) stmt[i] = (Qrystr[i] == '`') ? *qc : Qrystr[i]; stmt[i] = 0; - k = i + (int)strlen(Name); + k += i + (int)strlen(Name); if (qtd && *(p-1) == ' ') strcat(strcat(strcat(stmt, qc), TableName), qc); @@ -765,6 +769,9 @@ bool TDBJDBC::MakeCommand(PGLOBAL g) return NULL; } // endif p + if (trace) + htrc("Command=%s\n", stmt); + Query = new(g)STRING(g, 0, stmt); return (!Query->GetSize()); } // end of MakeCommand @@ -1214,6 +1221,10 @@ int TDBJDBC::WriteDB(PGLOBAL g) } // endif oom Query->RepLast(')'); + + if (trace > 1) + htrc("Inserting: %s\n", Query->GetStr()); + rc = Jcp->ExecuteUpdate(Query->GetStr()); Query->Truncate(len); // Restore query From ed4a6f12b3db90de2168273871e7153fb458aee6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 10 Oct 2016 12:49:10 +0000 Subject: [PATCH 096/295] MDEV-10823 amend : Use opt_log_basename instead of hostname to test filesystem case sensitivity. Hostname can include characters, which are invalid for use as filename, thus case sensitivity test will fail to produce meaningful results. --- sql/mysqld.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 466a54b21a8dd..2f0fd46845c7f 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -9887,9 +9887,9 @@ static int test_if_case_insensitive(const char *dir_name) MY_STAT stat_info; DBUG_ENTER("test_if_case_insensitive"); - fn_format(buff, glob_hostname, dir_name, ".lower-test", + fn_format(buff, opt_log_basename, dir_name, ".lower-test", MY_UNPACK_FILENAME | MY_REPLACE_EXT | MY_REPLACE_DIR); - fn_format(buff2, glob_hostname, dir_name, ".LOWER-TEST", + fn_format(buff2, opt_log_basename, dir_name, ".LOWER-TEST", MY_UNPACK_FILENAME | MY_REPLACE_EXT | MY_REPLACE_DIR); mysql_file_delete(key_file_casetest, buff2, MYF(0)); if ((file= mysql_file_create(key_file_casetest, From 6100f59ffaed0a4d4c224aa771999983f8acd496 Mon Sep 17 00:00:00 2001 From: sensssz Date: Tue, 11 Oct 2016 20:52:35 -0400 Subject: [PATCH 097/295] Implement VATS both in InnoDB and XtraDB. Add configuration options for it in both of them. --- storage/innobase/handler/ha_innodb.cc | 29 +++++ storage/innobase/include/lock0lock.h | 9 ++ storage/innobase/lock/lock0lock.cc | 158 +++++++++++++++++++++++--- storage/xtradb/handler/ha_innodb.cc | 30 +++++ storage/xtradb/include/lock0lock.h | 9 ++ storage/xtradb/lock/lock0lock.cc | 155 ++++++++++++++++++++++--- 6 files changed, 362 insertions(+), 28 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7ba54a1c360a6..a436d079d03d7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -300,6 +300,22 @@ static TYPELIB innodb_checksum_algorithm_typelib = { NULL }; +/** Possible values of the parameter innodb_lock_schedule_algorithm */ +static const char* innodb_lock_schedule_algorithm_names[] = { + "fcfs", + "vats", + NullS +}; + +/** Used to define an enumerate type of the system variable +innodb_lock_schedule_algorithm. */ +static TYPELIB innodb_lock_schedule_algorithm_typelib = { + array_elements(innodb_lock_schedule_algorithm_names) - 1, + "innodb_lock_schedule_algorithm_typelib", + innodb_lock_schedule_algorithm_names, + NULL +}; + /* The following counter is used to convey information to InnoDB about server activity: in case of normal DML ops it is not sensible to call srv_active_wake_master_thread after each @@ -19013,6 +19029,18 @@ static MYSQL_SYSVAR_ULONG(doublewrite_batch_size, srv_doublewrite_batch_size, NULL, NULL, 120, 1, 127, 0); #endif /* defined UNIV_DEBUG || defined UNIV_PERF_DEBUG */ +static MYSQL_SYSVAR_ENUM(lock_schedule_algorithm, innodb_lock_schedule_algorithm, + PLUGIN_VAR_RQCMDARG, + "The algorithm Innodb uses for deciding which locks to grant next when" + " a lock is released. Possible values are" + " FCFS" + " grant the locks in First-Come-First-Served order;" + " VATS" + " use the Variance-Aware-Transaction-Scheduling algorithm, which" + " uses an Eldest-Transaction-First heuristic.", + NULL, NULL, INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, + &innodb_lock_schedule_algorithm_typelib); + static MYSQL_SYSVAR_LONG(buffer_pool_instances, innobase_buffer_pool_instances, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Number of buffer pool instances, set to higher value on high-end machines to increase scalability", @@ -19828,6 +19856,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(ft_sort_pll_degree), MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(force_load_corrupted), + MYSQL_SYSVAR(lock_schedule_algorithm), MYSQL_SYSVAR(locks_unsafe_for_binlog), MYSQL_SYSVAR(lock_wait_timeout), #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index a6fafd9575448..2b1158eddf87c 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -43,6 +43,15 @@ Created 5/7/1996 Heikki Tuuri extern ibool lock_print_waits; #endif /* UNIV_DEBUG */ +/** Alternatives for innodb_lock_schedule_algorithm, which can be changed by + setting innodb_lock_schedule_algorithm. */ +enum innodb_lock_schedule_algorithm_t { + INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, /*!< First Come First Served */ + INNODB_LOCK_SCHEDULE_ALGORITHM_VATS /*!< Variance-Aware-Transaction-Scheduling */ +}; + +extern ulong innodb_lock_schedule_algorithm; + /*********************************************************************//** Gets the size of a lock struct. @return size in bytes */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index b4834c05b81fd..37f15c0d3e897 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -76,6 +76,9 @@ bitmap */ #define LOCK_PAGE_BITMAP_MARGIN 64 +/** Lock scheduling algorithm */ +ulong innodb_lock_schedule_algorithm = INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS; + /* An explicit record lock affects both the record and the gap before it. An implicit x-lock does not affect the gap, it only locks the index record from read or update. @@ -1982,6 +1985,72 @@ wsrep_print_wait_locks( } #endif /* WITH_WSREP */ +/*********************************************************************//** +Check if lock1 has higher priority than lock2. +NULL has lowest priority. +If neither of them is wait lock, the first one has higher priority. +If only one of them is a wait lock, it has lower priority. +Otherwise, the one with an older transaction has higher priority. +@returns true if lock1 has higher priority, false otherwise. */ +bool +has_higher_priority( + lock_t *lock1, + lock_t *lock2) +{ + if (lock1 == NULL) { + return false; + } else if (lock2 == NULL) { + return true; + } + if (!lock_get_wait(lock1)) { + return true; + } else if (!lock_get_wait(lock2)) { + return false; + } + return lock1->trx->start_time < lock2->trx->start_time; +} + +/*********************************************************************//** +Insert a lock to the hash list according to the mode (whether it is a wait lock) +and the age of the transaction the it is associated with. +If the lock is not a wait lock, insert it to the head of the hash list. +Otherwise, insert it to the middle of the wait locks according to the age of the +transaciton. +*/ +static +void +lock_rec_insert_by_trx_age( + lock_t *in_lock, /*!< in: lock to be insert */ + bool wait) /*!< in: whether it's a wait lock */ +{ + ulint space; + ulint page_no; + ulint rec_fold; + hash_cell_t cell; + lock_t* node; + lock_t* next; + + space = in_lock->un_member.rec_lock.space; + page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + + node = (lock_t *) cell->node; + // If in_lock is not a wait lock, we insert it to the head of the list. + if (node == NULL || !wait || has_higher_priority(in_lock, node)) { + cell->node = in_lock; + in_lock->hash = node; + return; + } + while (node != NULL && has_higher_priority((lock_t *) node->hash, in_lock)) { + node = (lock_t *) node->hash; + } + next = (lock_t *) node->hash; + node->hash = in_lock; + in_lock->hash = next; +} + /*********************************************************************//** Creates a new record lock and inserts it to the lock queue. Does NOT check for deadlocks or lock compatibility! @@ -2144,13 +2213,19 @@ lock_rec_create( return(lock); } trx_mutex_exit(c_lock->trx); + } else if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { + lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); } else { HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + lock_rec_fold(space, page_no), lock); } #else - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { + lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); + } else { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(space, page_no), lock); + } #endif /* WITH_WSREP */ if (!caller_owns_trx_mutex) { @@ -2822,6 +2897,27 @@ lock_rec_cancel( trx_mutex_exit(lock->trx); } +/*************************************************************//** +Move the lock to the head of the hash list. */ +static +void +lock_rec_move_to_front( + lock_t *lock_to_move, /*!< in: lock to be moved */ + ulint rec_fold) /*!< in: rec fold of the lock */ +{ + if (lock_to_move != NULL) + { + // Move the target lock to the head of the list + hash_cell_t* cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + if (lock_to_move != cell->node) { + lock_t *next = (lock_t *) cell->node; + cell->node = lock_to_move; + lock_to_move->hash = next; + } + } +} + /*************************************************************//** Removes a record lock request, waiting or granted, from the queue and grants locks to other transactions in the queue if they now are entitled @@ -2839,7 +2935,9 @@ lock_rec_dequeue_from_page( { ulint space; ulint page_no; + ulint rec_fold lock_t* lock; + lock_t* previous = NULL; trx_lock_t* trx_lock; ut_ad(lock_mutex_own()); @@ -2850,6 +2948,7 @@ lock_rec_dequeue_from_page( space = in_lock->un_member.rec_lock.space; page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); in_lock->index->table->n_rec_locks--; @@ -2861,20 +2960,51 @@ lock_rec_dequeue_from_page( MONITOR_INC(MONITOR_RECLOCK_REMOVED); MONITOR_DEC(MONITOR_NUM_RECLOCK); - /* Check if waiting locks in the queue can now be granted: grant - locks if there are no conflicting locks ahead. Stop at the first - X lock that is waiting or has been granted. */ + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { + /* Check if waiting locks in the queue can now be granted: grant + locks if there are no conflicting locks ahead. Stop at the first + X lock that is waiting or has been granted. */ - for (lock = lock_rec_get_first_on_page_addr(space, page_no); - lock != NULL; - lock = lock_rec_get_next_on_page(lock)) { + for (lock = lock_rec_get_first_on_page_addr(space, page_no); + lock != NULL; + lock = lock_rec_get_next_on_page(lock)) { - if (lock_get_wait(lock) - && !lock_rec_has_to_wait_in_queue(lock)) { + if (lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { - /* Grant the lock */ - ut_ad(lock->trx != in_lock->trx); - lock_grant(lock); + /* Grant the lock */ + ut_ad(lock->trx != in_lock->trx); + lock_grant(lock); + } + } + } else { + /* Grant locks if there are no conflicting locks ahead. + Move granted locks to the head of the list. */ + for (lock = lock_rec_get_first_on_page_addr(space, page_no); + lock != NULL;) { + + /* If the lock is a wait lock on this page, and it does not need to wait. */ + if ((lock->un_member.rec_lock.space == space) + && (lock->un_member.rec_lock.page_no == page_no) + && lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + lock_grant(lock); + + if (previous != NULL) { + /* Move the lock to the head of the list. */ + HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); + lock_rec_move_to_front(lock, rec_fold); + } else { + /* Already at the head of the list. */ + previous = lock; + } + /* Move on to the next lock. */ + lock = static_cast(HASH_GET_NEXT(hash, previous)); + } else { + previous = lock; + lock = static_cast(HASH_GET_NEXT(hash, lock)); + } } } } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 320b900d019f6..7ce65e5bc6fc6 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -348,6 +348,23 @@ static TYPELIB innodb_empty_free_list_algorithm_typelib = { NULL }; +/** Possible values of the parameter innodb_lock_schedule_algorithm */ +static const char* innodb_lock_schedule_algorithm_names[] = { + "fcfs", + "vats", + NullS +}; + +/** Used to define an enumerate type of the system variable +innodb_lock_schedule_algorithm. */ +static TYPELIB innodb_lock_schedule_algorithm_typelib = { + array_elements(innodb_lock_schedule_algorithm_names) - 1, + "innodb_lock_schedule_algorithm_typelib", + innodb_lock_schedule_algorithm_names, + NULL +}; + + /* The following counter is used to convey information to InnoDB about server activity: in case of normal DML ops it is not sensible to call srv_active_wake_master_thread after each @@ -20473,6 +20490,18 @@ static MYSQL_SYSVAR_ENUM(empty_free_list_algorithm, innodb_srv_empty_free_list_algorithm_validate, NULL, SRV_EMPTY_FREE_LIST_BACKOFF, &innodb_empty_free_list_algorithm_typelib); +static MYSQL_SYSVAR_ENUM(lock_schedule_algorithm, innodb_lock_schedule_algorithm, + PLUGIN_VAR_RQCMDARG, + "The algorithm Innodb uses for deciding which locks to grant next when" + " a lock is released. Possible values are" + " FCFS" + " grant the locks in First-Come-First-Served order;" + " VATS" + " use the Variance-Aware-Transaction-Scheduling algorithm, which" + " uses an Eldest-Transaction-First heuristic.", + NULL, NULL, INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, + &innodb_lock_schedule_algorithm_typelib); + static MYSQL_SYSVAR_LONG(buffer_pool_instances, innobase_buffer_pool_instances, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Number of buffer pool instances, set to higher value on high-end machines to increase scalability", @@ -21366,6 +21395,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(ft_sort_pll_degree), MYSQL_SYSVAR(large_prefix), MYSQL_SYSVAR(force_load_corrupted), + MYSQL_SYSVAR(lock_schedule_algorithm), MYSQL_SYSVAR(locks_unsafe_for_binlog), MYSQL_SYSVAR(lock_wait_timeout), #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/xtradb/include/lock0lock.h b/storage/xtradb/include/lock0lock.h index b6100d470ccf7..742da74f1b634 100644 --- a/storage/xtradb/include/lock0lock.h +++ b/storage/xtradb/include/lock0lock.h @@ -45,6 +45,15 @@ Created 5/7/1996 Heikki Tuuri extern ibool lock_print_waits; #endif /* UNIV_DEBUG */ +/** Alternatives for innodb_lock_schedule_algorithm, which can be changed by + setting innodb_lock_schedule_algorithm. */ +enum innodb_lock_schedule_algorithm_t { + INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, /*!< First Come First Served */ + INNODB_LOCK_SCHEDULE_ALGORITHM_VATS /*!< Variance-Aware-Transaction-Scheduling */ +}; + +extern ulong innodb_lock_schedule_algorithm; + extern ulint srv_n_lock_deadlock_count; /*********************************************************************//** diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 8650cdd106a95..eb47ef6e685b0 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -76,6 +76,9 @@ bitmap */ #define LOCK_PAGE_BITMAP_MARGIN 64 +/** Lock scheduling algorithm */ +ulong innodb_lock_schedule_algorithm = INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS; + /* An explicit record lock affects both the record and the gap before it. An implicit x-lock does not affect the gap, it only locks the index record from read or update. @@ -2005,6 +2008,72 @@ wsrep_print_wait_locks( } #endif /* WITH_WSREP */ +/*********************************************************************//** +Check if lock1 has higher priority than lock2. +NULL has lowest priority. +If neither of them is wait lock, the first one has higher priority. +If only one of them is a wait lock, it has lower priority. +Otherwise, the one with an older transaction has higher priority. +@returns true if lock1 has higher priority, false otherwise. */ +bool +has_higher_priority( + lock_t *lock1, + lock_t *lock2) +{ + if (lock1 == NULL) { + return false; + } else if (lock2 == NULL) { + return true; + } + if (!lock_get_wait(lock1)) { + return true; + } else if (!lock_get_wait(lock2)) { + return false; + } + return lock1->trx->start_time < lock2->trx->start_time; +} + +/*********************************************************************//** +Insert a lock to the hash list according to the mode (whether it is a wait lock) +and the age of the transaction the it is associated with. +If the lock is not a wait lock, insert it to the head of the hash list. +Otherwise, insert it to the middle of the wait locks according to the age of the +transaciton. +*/ +static +void +lock_rec_insert_by_trx_age( + lock_t *in_lock, /*!< in: lock to be insert */ + bool wait) /*!< in: whether it's a wait lock */ +{ + ulint space; + ulint page_no; + ulint rec_fold; + hash_cell_t cell; + lock_t* node; + lock_t* next; + + space = in_lock->un_member.rec_lock.space; + page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + + node = (lock_t *) cell->node; + // If in_lock is not a wait lock, we insert it to the head of the list. + if (node == NULL || !wait || has_higher_priority(in_lock, node)) { + cell->node = in_lock; + in_lock->hash = node; + return; + } + while (node != NULL && has_higher_priority((lock_t *) node->hash, in_lock)) { + node = (lock_t *) node->hash; + } + next = (lock_t *) node->hash; + node->hash = in_lock; + in_lock->hash = next; +} + /*********************************************************************//** Creates a new record lock and inserts it to the lock queue. Does NOT check for deadlocks or lock compatibility! @@ -2167,13 +2236,19 @@ lock_rec_create( return(lock); } trx_mutex_exit(c_lock->trx); + } else if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { + lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); } else { HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + lock_rec_fold(space, page_no), lock); } #else - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { + lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); + } else { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(space, page_no), lock); + } #endif /* WITH_WSREP */ lock_sys->rec_num++; @@ -2858,6 +2933,27 @@ lock_rec_cancel( trx_mutex_exit(lock->trx); } +/*************************************************************//** +Move the lock to the head of the hash list. */ +static +void +lock_rec_move_to_front( + lock_t *lock_to_move, /*!< in: lock to be moved */ + ulint rec_fold) /*!< in: rec fold of the lock */ +{ + if (lock_to_move != NULL) + { + // Move the target lock to the head of the list + hash_cell_t* cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + if (lock_to_move != cell->node) { + lock_t *next = (lock_t *) cell->node; + cell->node = lock_to_move; + lock_to_move->hash = next; + } + } +} + /*************************************************************//** Removes a record lock request, waiting or granted, from the queue and grants locks to other transactions in the queue if they now are entitled @@ -2898,20 +2994,51 @@ lock_rec_dequeue_from_page( MONITOR_INC(MONITOR_RECLOCK_REMOVED); MONITOR_DEC(MONITOR_NUM_RECLOCK); - /* Check if waiting locks in the queue can now be granted: grant - locks if there are no conflicting locks ahead. Stop at the first - X lock that is waiting or has been granted. */ + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { + /* Check if waiting locks in the queue can now be granted: grant + locks if there are no conflicting locks ahead. Stop at the first + X lock that is waiting or has been granted. */ - for (lock = lock_rec_get_first_on_page_addr(space, page_no); - lock != NULL; - lock = lock_rec_get_next_on_page(lock)) { + for (lock = lock_rec_get_first_on_page_addr(space, page_no); + lock != NULL; + lock = lock_rec_get_next_on_page(lock)) { - if (lock_get_wait(lock) - && !lock_rec_has_to_wait_in_queue(lock)) { + if (lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { - /* Grant the lock */ - ut_ad(lock->trx != in_lock->trx); - lock_grant(lock); + /* Grant the lock */ + ut_ad(lock->trx != in_lock->trx); + lock_grant(lock); + } + } + } else { + /* Grant locks if there are no conflicting locks ahead. + Move granted locks to the head of the list. */ + for (lock = lock_rec_get_first_on_page_addr(space, page_no); + lock != NULL;) { + + /* If the lock is a wait lock on this page, and it does not need to wait. */ + if ((lock->un_member.rec_lock.space == space) + && (lock->un_member.rec_lock.page_no == page_no) + && lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + lock_grant(lock); + + if (previous != NULL) { + /* Move the lock to the head of the list. */ + HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); + lock_rec_move_to_front(lock, rec_fold); + } else { + /* Already at the head of the list. */ + previous = lock; + } + /* Move on to the next lock. */ + lock = static_cast(HASH_GET_NEXT(hash, previous)); + } else { + previous = lock; + lock = static_cast(HASH_GET_NEXT(hash, lock)); + } } } } From e93d44f2d75f425b0a8bfa2fe4309b93d51e1b33 Mon Sep 17 00:00:00 2001 From: sensssz Date: Tue, 11 Oct 2016 23:02:26 -0400 Subject: [PATCH 098/295] Bug fix: add undeclared variables. --- storage/xtradb/lock/lock0lock.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index eb47ef6e685b0..fbaa09edf040a 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -2971,7 +2971,9 @@ lock_rec_dequeue_from_page( { ulint space; ulint page_no; + ulint rec_fold lock_t* lock; + lock_t* previous = NULL; trx_lock_t* trx_lock; ut_ad(lock_mutex_own()); @@ -2982,6 +2984,7 @@ lock_rec_dequeue_from_page( space = in_lock->un_member.rec_lock.space; page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); in_lock->index->table->n_rec_locks--; From 288796f9272e5b714b16c9a0c3df88829c33ea71 Mon Sep 17 00:00:00 2001 From: sensssz Date: Tue, 11 Oct 2016 23:05:02 -0400 Subject: [PATCH 099/295] Bug fix: missing * and ; --- storage/xtradb/lock/lock0lock.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index fbaa09edf040a..c02daabe03a67 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -2049,7 +2049,7 @@ lock_rec_insert_by_trx_age( ulint space; ulint page_no; ulint rec_fold; - hash_cell_t cell; + hash_cell_t* cell; lock_t* node; lock_t* next; @@ -2971,7 +2971,7 @@ lock_rec_dequeue_from_page( { ulint space; ulint page_no; - ulint rec_fold + ulint rec_fold; lock_t* lock; lock_t* previous = NULL; trx_lock_t* trx_lock; From 55d2bff882a60211a05bc368102e6d58835c2e67 Mon Sep 17 00:00:00 2001 From: sensssz Date: Tue, 11 Oct 2016 23:27:03 -0400 Subject: [PATCH 100/295] Bug fix: add * and ; for innodb --- storage/innobase/lock/lock0lock.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 37f15c0d3e897..e100ea40a66a1 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -2026,7 +2026,7 @@ lock_rec_insert_by_trx_age( ulint space; ulint page_no; ulint rec_fold; - hash_cell_t cell; + hash_cell_t* cell; lock_t* node; lock_t* next; @@ -2935,7 +2935,7 @@ lock_rec_dequeue_from_page( { ulint space; ulint page_no; - ulint rec_fold + ulint rec_fold; lock_t* lock; lock_t* previous = NULL; trx_lock_t* trx_lock; From 5dc7ad87b8302afbe91824121e7eca6b43ae9256 Mon Sep 17 00:00:00 2001 From: sensssz Date: Wed, 12 Oct 2016 21:52:14 -0400 Subject: [PATCH 101/295] Reduce conflict during in-order replication. --- storage/innobase/lock/lock0lock.cc | 15 +++++++++++++++ storage/xtradb/lock/lock0lock.cc | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index e100ea40a66a1..0ba217511f3d5 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -388,6 +388,9 @@ extern "C" int thd_need_wait_for(const MYSQL_THD thd); extern "C" int thd_need_ordering_with(const MYSQL_THD thd, const MYSQL_THD other_thd); +extern "C" +int thd_deadlock_victim_preference(const MYSQL_THD thd1, const MYSQL_THD thd2); + /** Stack to use during DFS search. Currently only a single stack is required because there is no parallel deadlock check. This stack is protected by the lock_sys_t::mutex. */ @@ -1988,6 +1991,8 @@ wsrep_print_wait_locks( /*********************************************************************//** Check if lock1 has higher priority than lock2. NULL has lowest priority. +Respect the preference of the upper server layer to reduce conflict +during in-order parallel replication. If neither of them is wait lock, the first one has higher priority. If only one of them is a wait lock, it has lower priority. Otherwise, the one with an older transaction has higher priority. @@ -2002,6 +2007,16 @@ has_higher_priority( } else if (lock2 == NULL) { return true; } + // Ask the upper server layer if any of the two trx should be prefered. + int preference = thd_deadlock_victim_preference(lock1->thd, lock2->thd); + if (preference == -1) { + // lock1 is preferred as a victim, so lock2 has higher priority + return false; + } else if (preference == 1) { + // lock2 is preferred as a victim, so lock1 has higher priority + return true; + } + // No preference. Compre them by wait mode and trx age. if (!lock_get_wait(lock1)) { return true; } else if (!lock_get_wait(lock2)) { diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index c02daabe03a67..d98d09d34e499 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -388,6 +388,9 @@ extern "C" int thd_need_wait_for(const MYSQL_THD thd); extern "C" int thd_need_ordering_with(const MYSQL_THD thd, const MYSQL_THD other_thd); +extern "C" +int thd_deadlock_victim_preference(const MYSQL_THD thd1, const MYSQL_THD thd2); + /** Stack to use during DFS search. Currently only a single stack is required because there is no parallel deadlock check. This stack is protected by the lock_sys_t::mutex. */ @@ -2011,6 +2014,8 @@ wsrep_print_wait_locks( /*********************************************************************//** Check if lock1 has higher priority than lock2. NULL has lowest priority. +Respect the preference of the upper server layer to reduce conflict +during in-order parallel replication. If neither of them is wait lock, the first one has higher priority. If only one of them is a wait lock, it has lower priority. Otherwise, the one with an older transaction has higher priority. @@ -2025,6 +2030,16 @@ has_higher_priority( } else if (lock2 == NULL) { return true; } + // Ask the upper server layer if any of the two trx should be prefered. + int preference = thd_deadlock_victim_preference(lock1->thd, lock2->thd); + if (preference == -1) { + // lock1 is preferred as a victim, so lock2 has higher priority + return false; + } else if (preference == 1) { + // lock2 is preferred as a victim, so lock1 has higher priority + return true; + } + // No preference. Compre them by wait mode and trx age. if (!lock_get_wait(lock1)) { return true; } else if (!lock_get_wait(lock2)) { From 0a769b00b5bfa5384a69f1f0d526086d3943ba03 Mon Sep 17 00:00:00 2001 From: sensssz Date: Wed, 12 Oct 2016 21:54:31 -0400 Subject: [PATCH 102/295] Get thd by lock->trx->mysql_thd. --- storage/innobase/lock/lock0lock.cc | 2 +- storage/xtradb/lock/lock0lock.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 0ba217511f3d5..bdd85fb0e35cd 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -2008,7 +2008,7 @@ has_higher_priority( return true; } // Ask the upper server layer if any of the two trx should be prefered. - int preference = thd_deadlock_victim_preference(lock1->thd, lock2->thd); + int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); if (preference == -1) { // lock1 is preferred as a victim, so lock2 has higher priority return false; diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index d98d09d34e499..abc0cd32d58ce 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -2031,7 +2031,7 @@ has_higher_priority( return true; } // Ask the upper server layer if any of the two trx should be prefered. - int preference = thd_deadlock_victim_preference(lock1->thd, lock2->thd); + int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); if (preference == -1) { // lock1 is preferred as a victim, so lock2 has higher priority return false; From 183c02839f032e1d9057fd4e278806c26b016826 Mon Sep 17 00:00:00 2001 From: sensssz Date: Thu, 13 Oct 2016 01:23:21 -0400 Subject: [PATCH 103/295] Move the lock after deadlock is resolved. --- storage/innobase/lock/lock0lock.cc | 26 +++++++++++++++----------- storage/xtradb/lock/lock0lock.cc | 26 +++++++++++++++----------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index bdd85fb0e35cd..572dc0e1dc429 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -2228,19 +2228,13 @@ lock_rec_create( return(lock); } trx_mutex_exit(c_lock->trx); - } else if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { - lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); } else { HASH_INSERT(lock_t, hash, lock_sys->rec_hash, lock_rec_fold(space, page_no), lock); } #else - if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { - lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); - } else { - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); - } + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(space, page_no), lock); #endif /* WITH_WSREP */ if (!caller_owns_trx_mutex) { @@ -2371,6 +2365,13 @@ lock_rec_enqueue_waiting( return(DB_SUCCESS_LOCKED_REC); } + // Move it only when it does not cause a deadlock. + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { + HASH_DELETE(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(buf_block_get_space(block), buf_block_get_page_no(block)), lock); + lock_rec_insert_by_trx_age(lock, true); + } + trx->lock.que_state = TRX_QUE_LOCK_WAIT; trx->lock.was_chosen_as_deadlock_victim = FALSE; @@ -4225,7 +4226,8 @@ lock_get_first_lock( } ut_a(lock != NULL); - ut_a(lock != ctx->wait_lock); + ut_a(lock != ctx->wait_lock || + innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS); ut_ad(lock_get_type_low(lock) == lock_get_type_low(ctx->wait_lock)); return(lock); @@ -6432,8 +6434,10 @@ lock_rec_queue_validate( mode, 0, 0, block, heap_no, lock->trx)); #endif /* WITH_WSREP */ - } else if (lock_get_wait(lock) && !lock_rec_get_gap(lock)) { - + } else if (lock_get_wait(lock) && !lock_rec_get_gap(lock) + && innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { + // If using VATS, it's possible that a wait lock is inserted to a place in the list + // such that it does not need to wait. ut_a(lock_rec_has_to_wait_in_queue(lock)); } } diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index abc0cd32d58ce..2091f926153f8 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -2251,19 +2251,13 @@ lock_rec_create( return(lock); } trx_mutex_exit(c_lock->trx); - } else if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { - lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); } else { HASH_INSERT(lock_t, hash, lock_sys->rec_hash, lock_rec_fold(space, page_no), lock); } #else - if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { - lock_rec_insert_by_trx_age(lock, type_mode & LOCK_WAIT); - } else { - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); - } + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(space, page_no), lock); #endif /* WITH_WSREP */ lock_sys->rec_num++; @@ -2399,6 +2393,13 @@ lock_rec_enqueue_waiting( return(DB_SUCCESS_LOCKED_REC); } + // Move it only when it does not cause a deadlock. + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { + HASH_DELETE(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(buf_block_get_space(block), buf_block_get_page_no(block)), lock); + lock_rec_insert_by_trx_age(lock, true); + } + trx->lock.que_state = TRX_QUE_LOCK_WAIT; trx->lock.was_chosen_as_deadlock_victim = FALSE; @@ -4263,7 +4264,8 @@ lock_get_first_lock( } ut_a(lock != NULL); - ut_a(lock != ctx->wait_lock); + ut_a(lock != ctx->wait_lock || + innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS); ut_ad(lock_get_type_low(lock) == lock_get_type_low(ctx->wait_lock)); return(lock); @@ -6495,8 +6497,10 @@ lock_rec_queue_validate( mode, 0, 0, block, heap_no, lock->trx->id)); #endif /* WITH_WSREP */ - } else if (lock_get_wait(lock) && !lock_rec_get_gap(lock)) { - + } else if (lock_get_wait(lock) && !lock_rec_get_gap(lock) + && innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { + // If using VATS, it's possible that a wait lock is inserted to a place in the list + // such that it does not need to wait. ut_a(lock_rec_has_to_wait_in_queue(lock)); } } From 6010a27c87785643f8880d19c0dced3b724c54da Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 13 Oct 2016 12:23:16 +0200 Subject: [PATCH 104/295] 5.5.52-38.3 --- storage/xtradb/btr/btr0btr.c | 4 +- storage/xtradb/handler/ha_innodb.cc | 108 +++++++++++++++++++++++----- storage/xtradb/include/buf0buf.h | 12 ++++ storage/xtradb/include/buf0buf.ic | 14 ++++ storage/xtradb/include/srv0srv.h | 8 +++ storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0log.c | 8 ++- storage/xtradb/log/log0online.c | 12 ++-- storage/xtradb/log/log0recv.c | 2 +- storage/xtradb/mach/mach0data.c | 13 +++- storage/xtradb/srv/srv0srv.c | 42 +++++++---- 11 files changed, 179 insertions(+), 46 deletions(-) diff --git a/storage/xtradb/btr/btr0btr.c b/storage/xtradb/btr/btr0btr.c index dec42f27d3b99..0c4293637897f 100644 --- a/storage/xtradb/btr/btr0btr.c +++ b/storage/xtradb/btr/btr0btr.c @@ -76,7 +76,7 @@ btr_corruption_report( buf_block_get_zip_size(block), BUF_PAGE_PRINT_NO_CRASH); } - buf_page_print(buf_block_get_frame(block), 0, 0); + buf_page_print(buf_nonnull_block_get_frame(block), 0, 0); } #ifndef UNIV_HOTBACKUP @@ -1077,7 +1077,7 @@ btr_get_size( SRV_CORRUPT_TABLE_CHECK(root, { mtr_commit(mtr); - return(0); + return(ULINT_UNDEFINED); }); if (flag == BTR_N_LEAF_PAGES) { diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 896b27a204785..010aec1ea0db5 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -475,6 +475,19 @@ innobase_is_fake_change( THD* thd); /*!< in: MySQL thread handle of the user for whom the transaction is being committed */ +/** Get the list of foreign keys referencing a specified table +table. +@param thd The thread handle +@param path Path to the table +@param f_key_list[out] The list of foreign keys + +@return error code or zero for success */ +static +int +innobase_get_parent_fk_list( + THD* thd, + const char* path, + List* f_key_list); /******************************************************************//** Maps a MySQL trx isolation level code to the InnoDB isolation level code @@ -2710,6 +2723,7 @@ innobase_init( innobase_hton->purge_changed_page_bitmaps = innobase_purge_changed_page_bitmaps; innobase_hton->is_fake_change = innobase_is_fake_change; + innobase_hton->get_parent_fk_list = innobase_get_parent_fk_list; ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); @@ -9721,7 +9735,14 @@ ha_innobase::check( prebuilt->select_lock_type = LOCK_NONE; - if (!row_check_index_for_mysql(prebuilt, index, &n_rows)) { + bool check_result + = row_check_index_for_mysql(prebuilt, index, &n_rows); + DBUG_EXECUTE_IF( + "dict_set_index_corrupted", + if (!(index->type & DICT_CLUSTERED)) { + check_result = false; + }); + if (!check_result) { innobase_format_name( index_name, sizeof index_name, index->name, TRUE); @@ -10057,6 +10078,73 @@ get_foreign_key_info( return(pf_key_info); } +/** Get the list of foreign keys referencing a specified table +table. +@param thd The thread handle +@param path Path to the table +@param f_key_list[out] The list of foreign keys */ +static +void +fill_foreign_key_list(THD* thd, + const dict_table_t* table, + List* f_key_list) +{ + ut_ad(mutex_own(&dict_sys->mutex)); + + for (dict_foreign_t* foreign + = UT_LIST_GET_FIRST(table->referenced_list); + foreign != NULL; + foreign = UT_LIST_GET_NEXT(referenced_list, foreign)) { + + FOREIGN_KEY_INFO* pf_key_info + = get_foreign_key_info(thd, foreign); + if (pf_key_info) { + f_key_list->push_back(pf_key_info); + } + } +} + +/** Get the list of foreign keys referencing a specified table +table. +@param thd The thread handle +@param path Path to the table +@param f_key_list[out] The list of foreign keys + +@return error code or zero for success */ +static +int +innobase_get_parent_fk_list( + THD* thd, + const char* path, + List* f_key_list) +{ + ut_a(strlen(path) <= FN_REFLEN); + char norm_name[FN_REFLEN + 1]; + normalize_table_name(norm_name, path); + + trx_t* parent_trx = check_trx_exists(thd); + parent_trx->op_info = "getting list of referencing foreign keys"; + trx_search_latch_release_if_reserved(parent_trx); + + mutex_enter(&dict_sys->mutex); + + dict_table_t* table + = dict_table_get_low(norm_name, + static_cast( + DICT_ERR_IGNORE_INDEX_ROOT + | DICT_ERR_IGNORE_CORRUPT)); + if (!table) { + mutex_exit(&dict_sys->mutex); + return(HA_ERR_NO_SUCH_TABLE); + } + + fill_foreign_key_list(thd, table, f_key_list); + + mutex_exit(&dict_sys->mutex); + parent_trx->op_info = ""; + return(0); +} + /*******************************************************************//** Gets the list of foreign keys in this table. @return always 0, that is, always succeeds */ @@ -10105,9 +10193,6 @@ ha_innobase::get_parent_foreign_key_list( THD* thd, /*!< in: user thread handle */ List* f_key_list) /*!< out: foreign key list */ { - FOREIGN_KEY_INFO* pf_key_info; - dict_foreign_t* foreign; - ut_a(prebuilt != NULL); update_thd(ha_thd()); @@ -10116,16 +10201,7 @@ ha_innobase::get_parent_foreign_key_list( trx_search_latch_release_if_reserved(prebuilt->trx); mutex_enter(&(dict_sys->mutex)); - - for (foreign = UT_LIST_GET_FIRST(prebuilt->table->referenced_list); - foreign != NULL; - foreign = UT_LIST_GET_NEXT(referenced_list, foreign)) { - pf_key_info = get_foreign_key_info(thd, foreign); - if (pf_key_info) { - f_key_list->push_back(pf_key_info); - } - } - + fill_foreign_key_list(thd, prebuilt->table, f_key_list); mutex_exit(&(dict_sys->mutex)); prebuilt->trx->op_info = ""; @@ -12539,7 +12615,6 @@ innodb_track_changed_pages_validate( for update function */ struct st_mysql_value* value) /*!< in: incoming bool */ { - static bool enabled_on_startup = false; long long intbuf = 0; if (value->val_int(value, &intbuf)) { @@ -12547,8 +12622,7 @@ innodb_track_changed_pages_validate( return 1; } - if (srv_track_changed_pages || enabled_on_startup) { - enabled_on_startup = true; + if (srv_redo_log_thread_started) { *reinterpret_cast(save) = static_cast(intbuf); return 0; diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 77025c1637396..23692c92c09ca 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1110,8 +1110,20 @@ buf_block_get_frame( /*================*/ const buf_block_t* block) /*!< in: pointer to the control block */ __attribute__((pure)); + +/*********************************************************************//** +Gets a pointer to the memory frame of a block, where block is known not to be +NULL. +@return pointer to the frame */ +UNIV_INLINE +buf_frame_t* +buf_nonnull_block_get_frame( + const buf_block_t* block) /*!< in: pointer to the control block */ + __attribute__((pure)); + #else /* UNIV_DEBUG */ # define buf_block_get_frame(block) (block ? (block)->frame : 0) +# define buf_nonnull_block_get_frame(block) ((block)->frame) #endif /* UNIV_DEBUG */ /*********************************************************************//** Gets the space id of a block. diff --git a/storage/xtradb/include/buf0buf.ic b/storage/xtradb/include/buf0buf.ic index f214112c7ce96..fae44a1ac4a2b 100644 --- a/storage/xtradb/include/buf0buf.ic +++ b/storage/xtradb/include/buf0buf.ic @@ -718,6 +718,19 @@ buf_block_get_frame( { SRV_CORRUPT_TABLE_CHECK(block, return(0);); + return(buf_nonnull_block_get_frame(block)); +} + +/*********************************************************************//** +Gets a pointer to the memory frame of a block, where block is known not to be +NULL. +@return pointer to the frame */ +UNIV_INLINE +buf_frame_t* +buf_nonnull_block_get_frame( +/*========================*/ + const buf_block_t* block) /*!< in: pointer to the control block */ +{ switch (buf_block_get_state(block)) { case BUF_BLOCK_ZIP_FREE: case BUF_BLOCK_ZIP_PAGE: @@ -739,6 +752,7 @@ buf_block_get_frame( ok: return((buf_frame_t*) block->frame); } + #endif /* UNIV_DEBUG */ /*********************************************************************//** diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 3ccad0640b663..12ab9d9ed876f 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -74,6 +74,11 @@ extern os_event_t srv_checkpoint_completed_event; that the (slow) shutdown may proceed */ extern os_event_t srv_redo_log_thread_finished_event; +/** Whether the redo log tracker thread has been started. Does not take into +account whether the tracking is currently enabled (see srv_track_changed_pages +for that) */ +extern my_bool srv_redo_log_thread_started; + /* If the last data file is auto-extended, we add this many pages to it at a time */ #define SRV_AUTO_EXTEND_INCREMENT \ @@ -141,6 +146,9 @@ extern char* srv_doublewrite_file; extern ibool srv_recovery_stats; +/** Whether the redo log tracking is currently enabled. Note that it is +possible for the log tracker thread to be running and the tracking to be +disabled */ extern my_bool srv_track_changed_pages; extern ib_uint64_t srv_max_bitmap_file_size; diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index b158a12027f9c..cc589166f8de7 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -64,7 +64,7 @@ component, i.e. we show M.N.P as M.N */ (INNODB_VERSION_MAJOR << 8 | INNODB_VERSION_MINOR) #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 38.0 +#define PERCONA_INNODB_VERSION 38.3 #endif #define INNODB_VERSION_STR MYSQL_SERVER_VERSION diff --git a/storage/xtradb/log/log0log.c b/storage/xtradb/log/log0log.c index b4b48a065f985..49ee8407b2c09 100644 --- a/storage/xtradb/log/log0log.c +++ b/storage/xtradb/log/log0log.c @@ -3326,6 +3326,8 @@ logs_empty_and_mark_files_at_shutdown(void) algorithm only works if the server is idle at shutdown */ srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; + + srv_wake_purge_thread(); loop: os_thread_sleep(100000); @@ -3499,7 +3501,7 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; /* Wake the log tracking thread which will then immediatelly quit because of srv_shutdown_state value */ - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { os_event_set(srv_checkpoint_completed_event); os_event_wait(srv_redo_log_thread_finished_event); } @@ -3576,7 +3578,7 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; /* Signal the log following thread to quit */ - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { os_event_set(srv_checkpoint_completed_event); } @@ -3600,7 +3602,7 @@ logs_empty_and_mark_files_at_shutdown(void) fil_flush_file_spaces(FIL_TABLESPACE); - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { os_event_wait(srv_redo_log_thread_finished_event); } diff --git a/storage/xtradb/log/log0online.c b/storage/xtradb/log/log0online.c index d0127488f6791..fa2c8b882bfd6 100644 --- a/storage/xtradb/log/log0online.c +++ b/storage/xtradb/log/log0online.c @@ -1813,7 +1813,7 @@ log_online_purge_changed_page_bitmaps( lsn = IB_ULONGLONG_MAX; } - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { /* User requests might happen with both enabled and disabled tracking */ mutex_enter(&log_bmp_sys->mutex); @@ -1821,13 +1821,13 @@ log_online_purge_changed_page_bitmaps( if (!log_online_setup_bitmap_file_range(&bitmap_files, 0, IB_ULONGLONG_MAX)) { - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { mutex_exit(&log_bmp_sys->mutex); } return TRUE; } - if (srv_track_changed_pages && lsn > log_bmp_sys->end_lsn) { + if (srv_redo_log_thread_started && lsn > log_bmp_sys->end_lsn) { /* If we have to delete the current output file, close it first. */ os_file_close(log_bmp_sys->out.file); @@ -1858,7 +1858,7 @@ log_online_purge_changed_page_bitmaps( } } - if (srv_track_changed_pages) { + if (srv_redo_log_thread_started) { if (lsn > log_bmp_sys->end_lsn) { ib_uint64_t new_file_lsn; if (lsn == IB_ULONGLONG_MAX) { @@ -1869,9 +1869,7 @@ log_online_purge_changed_page_bitmaps( new_file_lsn = log_bmp_sys->end_lsn; } if (!log_online_rotate_bitmap_file(new_file_lsn)) { - /* If file create failed, signal the log - tracking thread to quit next time it wakes - up. */ + /* If file create failed, stop log tracking */ srv_track_changed_pages = FALSE; } } diff --git a/storage/xtradb/log/log0recv.c b/storage/xtradb/log/log0recv.c index 6c2a121967e6f..527e7b3af0f6e 100644 --- a/storage/xtradb/log/log0recv.c +++ b/storage/xtradb/log/log0recv.c @@ -3015,7 +3015,7 @@ recv_recovery_from_checkpoint_start_func( ib_uint64_t checkpoint_lsn; ib_uint64_t checkpoint_no; ib_uint64_t old_scanned_lsn; - ib_uint64_t group_scanned_lsn; + ib_uint64_t group_scanned_lsn = 0; ib_uint64_t contiguous_lsn; #ifdef UNIV_LOG_ARCHIVE ib_uint64_t archived_lsn; diff --git a/storage/xtradb/mach/mach0data.c b/storage/xtradb/mach/mach0data.c index 95b135b09541e..00378f036c957 100644 --- a/storage/xtradb/mach/mach0data.c +++ b/storage/xtradb/mach/mach0data.c @@ -56,7 +56,18 @@ mach_parse_compressed( *val = flag; return(ptr + 1); - } else if (flag < 0xC0UL) { + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) + asm volatile("": : :"memory"); +#endif + + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } diff --git a/storage/xtradb/srv/srv0srv.c b/storage/xtradb/srv/srv0srv.c index 26ad32cb1d1ed..0acce91f2c4b9 100644 --- a/storage/xtradb/srv/srv0srv.c +++ b/storage/xtradb/srv/srv0srv.c @@ -179,6 +179,9 @@ UNIV_INTERN char* srv_doublewrite_file = NULL; UNIV_INTERN ibool srv_recovery_stats = FALSE; +/** Whether the redo log tracking is currently enabled. Note that it is +possible for the log tracker thread to be running and the tracking to be +disabled */ UNIV_INTERN my_bool srv_track_changed_pages = FALSE; UNIV_INTERN ib_uint64_t srv_max_bitmap_file_size = 100 * 1024 * 1024; @@ -809,6 +812,11 @@ UNIV_INTERN os_event_t srv_checkpoint_completed_event; UNIV_INTERN os_event_t srv_redo_log_thread_finished_event; +/** Whether the redo log tracker thread has been started. Does not take into +account whether the tracking is currently enabled (see srv_track_changed_pages +for that) */ +UNIV_INTERN my_bool srv_redo_log_thread_started = FALSE; + UNIV_INTERN srv_sys_t* srv_sys = NULL; /* padding to prevent other memory update hotspots from residing on @@ -3179,18 +3187,15 @@ srv_redo_log_follow_thread( #endif my_thread_init(); + srv_redo_log_thread_started = TRUE; do { os_event_wait(srv_checkpoint_completed_event); os_event_reset(srv_checkpoint_completed_event); -#ifdef UNIV_DEBUG - if (!srv_track_changed_pages) { - continue; - } -#endif + if (srv_track_changed_pages + && srv_shutdown_state < SRV_SHUTDOWN_LAST_PHASE) { - if (srv_shutdown_state < SRV_SHUTDOWN_LAST_PHASE) { if (!log_online_follow_redo_log()) { /* TODO: sync with I_S log tracking status? */ fprintf(stderr, @@ -3206,6 +3211,7 @@ srv_redo_log_follow_thread( srv_track_changed_pages = FALSE; log_online_read_shutdown(); os_event_set(srv_redo_log_thread_finished_event); + srv_redo_log_thread_started = FALSE; /* Defensive, not required */ my_thread_end(); os_thread_exit(NULL); @@ -3327,7 +3333,7 @@ srv_master_do_purge(void) ut_ad(!mutex_own(&kernel_mutex)); - ut_a(srv_n_purge_threads == 0 || (srv_shutdown_state > 0 && srv_n_threads_active[SRV_WORKER] == 0)); + ut_a(srv_n_purge_threads == 0); do { /* Check for shutdown and change in purge config. */ @@ -3848,7 +3854,7 @@ srv_master_thread( /* Flush logs if needed */ srv_sync_log_buffer_in_background(); - if (srv_n_purge_threads == 0 || (srv_shutdown_state > 0 && srv_n_threads_active[SRV_WORKER] == 0)) { + if (srv_n_purge_threads == 0) { srv_main_thread_op_info = "master purging"; srv_master_do_purge(); @@ -3926,7 +3932,7 @@ srv_master_thread( } } - if (srv_n_purge_threads == 0 || (srv_shutdown_state > 0 && srv_n_threads_active[SRV_WORKER] == 0)) { + if (srv_n_purge_threads == 0) { srv_main_thread_op_info = "master purging"; srv_master_do_purge(); @@ -4142,9 +4148,10 @@ srv_purge_thread( We peek at the history len without holding any mutex because in the worst case we will end up waiting for the next purge event. */ - if (trx_sys->rseg_history_len < srv_purge_batch_size - || (n_total_purged == 0 - && retries >= TRX_SYS_N_RSEGS)) { + if (srv_shutdown_state == SRV_SHUTDOWN_NONE + && (trx_sys->rseg_history_len < srv_purge_batch_size + || (n_total_purged == 0 + && retries >= TRX_SYS_N_RSEGS))) { mutex_enter(&kernel_mutex); @@ -4159,8 +4166,12 @@ srv_purge_thread( /* Check for shutdown and whether we should do purge at all. */ if (srv_force_recovery >= SRV_FORCE_NO_BACKGROUND - || srv_shutdown_state != 0 - || srv_fast_shutdown) { + || (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown) + || (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown == 0 + && n_total_purged == 0 + && retries >= TRX_SYS_N_RSEGS)) { break; } @@ -4183,6 +4194,9 @@ srv_purge_thread( srv_sync_log_buffer_in_background(); + if (srv_shutdown_state != SRV_SHUTDOWN_NONE) + continue; + cur_time = ut_time_ms(); if (next_itr_time > cur_time) { os_thread_sleep(ut_min(1000000, From 383007c75d6ef5043fa5781956a6a02b24e2b79e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 13 Oct 2016 21:35:01 +0200 Subject: [PATCH 105/295] mysql cli: fix USE command quoting * use proper sql quoting rules for USE, while preserving as much of historical behavior as possible * short commands (\u) behave as before --- client/mysql.cc | 56 +++++++++++++++++++++------------------ mysql-test/r/mysql.result | 8 ++++++ mysql-test/t/mysql.test | 8 ++++++ 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 9d255b55430a1..9b1999f2c3849 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -245,7 +245,8 @@ static void end_pager(); static void init_tee(const char *); static void end_tee(); static const char* construct_prompt(); -static char *get_arg(char *line, my_bool get_next_arg); +enum get_arg_mode { CHECK, GET, GET_NEXT}; +static char *get_arg(char *line, get_arg_mode mode); static void init_username(); static void add_int_to_prompt(int toadd); static int get_result_width(MYSQL_RES *res); @@ -2223,7 +2224,7 @@ static COMMANDS *find_command(char *name) if (!my_strnncoll(&my_charset_latin1, (uchar*) name, len, (uchar*) commands[i].name, len) && (commands[i].name[len] == '\0') && - (!end || commands[i].takes_params)) + (!end || (commands[i].takes_params && get_arg(name, CHECK)))) { index= i; break; @@ -3143,7 +3144,7 @@ com_charset(String *buffer __attribute__((unused)), char *line) char buff[256], *param; CHARSET_INFO * new_cs; strmake_buf(buff, line); - param= get_arg(buff, 0); + param= get_arg(buff, GET); if (!param || !*param) { return put_info("Usage: \\C charset_name | charset charset_name", @@ -4228,12 +4229,12 @@ com_connect(String *buffer, char *line) #ifdef EXTRA_DEBUG tmp[1]= 0; #endif - tmp= get_arg(buff, 0); + tmp= get_arg(buff, GET); if (tmp && *tmp) { my_free(current_db); current_db= my_strdup(tmp, MYF(MY_WME)); - tmp= get_arg(buff, 1); + tmp= get_arg(buff, GET_NEXT); if (tmp) { my_free(current_host); @@ -4336,7 +4337,7 @@ com_delimiter(String *buffer __attribute__((unused)), char *line) char buff[256], *tmp; strmake_buf(buff, line); - tmp= get_arg(buff, 0); + tmp= get_arg(buff, GET); if (!tmp || !*tmp) { @@ -4367,7 +4368,7 @@ com_use(String *buffer __attribute__((unused)), char *line) bzero(buff, sizeof(buff)); strmake_buf(buff, line); - tmp= get_arg(buff, 0); + tmp= get_arg(buff, GET); if (!tmp || !*tmp) { put_info("USE must be followed by a database name", INFO_ERROR); @@ -4452,23 +4453,22 @@ com_nowarnings(String *buffer __attribute__((unused)), } /* - Gets argument from a command on the command line. If get_next_arg is - not defined, skips the command and returns the first argument. The - line is modified by adding zero to the end of the argument. If - get_next_arg is defined, then the function searches for end of string - first, after found, returns the next argument and adds zero to the - end. If you ever wish to use this feature, remember to initialize all - items in the array to zero first. + Gets argument from a command on the command line. If mode is not GET_NEXT, + skips the command and returns the first argument. The line is modified by + adding zero to the end of the argument. If mode is GET_NEXT, then the + function searches for end of string first, after found, returns the next + argument and adds zero to the end. If you ever wish to use this feature, + remember to initialize all items in the array to zero first. */ -char *get_arg(char *line, my_bool get_next_arg) +static char *get_arg(char *line, get_arg_mode mode) { char *ptr, *start; - my_bool quoted= 0, valid_arg= 0; + bool short_cmd= false; char qtype= 0; ptr= line; - if (get_next_arg) + if (mode == GET_NEXT) { for (; *ptr; ptr++) ; if (*(ptr + 1)) @@ -4479,7 +4479,7 @@ char *get_arg(char *line, my_bool get_next_arg) /* skip leading white spaces */ while (my_isspace(charset_info, *ptr)) ptr++; - if (*ptr == '\\') // short command was used + if ((short_cmd= *ptr == '\\')) // short command was used ptr+= 2; else while (*ptr &&!my_isspace(charset_info, *ptr)) // skip command @@ -4492,24 +4492,28 @@ char *get_arg(char *line, my_bool get_next_arg) if (*ptr == '\'' || *ptr == '\"' || *ptr == '`') { qtype= *ptr; - quoted= 1; ptr++; } for (start=ptr ; *ptr; ptr++) { - if (*ptr == '\\' && ptr[1]) // escaped character + if ((*ptr == '\\' && ptr[1]) || // escaped character + (!short_cmd && qtype && *ptr == qtype && ptr[1] == qtype)) // quote { - // Remove the backslash - strmov_overlapp(ptr, ptr+1); + // Remove (or skip) the backslash (or a second quote) + if (mode != CHECK) + strmov_overlapp(ptr, ptr+1); + else + ptr++; } - else if ((!quoted && *ptr == ' ') || (quoted && *ptr == qtype)) + else if (*ptr == (qtype ? qtype : ' ')) { - *ptr= 0; + qtype= 0; + if (mode != CHECK) + *ptr= 0; break; } } - valid_arg= ptr != start; - return valid_arg ? start : NullS; + return ptr != start && !qtype ? start : NullS; } diff --git a/mysql-test/r/mysql.result b/mysql-test/r/mysql.result index cb705d285fe3d..dd0129df0d9fe 100644 --- a/mysql-test/r/mysql.result +++ b/mysql-test/r/mysql.result @@ -512,6 +512,14 @@ DROP DATABASE connected_db; create database `aa``bb````cc`; DATABASE() aa`bb``cc +DATABASE() +test +DATABASE() +aa`bb``cc +DATABASE() +test +DATABASE() +aa`bb``cc drop database `aa``bb````cc`; a >>\ndelimiter\n<< diff --git a/mysql-test/t/mysql.test b/mysql-test/t/mysql.test index 6281bb5f4c1b7..d59083d66b0c7 100644 --- a/mysql-test/t/mysql.test +++ b/mysql-test/t/mysql.test @@ -586,8 +586,16 @@ DROP DATABASE connected_db; # USE and names with backticks # --write_file $MYSQLTEST_VARDIR/tmp/backticks.sql +\u aa`bb``cc +SELECT DATABASE(); +USE test +SELECT DATABASE(); USE aa`bb``cc SELECT DATABASE(); +USE test +SELECT DATABASE(); +USE `aa``bb````cc` +SELECT DATABASE(); EOF create database `aa``bb````cc`; --exec $MYSQL < $MYSQLTEST_VARDIR/tmp/backticks.sql From 01b39b7b0730102b88d8ea43ec719a75e9316a1e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 13 Oct 2016 20:58:08 +0200 Subject: [PATCH 106/295] mysqltest: don't eat new lines in --exec pass them through as is --- client/mysqltest.cc | 4 ---- mysql-test/r/mysql_not_windows.result | 6 ++++++ mysql-test/r/mysqltest.result | 6 ------ mysql-test/t/mysql_not_windows.test | 9 +++++++++ mysql-test/t/mysqltest.test | 9 --------- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 3652d1a40e2be..acb9e8b1e0c06 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -3349,10 +3349,6 @@ void do_exec(struct st_command *command) #endif #endif - /* exec command is interpreted externally and will not take newlines */ - while(replace(&ds_cmd, "\n", 1, " ", 1) == 0) - ; - DBUG_PRINT("info", ("Executing '%s' as '%s'", command->first_argument, ds_cmd.str)); diff --git a/mysql-test/r/mysql_not_windows.result b/mysql-test/r/mysql_not_windows.result index d5670a1a9ca38..1df62d9a12dcf 100644 --- a/mysql-test/r/mysql_not_windows.result +++ b/mysql-test/r/mysql_not_windows.result @@ -3,3 +3,9 @@ a 1 End of tests +1 +1 +2 +2 +X +3 diff --git a/mysql-test/r/mysqltest.result b/mysql-test/r/mysqltest.result index 865c8d7077b96..0ebef585974bc 100644 --- a/mysql-test/r/mysqltest.result +++ b/mysql-test/r/mysqltest.result @@ -269,12 +269,6 @@ source database echo message echo message mysqltest: At line 1: Missing argument in exec -1 -1 -2 -2 -X -3 MySQL "MySQL" MySQL: The diff --git a/mysql-test/t/mysql_not_windows.test b/mysql-test/t/mysql_not_windows.test index 66853677f7bb6..591de74cbbf47 100644 --- a/mysql-test/t/mysql_not_windows.test +++ b/mysql-test/t/mysql_not_windows.test @@ -13,3 +13,12 @@ --echo --echo End of tests + +# Multi-line exec +exec $MYSQL \ + test -e "select 1"; +exec $MYSQL test -e "select + 2"; +let $query = select 3 + as X; +exec $MYSQL test -e "$query"; diff --git a/mysql-test/t/mysqltest.test b/mysql-test/t/mysqltest.test index ffbec36873ef5..6470ede4f14c0 100644 --- a/mysql-test/t/mysqltest.test +++ b/mysql-test/t/mysqltest.test @@ -741,15 +741,6 @@ echo ; --error 1 --exec echo "--exec " | $MYSQL_TEST 2>&1 -# Multi-line exec -exec $MYSQL - test -e "select 1"; -exec $MYSQL test -e "select - 2"; -let $query = select 3 - as X; -exec $MYSQL test -e "$query"; - # ---------------------------------------------------------------------------- # Test let command # ---------------------------------------------------------------------------- From 5a43a31ee81bc181eeb5ef2bf0704befa6e0594d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 14 Oct 2016 00:33:49 +0200 Subject: [PATCH 107/295] mysqldump: comments and identifiers with new lines don't let identifiers with new lines to break a comment --- client/mysqldump.c | 60 ++++++++++----- mysql-test/r/mysqldump-nl.result | 126 +++++++++++++++++++++++++++++++ mysql-test/t/mysqldump-nl.test | 38 ++++++++++ 3 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 mysql-test/r/mysqldump-nl.result create mode 100644 mysql-test/t/mysqldump-nl.test diff --git a/client/mysqldump.c b/client/mysqldump.c index 16b39b77cf15f..32c350d307869 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -547,9 +547,7 @@ static int dump_all_tablespaces(); static int dump_tablespaces_for_tables(char *db, char **table_names, int tables); static int dump_tablespaces_for_databases(char** databases); static int dump_tablespaces(char* ts_where); -static void print_comment(FILE *sql_file, my_bool is_error, const char *format, - ...); - +static void print_comment(FILE *, my_bool, const char *, ...); /* Print the supplied message if in verbose mode @@ -627,6 +625,30 @@ static void short_usage(FILE *f) } +/** returns a string fixed to be safely printed inside a -- comment + + that is, any new line in it gets prefixed with -- +*/ +static const char *fix_for_comment(const char *ident) +{ + static char buf[1024]; + char c, *s= buf; + + while ((c= *s++= *ident++)) + { + if (s >= buf + sizeof(buf) - 10) + { + strmov(s, "..."); + break; + } + if (c == '\n') + s= strmov(s, "-- "); + } + + return buf; +} + + static void write_header(FILE *sql_file, char *db_name) { if (opt_xml) @@ -649,8 +671,8 @@ static void write_header(FILE *sql_file, char *db_name) DUMP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); print_comment(sql_file, 0, "-- Host: %s Database: %s\n", - current_host ? current_host : "localhost", - db_name ? db_name : ""); + fix_for_comment(current_host ? current_host : "localhost"), + fix_for_comment(db_name ? db_name : "")); print_comment(sql_file, 0, "-- ------------------------------------------------------\n" ); @@ -2094,7 +2116,8 @@ static uint dump_events_for_db(char *db) /* nice comments */ print_comment(sql_file, 0, - "\n--\n-- Dumping events for database '%s'\n--\n", db); + "\n--\n-- Dumping events for database '%s'\n--\n", + fix_for_comment(db)); /* not using "mysql_query_with_error_report" because we may have not @@ -2307,7 +2330,8 @@ static uint dump_routines_for_db(char *db) /* nice comments */ print_comment(sql_file, 0, - "\n--\n-- Dumping routines for database '%s'\n--\n", db); + "\n--\n-- Dumping routines for database '%s'\n--\n", + fix_for_comment(db)); /* not using "mysql_query_with_error_report" because we may have not @@ -2580,11 +2604,11 @@ static uint get_table_structure(char *table, char *db, char *table_type, if (strcmp (table_type, "VIEW") == 0) /* view */ print_comment(sql_file, 0, "\n--\n-- Temporary table structure for view %s\n--\n\n", - result_table); + fix_for_comment(result_table)); else print_comment(sql_file, 0, "\n--\n-- Table structure for table %s\n--\n\n", - result_table); + fix_for_comment(result_table)); if (opt_drop) { @@ -2826,7 +2850,7 @@ static uint get_table_structure(char *table, char *db, char *table_type, print_comment(sql_file, 0, "\n--\n-- Table structure for table %s\n--\n\n", - result_table); + fix_for_comment(result_table)); if (opt_drop) fprintf(sql_file, "DROP TABLE IF EXISTS %s;\n", result_table); if (!opt_xml) @@ -3530,21 +3554,21 @@ static void dump_table(char *table, char *db) { print_comment(md_result_file, 0, "\n--\n-- Dumping data for table %s\n--\n", - result_table); + fix_for_comment(result_table)); dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * FROM "); dynstr_append_checked(&query_string, result_table); if (where) { - print_comment(md_result_file, 0, "-- WHERE: %s\n", where); + print_comment(md_result_file, 0, "-- WHERE: %s\n", fix_for_comment(where)); dynstr_append_checked(&query_string, " WHERE "); dynstr_append_checked(&query_string, where); } if (order_by) { - print_comment(md_result_file, 0, "-- ORDER BY: %s\n", order_by); + print_comment(md_result_file, 0, "-- ORDER BY: %s\n", fix_for_comment(order_by)); dynstr_append_checked(&query_string, " ORDER BY "); dynstr_append_checked(&query_string, order_by); @@ -4053,7 +4077,7 @@ static int dump_tablespaces(char* ts_where) if (first) { print_comment(md_result_file, 0, "\n--\n-- Logfile group: %s\n--\n", - row[0]); + fix_for_comment(row[0])); fprintf(md_result_file, "\nCREATE"); } @@ -4122,7 +4146,8 @@ static int dump_tablespaces(char* ts_where) first= 1; if (first) { - print_comment(md_result_file, 0, "\n--\n-- Tablespace: %s\n--\n", row[0]); + print_comment(md_result_file, 0, "\n--\n-- Tablespace: %s\n--\n", + fix_for_comment(row[0])); fprintf(md_result_file, "\nCREATE"); } else @@ -4326,7 +4351,8 @@ static int init_dumping(char *database, int init_func(char*)) char *qdatabase= quote_name(database,quoted_database_buf,opt_quoted); print_comment(md_result_file, 0, - "\n--\n-- Current Database: %s\n--\n", qdatabase); + "\n--\n-- Current Database: %s\n--\n", + fix_for_comment(qdatabase)); /* Call the view or table specific function */ init_func(qdatabase); @@ -5356,7 +5382,7 @@ static my_bool get_view_structure(char *table, char* db) print_comment(sql_file, 0, "\n--\n-- Final view structure for view %s\n--\n\n", - result_table); + fix_for_comment(result_table)); /* Table might not exist if this view was dumped with --tab. */ fprintf(sql_file, "/*!50001 DROP TABLE IF EXISTS %s*/;\n", opt_quoted_table); diff --git a/mysql-test/r/mysqldump-nl.result b/mysql-test/r/mysqldump-nl.result new file mode 100644 index 0000000000000..6de439bdf3c61 --- /dev/null +++ b/mysql-test/r/mysqldump-nl.result @@ -0,0 +1,126 @@ +create database `mysqltest1 +1tsetlqsym`; +use `mysqltest1 +1tsetlqsym`; +create table `t1 +1t` (`foobar +raboof` int); +create view `v1 +1v` as select * from `t1 +1t`; +create procedure sp() select * from `v1 +1v`; +flush tables; +use test; + +-- +-- Current Database: `mysqltest1 +-- 1tsetlqsym` +-- + +/*!40000 DROP DATABASE IF EXISTS `mysqltest1 +1tsetlqsym`*/; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest1 +1tsetlqsym` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest1 +1tsetlqsym`; + +-- +-- Table structure for table `t1 +-- 1t` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t1 +1t` ( + `foobar +raboof` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `t1 +-- 1t` +-- + +-- +-- Temporary table structure for view `v1 +-- 1v` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v1 +1v` ( + `foobar +raboof` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; + +-- +-- Dumping routines for database 'mysqltest1 +-- 1tsetlqsym' +-- +/*!50003 SET @saved_cs_client = @@character_set_client */ ; +/*!50003 SET @saved_cs_results = @@character_set_results */ ; +/*!50003 SET @saved_col_connection = @@collation_connection */ ; +/*!50003 SET character_set_client = latin1 */ ; +/*!50003 SET character_set_results = latin1 */ ; +/*!50003 SET collation_connection = latin1_swedish_ci */ ; +/*!50003 SET @saved_sql_mode = @@sql_mode */ ; +/*!50003 SET sql_mode = '' */ ; +DELIMITER ;; +CREATE DEFINER=`root`@`localhost` PROCEDURE `sp`() +select * from `v1 +1v` ;; +DELIMITER ; +/*!50003 SET sql_mode = @saved_sql_mode */ ; +/*!50003 SET character_set_client = @saved_cs_client */ ; +/*!50003 SET character_set_results = @saved_cs_results */ ; +/*!50003 SET collation_connection = @saved_col_connection */ ; + +-- +-- Current Database: `mysqltest1 +-- 1tsetlqsym` +-- + +USE `mysqltest1 +1tsetlqsym`; + +-- +-- Final view structure for view `v1 +-- 1v` +-- + +/*!50001 DROP TABLE IF EXISTS `v1 +1v`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v1 +1v` AS select `t1 +1t`.`foobar +raboof` AS `foobar +raboof` from `t1 +1t` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +show tables from `mysqltest1 +1tsetlqsym`; +Tables_in_mysqltest1 +1tsetlqsym +t1 +1t +v1 +1v +drop database `mysqltest1 +1tsetlqsym`; diff --git a/mysql-test/t/mysqldump-nl.test b/mysql-test/t/mysqldump-nl.test new file mode 100644 index 0000000000000..311996e77c305 --- /dev/null +++ b/mysql-test/t/mysqldump-nl.test @@ -0,0 +1,38 @@ +# +# New lines in identifiers +# + +# embedded server doesn't support external clients +--source include/not_embedded.inc +# cmd.exe doesn't like new lines on the command line +--source include/not_windows.inc + +create database `mysqltest1 +1tsetlqsym`; +use `mysqltest1 +1tsetlqsym`; + +create table `t1 +1t` (`foobar +raboof` int); +create view `v1 +1v` as select * from `t1 +1t`; + +create procedure sp() select * from `v1 +1v`; + +flush tables; +use test; + +exec $MYSQL_DUMP --compact --comment --routines --add-drop-database --databases 'mysqltest1 +1tsetlqsym'; + +exec $MYSQL_DUMP --compact --comment --routines --add-drop-database --databases 'mysqltest1 +1tsetlqsym' | $MYSQL; + +show tables from `mysqltest1 +1tsetlqsym`; + +drop database `mysqltest1 +1tsetlqsym`; From eac8d95ffcdea7cd31d60d273e30cb3dfec66add Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 14 Oct 2016 12:51:53 +0200 Subject: [PATCH 108/295] compilation warning after xtradb merge --- storage/xtradb/handler/ha_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 386920e689d1b..66fcc2799bb94 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -481,7 +481,7 @@ int innobase_get_parent_fk_list( THD* thd, const char* path, - List* f_key_list); + List* f_key_list) __attribute__((unused)); /******************************************************************//** Maps a MySQL trx isolation level code to the InnoDB isolation level code From b7aee7dbe71cf77199e28e905469f0d9fb6d4a80 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 14 Oct 2016 18:29:33 +0200 Subject: [PATCH 109/295] - Fix MDEV-10950. Null values not retrieved for numeric types. Now the null is tested using the result set getObject method. modified: storage/connect/JdbcInterface.java modified: storage/connect/jdbconn.cpp modified: storage/connect/jdbconn.h --- storage/connect/JdbcInterface.java | 4 +-- storage/connect/jdbconn.cpp | 39 ++++++++++++++++++++---------- storage/connect/jdbconn.h | 1 + 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index e339c9891133a..34af8c4e013c2 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -692,11 +692,11 @@ public int TimestampField(int n, String name) { return 0; } // end of TimestampField - public String ObjectField(int n, String name) { + public Object ObjectField(int n, String name) { if (rs == null) { System.out.println("No result set"); } else try { - return (n > 0) ? rs.getObject(n).toString() : rs.getObject(name).toString(); + return (n > 0) ? rs.getObject(n) : rs.getObject(name); } catch (SQLException se) { SetErrmsg(se); } //end try/catch diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index 229ade53ad1cf..dca9bd0eac4e5 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -512,7 +512,7 @@ JDBConn::JDBConn(PGLOBAL g, TDBJDBC *tdbp) xqid = xuid = xid = grs = readid = fetchid = typid = errid = nullptr; prepid = xpid = pcid = nullptr; chrfldid = intfldid = dblfldid = fltfldid = bigfldid = nullptr; - datfldid = timfldid = tspfldid = nullptr; + objfldid = datfldid = timfldid = tspfldid = nullptr; //m_LoginTimeout = DEFAULT_LOGIN_TIMEOUT; //m_QueryTimeout = DEFAULT_QUERY_TIMEOUT; //m_UpdateOptions = 0; @@ -1167,9 +1167,10 @@ void JDBConn::Close() /***********************************************************************/ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val) { - PGLOBAL& g = m_G; - jint ctyp; - jstring cn, jn = nullptr; + PGLOBAL& g = m_G; + jint ctyp; + jstring cn, jn = nullptr; + jobject jb = nullptr; if (rank == 0) if (!name || (jn = env->NewStringUTF(name)) == nullptr) { @@ -1185,21 +1186,32 @@ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val) longjmp(g->jumper[g->jump_level], TYPE_AM_JDBC); } // endif Check + if (val->GetNullable()) + if (!gmID(g, objfldid, "ObjectField", "(ILjava/lang/String;)Ljava/lang/Object;")) { + jb = env->CallObjectMethod(job, objfldid, (jint)rank, jn); + + if (jb == nullptr) { + val->Reset(); + val->SetNull(true); + goto chk; + } // endif job + + } // endif objfldid + switch (ctyp) { case 12: // VARCHAR case -1: // LONGVARCHAR case 1: // CHAR - if (!gmID(g, chrfldid, "StringField", "(ILjava/lang/String;)Ljava/lang/String;")) { + if (jb) + cn = (jstring)jb; + else if (!gmID(g, chrfldid, "StringField", "(ILjava/lang/String;)Ljava/lang/String;")) cn = (jstring)env->CallObjectMethod(job, chrfldid, (jint)rank, jn); + else + cn = nullptr; - if (cn) { - const char *field = env->GetStringUTFChars(cn, (jboolean)false); - val->SetValue_psz((PSZ)field); - } else { - val->Reset(); - val->SetNull(true); - } // endif cn - + if (cn) { + const char *field = env->GetStringUTFChars(cn, (jboolean)false); + val->SetValue_psz((PSZ)field); } else val->Reset(); @@ -1271,6 +1283,7 @@ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val) val->Reset(); } // endswitch Type + chk: if (Check()) { if (rank == 0) env->DeleteLocalRef(jn); diff --git a/storage/connect/jdbconn.h b/storage/connect/jdbconn.h index 095b1565bd276..0a1c52d4576d1 100644 --- a/storage/connect/jdbconn.h +++ b/storage/connect/jdbconn.h @@ -165,6 +165,7 @@ class JDBConn : public BLOCK { jmethodID xpid; // The ExecutePrep method ID jmethodID pcid; // The ClosePrepStmt method ID jmethodID errid; // The GetErrmsg method ID + jmethodID objfldid; // The ObjectField method ID jmethodID chrfldid; // The StringField method ID jmethodID intfldid; // The IntField method ID jmethodID dblfldid; // The DoubleField method ID From f6d4f82d6e49ed1ca2155c9e0e12f3dd8fcb1acf Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 14 Oct 2016 23:23:16 +0300 Subject: [PATCH 110/295] MDEV-11061 Valgrind builder produces endless warnings after switching to OpenSS --- mysql-test/valgrind.supp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 723b609de8f41..2dd10ff4008a2 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1228,3 +1228,25 @@ fun:dlopen@@GLIBC_2.2.5 } +{ + MDEV-11061: OpenSSL 0.9.8 - Conditional jump or move + Memcheck:Cond + fun:BN_* + ... + fun:ssl3_ctx_ctrl + fun:new_VioSSLFd + fun:new_VioSSLAcceptorFd + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 - Use of uninitialised value + Memcheck:Value8 + fun:BN_* + ... + fun:ssl3_ctx_ctrl + fun:new_VioSSLFd + fun:new_VioSSLAcceptorFd + ... +} + From 8a49e00f3f1a81b6645ac3f2d843c9e5dd0375ba Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 14 Oct 2016 23:23:49 +0300 Subject: [PATCH 111/295] More unstable tests --- mysql-test/unstable-tests | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 6a46602eb0795..2dbaeeebc0bd6 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -56,6 +56,7 @@ main.named_pipe : Modified on 2016-08-02 (MDEV-10383) main.openssl_1 : Modified on 2016-07-11 (MDEV-10211) main.parser : Modified on 2016-06-21 (merge) main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections +main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count main.ps_1general : Modified on 2016-07-12 (merge) main.range : Modified on 2016-08-10 (merge) main.range_mrr_icp : Modified on 2016-08-10 (merge) @@ -116,6 +117,7 @@ innodb.innodb_corrupt_bit : Modified on 2016-06-21 (merge) innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan innodb.innodb-fk-warnings : Modified on 2016-07-18 (MDEV-8569) innodb.innodb-fkcheck : Modified on 2016-06-13 (MDEV-10083) +innodb.innodb_monitor : MDEV-10939 - Testcase timeout innodb.innodb-wl5522 : rdiff file modified on 2016-08-10 (merge) innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption @@ -145,6 +147,7 @@ parts.partition_int_myisam : MDEV-10621 - Testcase timeout perfschema.digest_table_full : Modified on 2016-06-21 (merge) perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x +perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown perfschema.rpl_gtid_func : Modified on 2016-06-21 (merge) perfschema.sizing_low : Modified on 2016-04-26 (5.6.30 merge) perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match From 4192c468675220e0ad2de9eb722cfa457c0e5ced Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 16 Oct 2016 04:46:39 +0300 Subject: [PATCH 112/295] MDEV-11061 Valgrind builder produces endless warnings OpenSSL problems, part II --- mysql-test/valgrind.supp | 117 +++++++++++++++++++++++++++++++++++---- 1 file changed, 107 insertions(+), 10 deletions(-) diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 2dd10ff4008a2..77f17cf07ec0c 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1228,25 +1228,122 @@ fun:dlopen@@GLIBC_2.2.5 } +# +# MDEV-11061: OpenSSL 0.9.8 problems +# + { - MDEV-11061: OpenSSL 0.9.8 - Conditional jump or move + MDEV-11061: OpenSSL 0.9.8 Memcheck:Cond - fun:BN_* + obj:*/libz.so* + ... + obj:*/libcrypto.so.0.9.8 ... - fun:ssl3_ctx_ctrl - fun:new_VioSSLFd - fun:new_VioSSLAcceptorFd + obj:*/libssl.so.0.9.8 ... } { - MDEV-11061: OpenSSL 0.9.8 - Use of uninitialised value + MDEV-11061: OpenSSL 0.9.8 Memcheck:Value8 - fun:BN_* + obj:*/libz.so* + ... + obj:*/libcrypto.so.0.9.8 + ... + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Cond + obj:*/libcrypto.so.0.9.8 + ... + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Value8 + obj:*/libcrypto.so.0.9.8 + ... + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Cond + obj:*/libssl.so.0.9.8 + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Value8 + obj:*/libssl.so.0.9.8 + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Cond + fun:memcpy + obj:*/libcrypto.so.0.9.8 + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Value8 + fun:memcpy + obj:*/libcrypto.so.0.9.8 + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Cond + fun:is_overlap + fun:memcpy + obj:*/libcrypto.so.0.9.8 + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Cond + fun:memset + obj:*/libcrypto.so.0.9.8 + ... + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Value8 + fun:memset + obj:*/libcrypto.so.0.9.8 + ... + obj:*/libssl.so.0.9.8 + ... +} + +{ + MDEV-11061: OpenSSL 0.9.8 + Memcheck:Param + write(buf) + obj:*/libpthread-2.9.so* + obj:*/libcrypto.so.0.9.8 ... - fun:ssl3_ctx_ctrl - fun:new_VioSSLFd - fun:new_VioSSLAcceptorFd + obj:*/libssl.so.0.9.8 ... } From df87be5edafb402e36e9c16aa0f00b1d5104d920 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 17 Oct 2016 14:04:45 +0300 Subject: [PATCH 113/295] MDEV-11069 main.information_schema test fails if hostname includes 'user' Patch provided by Honza Horak --- mysql-test/r/information_schema.result | 8 ++++---- mysql-test/t/information_schema.test | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index d98f8168e9e03..1f765a70137ca 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -986,19 +986,19 @@ show grants; Grants for user3@localhost GRANT USAGE ON *.* TO 'user3'@'localhost' GRANT SELECT ON `mysqltest`.* TO 'user3'@'localhost' -select * from information_schema.column_privileges where grantee like '%user%' +select * from information_schema.column_privileges where grantee like '\'user%' order by grantee; GRANTEE TABLE_CATALOG TABLE_SCHEMA TABLE_NAME COLUMN_NAME PRIVILEGE_TYPE IS_GRANTABLE 'user1'@'localhost' def mysqltest t1 f1 SELECT NO -select * from information_schema.table_privileges where grantee like '%user%' +select * from information_schema.table_privileges where grantee like '\'user%' order by grantee; GRANTEE TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PRIVILEGE_TYPE IS_GRANTABLE 'user2'@'localhost' def mysqltest t2 SELECT NO -select * from information_schema.schema_privileges where grantee like '%user%' +select * from information_schema.schema_privileges where grantee like '\'user%' order by grantee; GRANTEE TABLE_CATALOG TABLE_SCHEMA PRIVILEGE_TYPE IS_GRANTABLE 'user3'@'localhost' def mysqltest SELECT NO -select * from information_schema.user_privileges where grantee like '%user%' +select * from information_schema.user_privileges where grantee like '\'user%' order by grantee; GRANTEE TABLE_CATALOG PRIVILEGE_TYPE IS_GRANTABLE 'user1'@'localhost' def USAGE NO diff --git a/mysql-test/t/information_schema.test b/mysql-test/t/information_schema.test index fb39f2e5d580b..943eb8bab8ab6 100644 --- a/mysql-test/t/information_schema.test +++ b/mysql-test/t/information_schema.test @@ -612,13 +612,13 @@ select * from information_schema.schema_privileges order by grantee; select * from information_schema.user_privileges order by grantee; show grants; connection con4; -select * from information_schema.column_privileges where grantee like '%user%' +select * from information_schema.column_privileges where grantee like '\'user%' order by grantee; -select * from information_schema.table_privileges where grantee like '%user%' +select * from information_schema.table_privileges where grantee like '\'user%' order by grantee; -select * from information_schema.schema_privileges where grantee like '%user%' +select * from information_schema.schema_privileges where grantee like '\'user%' order by grantee; -select * from information_schema.user_privileges where grantee like '%user%' +select * from information_schema.user_privileges where grantee like '\'user%' order by grantee; show grants; connection default; From 6e257274d98843b228e5bd08da74031f6f3a202d Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 17 Oct 2016 11:43:47 -0400 Subject: [PATCH 114/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d44c8b2800612..4f1ecb3a1972b 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=53 +MYSQL_VERSION_PATCH=54 MYSQL_VERSION_EXTRA= From 4dfb6a3f54cfb26535636197cc5fa70fe5bacc2e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 28 Sep 2016 14:16:38 +0000 Subject: [PATCH 115/295] MDEV-11083 performance schema test fail with threadpool Fix PSI idle and socket instrumentation in threadpool --- sql/threadpool_common.cc | 99 ++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/sql/threadpool_common.cc b/sql/threadpool_common.cc index 5bcea767aae06..9d263038bc95f 100644 --- a/sql/threadpool_common.cc +++ b/sql/threadpool_common.cc @@ -73,17 +73,16 @@ struct Worker_thread_context void save() { -#ifdef HAVE_PSI_INTERFACE - psi_thread= PSI_server?PSI_server->get_thread():0; +#ifdef HAVE_PSI_THREAD_INTERFACE + psi_thread = PSI_THREAD_CALL(get_thread)(); #endif mysys_var= (st_my_thread_var *)pthread_getspecific(THR_KEY_mysys); } void restore() { -#ifdef HAVE_PSI_INTERFACE - if (PSI_server) - PSI_server->set_thread(psi_thread); +#ifdef HAVE_PSI_THREAD_INTERFACE + PSI_THREAD_CALL(set_thread)(psi_thread); #endif pthread_setspecific(THR_KEY_mysys,mysys_var); pthread_setspecific(THR_THD, 0); @@ -92,6 +91,41 @@ struct Worker_thread_context }; +#ifdef HAVE_PSI_INTERFACE + +/* + The following fixes PSI "idle" psi instrumentation. + The server assumes that connection becomes idle + just before net_read_packet() and switches to active after it. + In out setup, server becomes idle when async socket io is made. +*/ + +extern void net_before_header_psi(struct st_net *net, void *user_data, size_t); + +static void dummy_before_header(struct st_net *, void *, size_t) +{ +} + +static void re_init_net_server_extension(THD *thd) +{ + thd->m_net_server_extension.m_before_header = dummy_before_header; +} + +#else + +#define re_init_net_server_extension(thd) + +#endif /* HAVE_PSI_INTERFACE */ + + +static inline void set_thd_idle(THD *thd) +{ + thd->net.reading_or_writing= 1; +#ifdef HAVE_PSI_INTERFACE + net_before_header_psi(&thd->net, thd, 0); +#endif +} + /* Attach/associate the connection with the OS thread, */ @@ -100,10 +134,10 @@ static bool thread_attach(THD* thd) pthread_setspecific(THR_KEY_mysys,thd->mysys_var); thd->thread_stack=(char*)&thd; thd->store_globals(); -#ifdef HAVE_PSI_INTERFACE - if (PSI_server) - PSI_server->set_thread(thd->event_scheduler.m_psi); +#ifdef HAVE_PSI_THREAD_INTERFACE + PSI_THREAD_CALL(set_thread)(thd->event_scheduler.m_psi); #endif + mysql_socket_set_thread_owner(thd->net.vio->mysql_socket); return 0; } @@ -130,40 +164,38 @@ int threadpool_add_connection(THD *thd) } /* Create new PSI thread for use with the THD. */ -#ifdef HAVE_PSI_INTERFACE - if (PSI_server) - { - thd->event_scheduler.m_psi = - PSI_server->new_thread(key_thread_one_connection, thd, thd->thread_id); - } +#ifdef HAVE_PSI_THREAD_INTERFACE + thd->event_scheduler.m_psi= + PSI_THREAD_CALL(new_thread)(key_thread_one_connection, thd, thd->thread_id); #endif /* Login. */ thread_attach(thd); + re_init_net_server_extension(thd); ulonglong now= microsecond_interval_timer(); thd->prior_thr_create_utime= now; thd->start_utime= now; thd->thr_create_utime= now; - if (!setup_connection_thread_globals(thd)) - { - if (!login_connection(thd)) - { - prepare_new_connection_state(thd); - - /* - Check if THD is ok, as prepare_new_connection_state() - can fail, for example if init command failed. - */ - if (thd_is_connection_alive(thd)) - { - retval= 0; - thd->net.reading_or_writing= 1; - thd->skip_wait_timeout= true; - } - } - } + if (setup_connection_thread_globals(thd)) + goto end; + + if (thd_prepare_connection(thd)) + goto end; + + /* + Check if THD is ok, as prepare_new_connection_state() + can fail, for example if init command failed. + */ + if (!thd_is_connection_alive(thd)) + goto end; + + retval= 0; + thd->skip_wait_timeout= true; + set_thd_idle(thd); + +end: worker_context.restore(); return retval; } @@ -245,12 +277,13 @@ int threadpool_process_request(THD *thd) goto end; } + set_thd_idle(thd); + vio= thd->net.vio; if (!vio->has_data(vio)) { /* More info on this debug sync is in sql_parse.cc*/ DEBUG_SYNC(thd, "before_do_command_net_read"); - thd->net.reading_or_writing= 1; goto end; } } From fd1f5072839467d2d0216b85c2cd576284bc4955 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 19 Oct 2016 03:01:36 +0300 Subject: [PATCH 116/295] Additions to the list of unstable tests --- mysql-test/unstable-tests | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index daba2ea8ae8c7..82713424daa7c 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -44,6 +44,7 @@ main.index_merge_innodb : MDEV-7142 - sporadic wrong execution plan main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure main.insert_innodb : Modified on 2016-06-14 (merge from upstream) +main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist main.loaddata : Modified on 2016-08-10 (merge) main.locale : Modified on 2016-06-21 (merge) main.mdev-504 : MDEV-10607 - sporadic "can't connect" @@ -59,6 +60,7 @@ main.openssl_1 : Modified on 2016-07-11 (MDEV-10211) main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan main.parser : Modified on 2016-06-21 (merge) main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections +main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count main.ps_1general : Modified on 2016-07-12 (merge) main.range : Modified on 2016-08-10 (merge) main.range_mrr_icp : Modified on 2016-08-10 (merge) @@ -147,7 +149,7 @@ galera_3nodes.galera_pc_weight : Modified in 10.1.17 encryption.create_or_replace : MDEV-9359 - Assertion failure encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures encryption.innodb_encryption_discard_import : MDEV-9099 - warnings, errors, crash -encryption.innodb_encryption_filekeys : MDEV-9062 - timeouts +encryption.innodb_encryption_filekeys : MDEV-9962 - timeouts encryption.innodb_first_page : MDEV-10689 - crashes encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results encryption.innodb-page_encryption : MDEV-10641 - mutex problem @@ -266,6 +268,7 @@ rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips +rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts From 25848978334f9682f0b4712bf9d7c8222bab606f Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 19 Oct 2016 03:02:13 +0300 Subject: [PATCH 117/295] MDEV-11082 mysql_client_test: test_ps_query_cache fails with group-concat-max-len=1M test_bug14169 was setting session group_concat_max_len=1024 and did not clean it up. Because of that test_ps_query_cache, when run with group-concat-max-len != 1024, had different values in connections, and was inserting into query cache when a hit was expected. Fixed by adding a clean-up for the value in test_bug14169 --- tests/mysql_client_test.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index b95127f954fd1..ad8933118ccf1 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -15375,6 +15375,9 @@ static void test_bug14169() rc= mysql_query(mysql, "drop table t1"); myquery(rc); + + rc= mysql_query(mysql, "set session group_concat_max_len=@@global.group_concat_max_len"); + myquery(rc); } /* From 998f987eda62e6b3481ac3914538282715e2df4a Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 21 Oct 2016 22:37:51 +0200 Subject: [PATCH 118/295] Upstream MIPS test fixes from Debian Bug 838557. https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=838557 MIPS has a different errno for "directory not empty". --- mysql-test/extra/binlog_tests/database.test | 2 +- mysql-test/suite/rpl/t/rpl_drop_db.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/extra/binlog_tests/database.test b/mysql-test/extra/binlog_tests/database.test index f111a028642e4..17f8e069fa3ce 100644 --- a/mysql-test/extra/binlog_tests/database.test +++ b/mysql-test/extra/binlog_tests/database.test @@ -52,7 +52,7 @@ eval SELECT 'hello' INTO OUTFILE 'fake_file.$prefix'; # Use '/' instead of '\' in the error message. On windows platform, dir is # formed with '\'. ---replace_regex /\\testing_1\\*/\/testing_1\// /66/39/ /17/39/ /247/39/ /File exists/Directory not empty/ +--replace_regex /\\testing_1\\*/\/testing_1\// /66/39/ /93/39/ /17/39/ /247/39/ /File exists/Directory not empty/ --error 1010 DROP DATABASE testing_1; let $wait_binlog_event= DROP TABLE IF EXIST; diff --git a/mysql-test/suite/rpl/t/rpl_drop_db.test b/mysql-test/suite/rpl/t/rpl_drop_db.test index dae1651dc93ae..f66187b12f515 100644 --- a/mysql-test/suite/rpl/t/rpl_drop_db.test +++ b/mysql-test/suite/rpl/t/rpl_drop_db.test @@ -13,7 +13,7 @@ insert into mysqltest1.t1 values (1); select * from mysqltest1.t1 into outfile 'mysqltest1/f1.txt'; create table mysqltest1.t2 (n int); create table mysqltest1.t3 (n int); ---replace_result \\ / 66 39 17 39 247 39 "File exists" "Directory not empty" +--replace_result \\ / 66 39 93 39 17 39 247 39 "File exists" "Directory not empty" --error 1010 drop database mysqltest1; use mysqltest1; @@ -30,7 +30,7 @@ while ($1) } --enable_query_log ---replace_result \\ / 66 39 17 39 247 39 "File exists" "Directory not empty" +--replace_result \\ / 66 39 93 39 17 39 247 39 "File exists" "Directory not empty" --error 1010 drop database mysqltest1; use mysqltest1; From 7eb4bd3f1ddd9b84425d51550b44c14ac0a8f1de Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 21 Oct 2016 22:43:46 +0200 Subject: [PATCH 119/295] Upstream patch from Debian Bug 838557 The patch fixes 128-bit multiply on mips64. This corrects a previous incorrect patch upstreamed from Debian. --- extra/yassl/taocrypt/src/integer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extra/yassl/taocrypt/src/integer.cpp b/extra/yassl/taocrypt/src/integer.cpp index fb8d9276bd9f8..dd8425396eda2 100644 --- a/extra/yassl/taocrypt/src/integer.cpp +++ b/extra/yassl/taocrypt/src/integer.cpp @@ -193,8 +193,9 @@ DWord() {} "a" (a), "rm" (b) : "cc"); #elif defined(__mips64) - __asm__("dmultu %2,%3" : "=d" (r.halfs_.high), "=l" (r.halfs_.low) - : "r" (a), "r" (b)); + unsigned __int128 t = (unsigned __int128) a * b; + r.halfs_.high = t >> 64; + r.halfs_.low = (word) t; #elif defined(_M_IX86) // for testing From 39b7affcb13f9f508242e90ecd5db03b3bb3cb85 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 21 Oct 2016 23:02:56 +0200 Subject: [PATCH 120/295] Upstream MIPS 32bit-build-on-64bit patch from Debian Bug#838914 From https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=838914 Fixes CMake so that when building a 32-bit mips binary on a 64-bit mips machine, the target is not set as 32-bit, which apparently confused some tests in mroonga. --- cmake/package_name.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/package_name.cmake b/cmake/package_name.cmake index 4ba8fc18e3f4c..48ca3a4814dd1 100644 --- a/cmake/package_name.cmake +++ b/cmake/package_name.cmake @@ -30,6 +30,10 @@ IF(NOT VERSION) SET(64BIT 1) ENDIF() + IF(NOT 64BIT AND CMAKE_SYSTEM_PROCESSOR MATCHES "^mips64") + SET(DEFAULT_MACHINE "mips") + ENDIF() + IF(CMAKE_SYSTEM_NAME MATCHES "Windows") SET(NEED_DASH_BETWEEN_PLATFORM_AND_MACHINE 0) SET(DEFAULT_PLATFORM "win") From fb38d2642011c574cc9103ae1a1f9dd77f7f027e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 22 Oct 2016 07:34:23 +0000 Subject: [PATCH 121/295] MDEV-11104 Fix client to correctly retrieve current user name on Windows Prior to this patch name of the user was read from environment variable USER, with a fallback to 'ODBC', if the environment variable is not set. The name of the env.variable is incorrect (USERNAME usually contains current user's name, but not USER), which made client to always determine current user as 'ODBC'. The fix is to use GetUserName() instead. --- libmysql/libmysql.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libmysql/libmysql.c b/libmysql/libmysql.c index 446f1da0b0c7d..3a08ea26b1d57 100644 --- a/libmysql/libmysql.c +++ b/libmysql/libmysql.c @@ -450,8 +450,9 @@ void read_user_name(char *name) void read_user_name(char *name) { - char *str=getenv("USER"); /* ODBC will send user variable */ - strmake(name,str ? str : "ODBC", USERNAME_LENGTH); + DWORD len= USERNAME_LENGTH; + if (!GetUserName(name, &len)) + strmov(name,"UNKNOWN_USER"); } #endif From 8f5e3e2a3491ae0f1b47400a82408b7853ae1efb Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 21 Oct 2016 16:20:47 +0000 Subject: [PATCH 122/295] Fix escaping '\' in a string constant. --- mysys/my_access.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysys/my_access.c b/mysys/my_access.c index 0da5e7f0bf009..91a536d214e12 100644 --- a/mysys/my_access.c +++ b/mysys/my_access.c @@ -186,7 +186,7 @@ static my_bool does_drive_exists(char drive_letter) @return TRUE if the file name is allowed, FALSE otherwise. */ -#define ILLEGAL_FILENAME_CHARS "<>:\"/\|?*" +#define ILLEGAL_FILENAME_CHARS "<>:\"/\\|?*" my_bool is_filename_allowed(const char *name __attribute__((unused)), size_t length __attribute__((unused)), From de5646f1a9aaf45f1b43d98623b40c95fb98ebce Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 22 Oct 2016 14:10:12 +0000 Subject: [PATCH 123/295] Prepare XtraDB to be used with xtrabackup. The changes are deliberately kept minimal - some functions are made global instead of static (they will be used in xtrabackup later on) - functions got additional parameter, deliberately unused for now : fil_load_single_tablespaces srv_undo_tablespaces_init - Global variables added, also unused for now : srv_archive_recovery srv_archive_recovery_limit_lsn srv_apply_log_only srv_backup_mode srv_close_files - To make xtrabackup link with sql.lib on Windows, added some missing source files to sql.lib - Fixed os_thread_ret_t to be DWORD on Windows --- sql/CMakeLists.txt | 9 ++- storage/xtradb/btr/btr0btr.cc | 4 +- storage/xtradb/fil/fil0fil.cc | 5 +- storage/xtradb/handler/ha_innodb.cc | 6 +- storage/xtradb/include/fil0fil.h | 2 +- storage/xtradb/include/srv0srv.h | 7 ++ storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0recv.cc | 99 ++++++++++++++++++++++++++++- storage/xtradb/os/os0file.cc | 1 - storage/xtradb/srv/srv0srv.cc | 7 ++ storage/xtradb/srv/srv0start.cc | 10 ++- 11 files changed, 135 insertions(+), 17 deletions(-) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index ec7e9207f38d5..1cc83ea2bf13a 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -77,6 +77,12 @@ IF(SSL_DEFINES) ADD_DEFINITIONS(${SSL_DEFINES}) ENDIF() +IF(WIN32) + SET(NT_SERVICE_SOURCES nt_servc.cc nt_servc.h ) +ELSE() + SET(NT_SERVICE_SOURCES) +ENDIF() + SET (SQL_SOURCE ../sql-common/client.c compat56.cc derror.cc des_key_file.cc discover.cc ../libmysql/errmsg.c field.cc field_conv.cc @@ -143,6 +149,7 @@ SET (SQL_SOURCE ${GEN_SOURCES} ${GEN_DIGEST_SOURCES} ${MYSYS_LIBWRAP_SOURCE} + ${NT_SERVICE_SOURCES} ) IF (CMAKE_SYSTEM_NAME MATCHES "Linux" OR @@ -172,7 +179,7 @@ TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} ${LIBSYSTEMD}) IF(WIN32) - SET(MYSQLD_SOURCE main.cc nt_servc.cc nt_servc.h message.rc) + SET(MYSQLD_SOURCE main.cc message.rc) TARGET_LINK_LIBRARIES(sql psapi) ELSE() SET(MYSQLD_SOURCE main.cc ${DTRACE_PROBES_ALL}) diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index bce81f95eaddb..bf6d850f3f08d 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -722,7 +722,7 @@ btr_root_fseg_validate( /**************************************************************//** Gets the root node of a tree and x- or s-latches it. @return root page, x- or s-latched */ -static + buf_block_t* btr_root_block_get( /*===============*/ @@ -1531,7 +1531,7 @@ btr_node_ptr_set_child_page_no( /************************************************************//** Returns the child page of a node pointer and x-latches it. @return child page, x-latched */ -static + buf_block_t* btr_node_ptr_get_child( /*===================*/ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 93df92e6e638f..ff1416fb16063 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -379,7 +379,6 @@ fil_node_get_space_id( /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ -UNIV_INLINE fil_space_t* fil_space_get_by_name( /*==================*/ @@ -4785,7 +4784,7 @@ directory. We retry 100 times if os_file_readdir_next_file() returns -1. The idea is to read as much good data as we can and jump over bad data. @return 0 if ok, -1 if error even after the retries, 1 if at the end of the directory */ -static + int fil_file_readdir_next_file( /*=======================*/ @@ -4826,7 +4825,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(void) +fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)) /*===================================*/ { int ret; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 320b900d019f6..a7c68ddf8c9ea 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -284,7 +284,7 @@ static TYPELIB innodb_stats_method_typelib = { /** Possible values for system variables "innodb_checksum_algorithm" and "innodb_log_checksum_algorithm". */ -static const char* innodb_checksum_algorithm_names[] = { +const char* innodb_checksum_algorithm_names[] = { "CRC32", "STRICT_CRC32", "INNODB", @@ -296,7 +296,7 @@ static const char* innodb_checksum_algorithm_names[] = { /** Used to define an enumerate type of the system variables innodb_checksum_algorithm and innodb_log_checksum_algorithm. */ -static TYPELIB innodb_checksum_algorithm_typelib = { +TYPELIB innodb_checksum_algorithm_typelib = { array_elements(innodb_checksum_algorithm_names) - 1, "innodb_checksum_algorithm_typelib", innodb_checksum_algorithm_names, @@ -3016,7 +3016,7 @@ trx_is_started( /****************************************************************//** Update log_checksum_algorithm_ptr with a pointer to the function corresponding to a given checksum algorithm. */ -static + void innodb_log_checksum_func_update( /*============================*/ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 95011ae61250d..476a5c69e54d3 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -868,7 +868,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(void); +fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)=0); /*===================================*/ /*******************************************************************//** Returns TRUE if a single-table tablespace does not exist in the memory cache, diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index f60cfde1264f1..92482529763d0 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -489,6 +489,8 @@ extern ulong srv_innodb_stats_method; #ifdef UNIV_LOG_ARCHIVE extern ibool srv_log_archive_on; +extern ibool srv_archive_recovery; +extern ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ extern char* srv_file_flush_method_str; @@ -541,6 +543,11 @@ extern ulong srv_pass_corrupt_table; extern ulong srv_log_checksum_algorithm; +extern ibool srv_apply_log_only; + +extern ibool srv_backup_mode; +extern ibool srv_close_files; + extern my_bool srv_force_primary_key; /* Helper macro to support srv_pass_corrupt_table checks. If 'cond' is FALSE, diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index a42b8b8bc25c8..556aba62658e9 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -647,7 +647,7 @@ functions. */ #ifdef __WIN__ #define usleep(a) Sleep((a)/1000) -typedef ulint os_thread_ret_t; +typedef DWORD os_thread_ret_t; #define OS_THREAD_DUMMY_RETURN return(0) #else typedef void* os_thread_ret_t; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 092c2ed88dc18..35558b6f9f2b1 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -713,7 +713,6 @@ recv_synchronize_groups( /***********************************************************************//** Checks the consistency of the checkpoint info @return TRUE if ok */ -static ibool recv_check_cp_is_consistent( /*========================*/ @@ -743,7 +742,7 @@ recv_check_cp_is_consistent( /********************************************************//** Looks for the maximum consistent checkpoint from the log groups. @return error code or DB_SUCCESS */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t recv_find_max_checkpoint( /*=====================*/ @@ -3784,6 +3783,102 @@ recv_reset_log_files_for_backup( } #endif /* UNIV_HOTBACKUP */ +/******************************************************//** +Checks the 4-byte checksum to the trailer checksum field of a log +block. We also accept a log block in the old format before +InnoDB-3.23.52 where the checksum field contains the log block number. +@return TRUE if ok, or if the log block may be in the format of InnoDB +version predating 3.23.52 */ +UNIV_INTERN +ibool +log_block_checksum_is_ok_or_old_format( +/*===================================*/ + const byte* block) /*!< in: pointer to a log block */ +{ +#ifdef UNIV_LOG_DEBUG + return(TRUE); +#endif /* UNIV_LOG_DEBUG */ + + ulint block_checksum = log_block_get_checksum(block); + + if (UNIV_LIKELY(srv_log_checksum_algorithm == + SRV_CHECKSUM_ALGORITHM_NONE || + log_block_calc_checksum(block) == block_checksum)) { + + return(TRUE); + } + + if (srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { + + const char* algo = NULL; + + ib_logf(IB_LOG_LEVEL_ERROR, + "log block checksum mismatch: expected " ULINTPF ", " + "calculated checksum " ULINTPF, + block_checksum, + log_block_calc_checksum(block)); + + if (block_checksum == LOG_NO_CHECKSUM_MAGIC) { + + algo = "none"; + } else if (block_checksum == + log_block_calc_checksum_crc32(block)) { + + algo = "crc32"; + } else if (block_checksum == + log_block_calc_checksum_innodb(block)) { + + algo = "innodb"; + } + + if (algo) { + + const char* current_algo; + + current_algo = buf_checksum_algorithm_name( + (srv_checksum_algorithm_t) + srv_log_checksum_algorithm); + + ib_logf(IB_LOG_LEVEL_ERROR, + "current InnoDB log checksum type: %s, " + "detected log checksum type: %s", + current_algo, + algo); + } + + ib_logf(IB_LOG_LEVEL_FATAL, + "STRICT method was specified for innodb_log_checksum, " + "so we intentionally assert here."); + } + + ut_ad(srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_CRC32 || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB); + + if (block_checksum == LOG_NO_CHECKSUM_MAGIC || + block_checksum == log_block_calc_checksum_crc32(block) || + block_checksum == log_block_calc_checksum_innodb(block)) { + + return(TRUE); + } + + if (log_block_get_hdr_no(block) == block_checksum) { + + /* We assume the log block is in the format of + InnoDB version < 3.23.52 and the block is ok */ +#if 0 + fprintf(stderr, + "InnoDB: Scanned old format < InnoDB-3.23.52" + " log block number %lu\n", + log_block_get_hdr_no(block)); +#endif + return(TRUE); + } + + return(FALSE); +} + void recv_dblwr_t::add(byte* page) { pages.push_back(page); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 007b100285dd0..e8f4bfc8c2445 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -1010,7 +1010,6 @@ os_file_lock( #ifndef UNIV_HOTBACKUP /****************************************************************//** Creates the seek mutexes used in positioned reads and writes. */ -static void os_io_init_simple(void) /*===================*/ diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index f9c75ffe5760b..1a7f67e562ead 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -375,6 +375,8 @@ UNIV_INTERN ulong srv_read_ahead_threshold = 56; #ifdef UNIV_LOG_ARCHIVE UNIV_INTERN ibool srv_log_archive_on = FALSE; +UNIV_INTERN ibool srv_archive_recovery = 0; +UNIV_INTERN ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ /* This parameter is used to throttle the number of insert buffers that are @@ -534,6 +536,11 @@ UNIV_INTERN ulong srv_doublewrite_batch_size = 120; UNIV_INTERN ulong srv_replication_delay = 0; +UNIV_INTERN ibool srv_apply_log_only = FALSE; + +UNIV_INTERN ibool srv_backup_mode = FALSE; +UNIV_INTERN ibool srv_close_files = TRUE; + UNIV_INTERN ulong srv_pass_corrupt_table = 0; /* 0:disable 1:enable */ UNIV_INTERN ulong srv_log_checksum_algorithm = diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 6b1e8c39b3818..6381b5b837ffb 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -136,7 +136,7 @@ SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ UNIV_INTERN enum srv_shutdown_state srv_shutdown_state = SRV_SHUTDOWN_NONE; /** Files comprising the system tablespace */ -static os_file_t files[1000]; +os_file_t files[1000]; /** io_handler_thread parameters for thread identification */ static ulint n[SRV_MAX_N_IO_THREADS]; @@ -826,7 +826,7 @@ open_log_file( /*********************************************************************//** Creates or opens database data files and closes them. @return DB_SUCCESS or error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t open_or_create_data_files( /*======================*/ @@ -1367,12 +1367,15 @@ srv_undo_tablespace_open( /******************************************************************** Opens the configured number of undo tablespaces. @return DB_SUCCESS or error code */ -static dberr_t srv_undo_tablespaces_init( /*======================*/ ibool create_new_db, /*!< in: TRUE if new db being created */ + ibool backup_mode, /*!< in: TRUE disables reading + the system tablespace (used in + XtraBackup), FALSE is passed on + recovery. */ const ulint n_conf_tablespaces, /*!< in: configured undo tablespaces */ ulint* n_opened) /*!< out: number of UNDO @@ -2415,6 +2418,7 @@ innobase_start_or_create_for_mysql(void) err = srv_undo_tablespaces_init( create_new_db, + FALSE, srv_undo_tablespaces, &srv_undo_tablespaces_open); From ee1d08c115819e7111ddeccb01ec0534d15fd871 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 23 Oct 2016 00:10:37 +0000 Subject: [PATCH 124/295] Revert "Prepare XtraDB to be used with xtrabackup." This reverts commit de5646f1a9aaf45f1b43d98623b40c95fb98ebce. --- sql/CMakeLists.txt | 9 +-- storage/xtradb/btr/btr0btr.cc | 4 +- storage/xtradb/fil/fil0fil.cc | 5 +- storage/xtradb/handler/ha_innodb.cc | 6 +- storage/xtradb/include/fil0fil.h | 2 +- storage/xtradb/include/srv0srv.h | 7 -- storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0recv.cc | 99 +---------------------------- storage/xtradb/os/os0file.cc | 1 + storage/xtradb/srv/srv0srv.cc | 7 -- storage/xtradb/srv/srv0start.cc | 10 +-- 11 files changed, 17 insertions(+), 135 deletions(-) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 1cc83ea2bf13a..ec7e9207f38d5 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -77,12 +77,6 @@ IF(SSL_DEFINES) ADD_DEFINITIONS(${SSL_DEFINES}) ENDIF() -IF(WIN32) - SET(NT_SERVICE_SOURCES nt_servc.cc nt_servc.h ) -ELSE() - SET(NT_SERVICE_SOURCES) -ENDIF() - SET (SQL_SOURCE ../sql-common/client.c compat56.cc derror.cc des_key_file.cc discover.cc ../libmysql/errmsg.c field.cc field_conv.cc @@ -149,7 +143,6 @@ SET (SQL_SOURCE ${GEN_SOURCES} ${GEN_DIGEST_SOURCES} ${MYSYS_LIBWRAP_SOURCE} - ${NT_SERVICE_SOURCES} ) IF (CMAKE_SYSTEM_NAME MATCHES "Linux" OR @@ -179,7 +172,7 @@ TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} ${LIBSYSTEMD}) IF(WIN32) - SET(MYSQLD_SOURCE main.cc message.rc) + SET(MYSQLD_SOURCE main.cc nt_servc.cc nt_servc.h message.rc) TARGET_LINK_LIBRARIES(sql psapi) ELSE() SET(MYSQLD_SOURCE main.cc ${DTRACE_PROBES_ALL}) diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index bf6d850f3f08d..bce81f95eaddb 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -722,7 +722,7 @@ btr_root_fseg_validate( /**************************************************************//** Gets the root node of a tree and x- or s-latches it. @return root page, x- or s-latched */ - +static buf_block_t* btr_root_block_get( /*===============*/ @@ -1531,7 +1531,7 @@ btr_node_ptr_set_child_page_no( /************************************************************//** Returns the child page of a node pointer and x-latches it. @return child page, x-latched */ - +static buf_block_t* btr_node_ptr_get_child( /*===================*/ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index ff1416fb16063..93df92e6e638f 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -379,6 +379,7 @@ fil_node_get_space_id( /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ +UNIV_INLINE fil_space_t* fil_space_get_by_name( /*==================*/ @@ -4784,7 +4785,7 @@ directory. We retry 100 times if os_file_readdir_next_file() returns -1. The idea is to read as much good data as we can and jump over bad data. @return 0 if ok, -1 if error even after the retries, 1 if at the end of the directory */ - +static int fil_file_readdir_next_file( /*=======================*/ @@ -4825,7 +4826,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)) +fil_load_single_table_tablespaces(void) /*===================================*/ { int ret; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index a7c68ddf8c9ea..320b900d019f6 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -284,7 +284,7 @@ static TYPELIB innodb_stats_method_typelib = { /** Possible values for system variables "innodb_checksum_algorithm" and "innodb_log_checksum_algorithm". */ -const char* innodb_checksum_algorithm_names[] = { +static const char* innodb_checksum_algorithm_names[] = { "CRC32", "STRICT_CRC32", "INNODB", @@ -296,7 +296,7 @@ const char* innodb_checksum_algorithm_names[] = { /** Used to define an enumerate type of the system variables innodb_checksum_algorithm and innodb_log_checksum_algorithm. */ -TYPELIB innodb_checksum_algorithm_typelib = { +static TYPELIB innodb_checksum_algorithm_typelib = { array_elements(innodb_checksum_algorithm_names) - 1, "innodb_checksum_algorithm_typelib", innodb_checksum_algorithm_names, @@ -3016,7 +3016,7 @@ trx_is_started( /****************************************************************//** Update log_checksum_algorithm_ptr with a pointer to the function corresponding to a given checksum algorithm. */ - +static void innodb_log_checksum_func_update( /*============================*/ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 476a5c69e54d3..95011ae61250d 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -868,7 +868,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)=0); +fil_load_single_table_tablespaces(void); /*===================================*/ /*******************************************************************//** Returns TRUE if a single-table tablespace does not exist in the memory cache, diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 92482529763d0..f60cfde1264f1 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -489,8 +489,6 @@ extern ulong srv_innodb_stats_method; #ifdef UNIV_LOG_ARCHIVE extern ibool srv_log_archive_on; -extern ibool srv_archive_recovery; -extern ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ extern char* srv_file_flush_method_str; @@ -543,11 +541,6 @@ extern ulong srv_pass_corrupt_table; extern ulong srv_log_checksum_algorithm; -extern ibool srv_apply_log_only; - -extern ibool srv_backup_mode; -extern ibool srv_close_files; - extern my_bool srv_force_primary_key; /* Helper macro to support srv_pass_corrupt_table checks. If 'cond' is FALSE, diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 556aba62658e9..a42b8b8bc25c8 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -647,7 +647,7 @@ functions. */ #ifdef __WIN__ #define usleep(a) Sleep((a)/1000) -typedef DWORD os_thread_ret_t; +typedef ulint os_thread_ret_t; #define OS_THREAD_DUMMY_RETURN return(0) #else typedef void* os_thread_ret_t; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 35558b6f9f2b1..092c2ed88dc18 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -713,6 +713,7 @@ recv_synchronize_groups( /***********************************************************************//** Checks the consistency of the checkpoint info @return TRUE if ok */ +static ibool recv_check_cp_is_consistent( /*========================*/ @@ -742,7 +743,7 @@ recv_check_cp_is_consistent( /********************************************************//** Looks for the maximum consistent checkpoint from the log groups. @return error code or DB_SUCCESS */ -MY_ATTRIBUTE((nonnull, warn_unused_result)) +static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t recv_find_max_checkpoint( /*=====================*/ @@ -3783,102 +3784,6 @@ recv_reset_log_files_for_backup( } #endif /* UNIV_HOTBACKUP */ -/******************************************************//** -Checks the 4-byte checksum to the trailer checksum field of a log -block. We also accept a log block in the old format before -InnoDB-3.23.52 where the checksum field contains the log block number. -@return TRUE if ok, or if the log block may be in the format of InnoDB -version predating 3.23.52 */ -UNIV_INTERN -ibool -log_block_checksum_is_ok_or_old_format( -/*===================================*/ - const byte* block) /*!< in: pointer to a log block */ -{ -#ifdef UNIV_LOG_DEBUG - return(TRUE); -#endif /* UNIV_LOG_DEBUG */ - - ulint block_checksum = log_block_get_checksum(block); - - if (UNIV_LIKELY(srv_log_checksum_algorithm == - SRV_CHECKSUM_ALGORITHM_NONE || - log_block_calc_checksum(block) == block_checksum)) { - - return(TRUE); - } - - if (srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 || - srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || - srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { - - const char* algo = NULL; - - ib_logf(IB_LOG_LEVEL_ERROR, - "log block checksum mismatch: expected " ULINTPF ", " - "calculated checksum " ULINTPF, - block_checksum, - log_block_calc_checksum(block)); - - if (block_checksum == LOG_NO_CHECKSUM_MAGIC) { - - algo = "none"; - } else if (block_checksum == - log_block_calc_checksum_crc32(block)) { - - algo = "crc32"; - } else if (block_checksum == - log_block_calc_checksum_innodb(block)) { - - algo = "innodb"; - } - - if (algo) { - - const char* current_algo; - - current_algo = buf_checksum_algorithm_name( - (srv_checksum_algorithm_t) - srv_log_checksum_algorithm); - - ib_logf(IB_LOG_LEVEL_ERROR, - "current InnoDB log checksum type: %s, " - "detected log checksum type: %s", - current_algo, - algo); - } - - ib_logf(IB_LOG_LEVEL_FATAL, - "STRICT method was specified for innodb_log_checksum, " - "so we intentionally assert here."); - } - - ut_ad(srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_CRC32 || - srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB); - - if (block_checksum == LOG_NO_CHECKSUM_MAGIC || - block_checksum == log_block_calc_checksum_crc32(block) || - block_checksum == log_block_calc_checksum_innodb(block)) { - - return(TRUE); - } - - if (log_block_get_hdr_no(block) == block_checksum) { - - /* We assume the log block is in the format of - InnoDB version < 3.23.52 and the block is ok */ -#if 0 - fprintf(stderr, - "InnoDB: Scanned old format < InnoDB-3.23.52" - " log block number %lu\n", - log_block_get_hdr_no(block)); -#endif - return(TRUE); - } - - return(FALSE); -} - void recv_dblwr_t::add(byte* page) { pages.push_back(page); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index e8f4bfc8c2445..007b100285dd0 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -1010,6 +1010,7 @@ os_file_lock( #ifndef UNIV_HOTBACKUP /****************************************************************//** Creates the seek mutexes used in positioned reads and writes. */ +static void os_io_init_simple(void) /*===================*/ diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 1a7f67e562ead..f9c75ffe5760b 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -375,8 +375,6 @@ UNIV_INTERN ulong srv_read_ahead_threshold = 56; #ifdef UNIV_LOG_ARCHIVE UNIV_INTERN ibool srv_log_archive_on = FALSE; -UNIV_INTERN ibool srv_archive_recovery = 0; -UNIV_INTERN ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ /* This parameter is used to throttle the number of insert buffers that are @@ -536,11 +534,6 @@ UNIV_INTERN ulong srv_doublewrite_batch_size = 120; UNIV_INTERN ulong srv_replication_delay = 0; -UNIV_INTERN ibool srv_apply_log_only = FALSE; - -UNIV_INTERN ibool srv_backup_mode = FALSE; -UNIV_INTERN ibool srv_close_files = TRUE; - UNIV_INTERN ulong srv_pass_corrupt_table = 0; /* 0:disable 1:enable */ UNIV_INTERN ulong srv_log_checksum_algorithm = diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 6381b5b837ffb..6b1e8c39b3818 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -136,7 +136,7 @@ SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ UNIV_INTERN enum srv_shutdown_state srv_shutdown_state = SRV_SHUTDOWN_NONE; /** Files comprising the system tablespace */ -os_file_t files[1000]; +static os_file_t files[1000]; /** io_handler_thread parameters for thread identification */ static ulint n[SRV_MAX_N_IO_THREADS]; @@ -826,7 +826,7 @@ open_log_file( /*********************************************************************//** Creates or opens database data files and closes them. @return DB_SUCCESS or error code */ -MY_ATTRIBUTE((nonnull, warn_unused_result)) +static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t open_or_create_data_files( /*======================*/ @@ -1367,15 +1367,12 @@ srv_undo_tablespace_open( /******************************************************************** Opens the configured number of undo tablespaces. @return DB_SUCCESS or error code */ +static dberr_t srv_undo_tablespaces_init( /*======================*/ ibool create_new_db, /*!< in: TRUE if new db being created */ - ibool backup_mode, /*!< in: TRUE disables reading - the system tablespace (used in - XtraBackup), FALSE is passed on - recovery. */ const ulint n_conf_tablespaces, /*!< in: configured undo tablespaces */ ulint* n_opened) /*!< out: number of UNDO @@ -2418,7 +2415,6 @@ innobase_start_or_create_for_mysql(void) err = srv_undo_tablespaces_init( create_new_db, - FALSE, srv_undo_tablespaces, &srv_undo_tablespaces_open); From 0c925aa9356ee9d31283510c2420d1b5f21f5c9c Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 23 Oct 2016 18:47:44 +0300 Subject: [PATCH 125/295] MDEV-11097 - Update the list of unstable tests --- mysql-test/unstable-tests | 89 ++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 2dbaeeebc0bd6..3e25115599fdc 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,78 +23,66 @@ # ############################################################################## -main.bootstrap : Modified on 2016-06-18 (MDEV-9969) main.create_delayed : MDEV-10605 - failed with timeout -main.create_or_replace : Modified on 2016-06-23 (MDEV-9728) -main.ctype_recoding : Modified on 2016-06-10 (MDEV-10181) -main.ctype_utf8 : Modified on 2016-06-21 (merge) -main.ctype_utf8mb4 : Modified on 2016-06-21 (merge) -main.events_1 : Modified on 2016-06-21 (MDEV-9524) +main.ctype_utf32 : Modified on 2016-09-27 (merge) main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_in : Modified on 2016-06-20 (MDEV-10020) main.func_math : Modified on 2016-08-10 (merge) main.func_misc : Modified on 2016-08-10 (merge) -main.grant2 : Modified on 2016-07-18 (MDEV-8569) -main.help : Modified on 2016-06-21 (MDEV-9524) +main.group_min_max_innodb : Modified on 2016-08-25 (MDEV-10595) main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout -main.index_merge_innodb : MDEV-7142 - sporadic wrong execution plan +main.index_merge_myisam : Modified on 2016-09-05 (include file changed) +main.index_merge_innodb : Modified on 2016-09-05 (MDEV-7142) main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.insert_innodb : Modified on 2016-06-14 (merge from upstream) main.loaddata : Modified on 2016-08-10 (merge) -main.locale : Modified on 2016-06-21 (merge) main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.multi_update : Modified on 2016-06-20 (MDEV-5973) main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) main.mysqlcheck : Modified on 2016-08-10 (merge) main.mysqldump : MDEV-10512 - sporadic assertion failure +main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqltest : MDEV-9269 - fails on Alpha main.named_pipe : Modified on 2016-08-02 (MDEV-10383) -main.openssl_1 : Modified on 2016-07-11 (MDEV-10211) -main.parser : Modified on 2016-06-21 (merge) main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.ps_1general : Modified on 2016-07-12 (merge) main.range : Modified on 2016-08-10 (merge) main.range_mrr_icp : Modified on 2016-08-10 (merge) main.query_cache : MDEV-10611 - sporadic mutex problem -main.shutdown : MDEV-10612 - sporadic crashes +main.shutdown : MDEV-10563 - sporadic crashes main.sp-prelocking : Modified on 2016-08-10 (merge) main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl : MDEV-10211 - different ciphers on some platforms -main.ssl_ca : Modified on 2016-07-11 (MDEV-10211) -main.ssl_compress : Modified on 2016-07-11 (MDEV-10211) -main.ssl_timeout : Modified on 2016-07-11 (MDEV-10211) +main.ssl_compress : MDEV-11110 - valgrind failures main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results -main.status_user : Modified on 2016-06-20 (MDEV-8633) main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.temp_table : Modified on 2016-06-18 (MDEV-8569) main.type_date : Modified on 2016-08-10 (merge) -main.type_datetime : Modified on 2016-06-16 (MDEV-9374) +main.type_uint : Modified on 2016-09-27 (merge) main.view : Modified on 2016-08-10 (merge) main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) #---------------------------------------------------------------- -archive.archive-big : MDEV-10615 - table is marked as crashed -archive.discover : MDEV-10510 - table is marked as crashed +archive.archive-big : MDEV-10615 - table is marked as crashed +archive.discover : MDEV-10510 - table is marked as crashed +archive.mysqlhotcopy_archive : MDEV-10995 - test hangs on debug build #---------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed -binlog.binlog_dmls_on_tmp_tables_readonly : New test, added on 2016-05-04 (upstream) binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results -connect.jdbc : New test, added on 2016-07-15 -connect.jdbc-new : New test, added on 2016-07-14 -connect.jdbc-oracle : New test, added on 2016-07-13 -connect.jdbc-postgresql : New test, added on 2016-07-13 + +#---------------------------------------------------------------- + +engines/rr_trx.* : MDEV-10998 - tests not maintained + +#---------------------------------------------------------------- + +extra/binlog_tests.database : Modified on 2016-10-21 (Upstream MIPS test fixes) #---------------------------------------------------------------- @@ -105,21 +93,19 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- -funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge) -funcs_1.processlist_priv_ps : Include file modified on 2016-07-12 (merge) +funcs_2/charset.* : MDEV-10999 - test not maintained #---------------------------------------------------------------- innodb.binlog_consistent : MDEV-10618 - Server fails to start innodb.innodb-alter-table : MDEV-10619 - Testcase timeout innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) -innodb.innodb_corrupt_bit : Modified on 2016-06-21 (merge) innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb-fk-warnings : Modified on 2016-07-18 (MDEV-8569) -innodb.innodb-fkcheck : Modified on 2016-06-13 (MDEV-10083) +innodb.innodb_bug54044 : Modified on 2016-09-27 (merge) innodb.innodb_monitor : MDEV-10939 - Testcase timeout innodb.innodb-wl5522 : rdiff file modified on 2016-08-10 (merge) innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption +innodb.system_tables : Added on 2016-09-23 (MDEV-10775) #---------------------------------------------------------------- @@ -144,22 +130,16 @@ parts.partition_int_myisam : MDEV-10621 - Testcase timeout #---------------------------------------------------------------- -perfschema.digest_table_full : Modified on 2016-06-21 (merge) perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown -perfschema.rpl_gtid_func : Modified on 2016-06-21 (merge) -perfschema.sizing_low : Modified on 2016-04-26 (5.6.30 merge) perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match -perfschema.start_server_low_digest : Modified on 2016-06-21 (merge) -perfschema.statement_digest : Modified on 2016-06-21 (merge) -perfschema.statement_digest_consumers : Modified on 2016-06-21 (merge) -perfschema.statement_digest_long_query : Modified on 2016-06-21 (merge) -perfschema.table_name : New test, added on 2016-04-26 (5.6.30 merge) + +perfschema_stress.* : MDEV-10996 - tests not maintained #---------------------------------------------------------------- -plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url +plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11112 - valgrind warnings plugins.pam : Modified on 2016-08-03 (MDEV-7329) plugins.pam_cleartext : Modified on 2016-08-03 plugins.server_audit : MDEV-9562 - crashes on sol10-sparc @@ -167,11 +147,6 @@ plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc #---------------------------------------------------------------- -roles.rpl_grant_revoke_current_role-8638 : New test, added on 2016-06-20 (MDEV-8638) -roles.set_role-9614 : New test, added on 2016-05-30 (MDEV-9614) - -#---------------------------------------------------------------- - rpl.last_insert_id : MDEV-10625 - warnings in error log rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips @@ -180,11 +155,11 @@ rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips +rpl.rpl_drop_db : Modified on 2016-10-21 (Upstream MIPS test fixes) rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log -rpl.rpl_ignore_table : Modified on 2016-06-22 rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x @@ -204,6 +179,8 @@ rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha +rpl/extra/rpl_tests.* : MDEV-10994 - tests not maintained + #---------------------------------------------------------------- spider.* : MDEV-9329 - tests are too memory-consuming @@ -217,6 +194,10 @@ spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x #---------------------------------------------------------------- +sphinx.* : MDEV-10747 - tests are not run in buildbot, they can't be stable + +#---------------------------------------------------------------- + stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- @@ -232,11 +213,14 @@ tokudb.background_job_manager : MDEV-10327 - Assertion failure on server tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan tokudb.* : MDEV-9891 - massive crashes on shutdown tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown +tokudb_backup.* : MDEV-11001 - tests don't work tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown tokudb_parts.* : MDEV-9891 - massive crashes on shutdown -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown, also modified on 2016-06-10 (Merge) +tokudb_rpl_suites.* : MDEV-11001 - tests don't work +tokudb_sys_vars.* : MDEV-11001 - tests don't work +rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown tokudb/tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown tokudb/tokudb_backup.* : MDEV-9891 - massive crashes on shutdown tokudb/tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown @@ -250,7 +234,6 @@ unit.ma_test_loghandler : MDEV-10638 - record read not ok #---------------------------------------------------------------- -vcol.charsets : Added on 2016-06-23 vcol.not_supported : MDEV-10639 - Testcase timeout vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout From 9401e6befd6c00ab07a13a5a6b2ff5bb208002b8 Mon Sep 17 00:00:00 2001 From: Hyeonseok Oh Date: Mon, 24 Oct 2016 14:58:41 +0900 Subject: [PATCH 126/295] Remove unnecessary semicolons --- storage/innobase/handler/ha_innodb.cc | 2 +- storage/xtradb/buf/buf0flu.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7ba54a1c360a6..213a9eabaab64 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13926,7 +13926,7 @@ ha_innobase::check( if (!dict_index_is_clust(index)) { prebuilt->index_usable = FALSE; row_mysql_lock_data_dictionary(prebuilt->trx); - dict_set_corrupted(index, prebuilt->trx, "dict_set_index_corrupted");; + dict_set_corrupted(index, prebuilt->trx, "dict_set_index_corrupted"); row_mysql_unlock_data_dictionary(prebuilt->trx); }); diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 873edec62b494..76358b8f8f4c9 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -2222,7 +2222,7 @@ buf_flush_single_page_from_LRU( if (ready) { bool evict_zip; - evict_zip = !buf_LRU_evict_from_unzip_LRU(buf_pool);; + evict_zip = !buf_LRU_evict_from_unzip_LRU(buf_pool); freed = buf_LRU_free_page(bpage, evict_zip); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 320b900d019f6..2ed99a4a26e33 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -14663,7 +14663,7 @@ ha_innobase::check( if (!dict_index_is_clust(index)) { prebuilt->index_usable = FALSE; row_mysql_lock_data_dictionary(prebuilt->trx); - dict_set_corrupted(index, prebuilt->trx, "dict_set_index_corrupted");; + dict_set_corrupted(index, prebuilt->trx, "dict_set_index_corrupted"); row_mysql_unlock_data_dictionary(prebuilt->trx); }); From ec5bd0d3855aa49afc3e356aa18b7edea3b7a18b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Oct 2016 09:25:36 +0300 Subject: [PATCH 127/295] MDEV-10969: innochecksum dumps core for some .ibd files due to floating point exception Check that index has pages before using number of pages in divide operation. --- extra/innochecksum.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 1ebcbf198efa7..e128271f4b817 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -639,10 +639,18 @@ print_stats() "\t#bytes_per_page\n"); for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { const per_index_stats& index = it->second; + longlong recs_per_page = index.total_n_recs; + longlong bytes_per_page = index.total_data_bytes; + if (index.total_n_recs && index.pages) { + recs_per_page = index.total_n_recs / index.pages; + } + if (index.total_data_bytes && index.pages) { + bytes_per_page = index.total_data_bytes / index.pages; + } printf("%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", it->first, index.pages, index.leaf_pages, - index.total_n_recs / index.pages, - index.total_data_bytes / index.pages); + recs_per_page, + bytes_per_page); } printf("\n"); printf("index_id\tpage_data_bytes_histgram(empty,...,oversized)\n"); From 1bfa37a79c78998e11c79227089864358d05bb4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Oct 2016 16:55:36 +0300 Subject: [PATCH 128/295] Add more information if encryption information is already stored for tablespace but page0 is not yet read. --- storage/innobase/fil/fil0fil.cc | 9 ++++++++- storage/xtradb/fil/fil0fil.cc | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index f1fdb0f281eeb..a88cf6e65bda6 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -7286,7 +7286,14 @@ fil_space_get_crypt_data( crypt_data = space->crypt_data; - ut_ad(space->page_0_crypt_read); + if (!space->page_0_crypt_read) { + ib_logf(IB_LOG_LEVEL_WARN, + "Space %lu name %s contains encryption %d information for key_id %d but page0 is not read.", + space->id, + space->name, + space->crypt_data ? space->crypt_data->encryption : 0, + space->crypt_data ? space->crypt_data->key_id : 0); + } } return(crypt_data); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 93df92e6e638f..440b95daa187d 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -7390,7 +7390,14 @@ fil_space_get_crypt_data( crypt_data = space->crypt_data; - ut_ad(space->page_0_crypt_read); + if (!space->page_0_crypt_read) { + ib_logf(IB_LOG_LEVEL_WARN, + "Space %lu name %s contains encryption %d information for key_id %d but page0 is not read.", + space->id, + space->name, + space->crypt_data ? space->crypt_data->encryption : 0, + space->crypt_data ? space->crypt_data->key_id : 0); + } } return(crypt_data); From 4edd4ad69807c11a2016ac1477805739270a8ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Oct 2016 22:25:54 +0300 Subject: [PATCH 129/295] MDEV-10970: Crash while loading mysqldump backup when InnoDB encryption is enabled Follow-up: Make sure we do not reference NULL-pointer when space is being dropped and does not contain any nodes. --- storage/innobase/fil/fil0fil.cc | 7 ++++++- storage/xtradb/fil/fil0fil.cc | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index a88cf6e65bda6..7a50ccd22711c 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1001,8 +1001,13 @@ fil_mutex_enter_and_prepare_for_io( does not exist, we handle the situation in the function which called this function */ - if (!space || UT_LIST_GET_FIRST(space->chain)->open) { + if (!space) { + return; + } + + fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + if (!node || node->open) { return; } diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 440b95daa187d..803f3a81cbdac 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1002,8 +1002,13 @@ fil_mutex_enter_and_prepare_for_io( /* If the file is already open, no need to do anything; if the space does not exist, we handle the situation in the function which called this function */ + if (!space) { + return; + } + + fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - if (!space || UT_LIST_GET_FIRST(space->chain)->open) { + if (!node || node->open) { return; } From 3321f1adc74b54e7534000c06eeca166730ccc4a Mon Sep 17 00:00:00 2001 From: Don Lewis Date: Tue, 21 Jun 2016 13:35:59 +1000 Subject: [PATCH 130/295] MDEV-5944: Compile fix for OQGRAPH with LLVM Clang/LLVM has more strict schemantics than gcc. This patch quantifies the namesspace such that it will compile using clang. --- storage/oqgraph/graphcore.cc | 2 +- storage/oqgraph/oqgraph_shim.h | 48 +++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/storage/oqgraph/graphcore.cc b/storage/oqgraph/graphcore.cc index 4346b94805c7e..7c8ca53c0969c 100644 --- a/storage/oqgraph/graphcore.cc +++ b/storage/oqgraph/graphcore.cc @@ -485,7 +485,7 @@ namespace open_query optional oqgraph_share::find_vertex(VertexID id) const { - return ::boost::find_vertex(id, g); + return oqgraph3::find_vertex(id, g); } #if 0 diff --git a/storage/oqgraph/oqgraph_shim.h b/storage/oqgraph/oqgraph_shim.h index af240b88ebdcb..004d7f0f7c509 100644 --- a/storage/oqgraph/oqgraph_shim.h +++ b/storage/oqgraph/oqgraph_shim.h @@ -274,6 +274,33 @@ namespace boost }; #endif + template<> + struct property_map + { + typedef void type; + typedef oqgraph3::edge_weight_property_map const_type; + }; + + template<> + struct property_map + { + typedef void type; + typedef oqgraph3::vertex_index_property_map const_type; + }; + + template<> + struct property_map + { + typedef void type; + typedef oqgraph3::edge_index_property_map const_type; + }; + +} + +namespace oqgraph3 +{ + using namespace boost; + inline graph_traits::vertex_descriptor source( const graph_traits::edge_descriptor& e, @@ -401,27 +428,6 @@ namespace boost return count; } - template<> - struct property_map - { - typedef void type; - typedef oqgraph3::edge_weight_property_map const_type; - }; - - template<> - struct property_map - { - typedef void type; - typedef oqgraph3::vertex_index_property_map const_type; - }; - - template<> - struct property_map - { - typedef void type; - typedef oqgraph3::edge_index_property_map const_type; - }; - inline property_map< oqgraph3::graph, edge_weight_t>::const_type::reference From ba11dd69fee7b82edf4e6afbb13e3fa94cd885ca Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 25 Oct 2016 12:21:53 +0000 Subject: [PATCH 131/295] MDEV-11127 : Fix innochecksum to work with large files on Windows. - don't use stat() for file size, it doesn not handle large size use GetFileSizeEx() instead - don't use lseek(), it can't handle large files, use _lseeki64() instead. - Also, switch off OS file buffering for innochecksum on Windows, to avoid thrashing file cache. --- extra/innochecksum.cc | 55 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 6018a4884ea5b..97d47b4563af7 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -243,10 +243,9 @@ int main(int argc, char **argv) time_t lastt; /* last time */ ulint oldcsum, oldcsumfield, csum, csumfield, crc32, logseq, logseqfield; /* ulints for checksum storage */ - struct stat st; /* for stat, if you couldn't guess */ unsigned long long int size; /* size of file (has to be 64 bits) */ ulint pages; /* number of pages in file */ - off_t offset= 0; + unsigned long long offset= 0; int fd; printf("InnoDB offline file checksum utility.\n"); @@ -269,6 +268,47 @@ int main(int argc, char **argv) goto error; } +#ifdef _WIN32 + /* Switch off OS file buffering for the file. */ + + HANDLE h = CreateFile(filename, GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE, 0, + OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, 0); + + if (!h) + { + fprintf(stderr, "Error; cant open file\n"); + goto error; + } + + if (!GetFileSizeEx(h, (LARGE_INTEGER *)&size)) + { + fprintf(stderr, "Error; GetFileSize() failed\n"); + goto error; + } + + fd = _open_osfhandle ((intptr_t) h, _O_RDONLY); + if (fd < 0) + { + fprintf(stderr, "Error; _open_osfhandle() failed\n"); + goto error; + } + + f = _fdopen(fd, "rb"); + if (!f) + { + fprintf(stderr, "Error; fdopen() failed\n"); + goto error; + } + + /* + Disable stdio buffering (FILE_FLAG_NO_BUFFERING requires properly IO buffers + which stdio does not guarantee. + */ + setvbuf(f, NULL, _IONBF, 0); + +#else + struct stat st; /* stat the file to get size and page count */ if (stat(filename, &st)) { @@ -279,6 +319,8 @@ int main(int argc, char **argv) /* Open the file for reading */ f= fopen(filename, "rb"); +#endif + if (f == NULL) { fprintf(stderr, "Error; %s cannot be opened", filename); @@ -323,7 +365,7 @@ int main(int argc, char **argv) } else if (verbose) { - printf("file %s = %llu bytes (%lu pages)...\n", filename, size, pages); + printf("file %s = %llu bytes (%lu pages)...\n", filename, size, (ulong)pages); if (do_one_page) printf("InnoChecksum; checking page %lu\n", do_page); else @@ -348,9 +390,12 @@ int main(int argc, char **argv) goto error; } - offset= (off_t)start_page * (off_t)physical_page_size; - + offset= (ulonglong)start_page * (ulonglong)physical_page_size; +#ifdef _WIN32 + if (_lseeki64(fd, offset, SEEK_SET) != offset) +#else if (lseek(fd, offset, SEEK_SET) != offset) +#endif { perror("Error; Unable to seek to necessary offset"); goto error; From 39dceaae607e2c9f53146d5b23f8dee330643cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 9 Oct 2016 12:09:44 +0200 Subject: [PATCH 132/295] MDEV-10983: TokuDB does not compile on OS X 10.12 Make use of a different function to get the current tid. Additionally, librt doesn't exist on OS X. Use System library instead. --- .../PerconaFT/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/PerconaFT/portability/portability.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake index 4c5004cd6a538..883f35041e2fd 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake @@ -97,7 +97,7 @@ if (NOT HAVE_BACKTRACE_WITHOUT_EXECINFO) endif () endif () -if(HAVE_CLOCK_REALTIME) +if(HAVE_CLOCK_REALTIME AND (NOT APPLE)) list(APPEND EXTRA_SYSTEM_LIBS rt) else() list(APPEND EXTRA_SYSTEM_LIBS System) @@ -109,6 +109,8 @@ check_function_exists(pthread_rwlockattr_setkind_np HAVE_PTHREAD_RWLOCKATTR_SETK ## check for the right way to yield using pthreads check_function_exists(pthread_yield HAVE_PTHREAD_YIELD) check_function_exists(pthread_yield_np HAVE_PTHREAD_YIELD_NP) +## check if we have pthread_threadid_np() (i.e. osx) +check_function_exists(pthread_threadid_np HAVE_PTHREAD_THREADID_NP) ## check if we have pthread_getthreadid_np() (i.e. freebsd) check_function_exists(pthread_getthreadid_np HAVE_PTHREAD_GETTHREADID_NP) check_function_exists(sched_getcpu HAVE_SCHED_GETCPU) diff --git a/storage/tokudb/PerconaFT/portability/portability.cc b/storage/tokudb/PerconaFT/portability/portability.cc index ba9f8d48ed5dc..19f445a85d7f4 100644 --- a/storage/tokudb/PerconaFT/portability/portability.cc +++ b/storage/tokudb/PerconaFT/portability/portability.cc @@ -63,6 +63,9 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #if defined(HAVE_SYS_SYSCTL_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif #if defined(HAVE_PTHREAD_NP_H) # include #endif @@ -102,7 +105,11 @@ toku_os_getpid(void) { int toku_os_gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; // Used for instrumentation so overflow is ok here. +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/PerconaFT/portability/tests/test-xid.cc b/storage/tokudb/PerconaFT/portability/tests/test-xid.cc index 9ee68906bb37a..71736f898ef87 100644 --- a/storage/tokudb/PerconaFT/portability/tests/test-xid.cc +++ b/storage/tokudb/PerconaFT/portability/tests/test-xid.cc @@ -51,11 +51,18 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #if defined(HAVE_PTHREAD_NP_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif // since we implement the same thing here as in toku_os_gettid, this test // is pretty pointless static int gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/PerconaFT/portability/toku_config.h.in b/storage/tokudb/PerconaFT/portability/toku_config.h.in index 1a34bf1ef45a7..18f6779796fe5 100644 --- a/storage/tokudb/PerconaFT/portability/toku_config.h.in +++ b/storage/tokudb/PerconaFT/portability/toku_config.h.in @@ -87,6 +87,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #cmakedefine HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP 1 #cmakedefine HAVE_PTHREAD_YIELD 1 #cmakedefine HAVE_PTHREAD_YIELD_NP 1 +#cmakedefine HAVE_PTHREAD_THREADID_NP 1 #cmakedefine HAVE_PTHREAD_GETTHREADID_NP 1 #cmakedefine PTHREAD_YIELD_RETURNS_INT 1 From 1daf746e31e38a3ec1cdcb9427153b65f744dcda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 25 Oct 2016 16:34:22 +0300 Subject: [PATCH 133/295] Add tokuftdump man page The man page was already present in the debian release of MariaDB 10.0. --- man/CMakeLists.txt | 2 +- tokuftdump.1 | 237 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 tokuftdump.1 diff --git a/man/CMakeLists.txt b/man/CMakeLists.txt index c4383b31a1787..c6163fee5372d 100644 --- a/man/CMakeLists.txt +++ b/man/CMakeLists.txt @@ -22,7 +22,7 @@ SET(MAN1_SERVER innochecksum.1 my_print_defaults.1 myisam_ftdump.1 myisamchk.1 mysql_tzinfo_to_sql.1 mysql_upgrade.1 mysql_zap.1 mysqld_multi.1 mysqld_safe.1 mysqldumpslow.1 mysqlhotcopy.1 mysqltest.1 perror.1 replace.1 resolve_stack_dump.1 - resolveip.1 mysqlbug.1) + resolveip.1 mysqlbug.1 tokuftdump.1) SET(MAN8_SERVER mysqld.8) SET(MAN1_CLIENT msql2mysql.1 mysql.1 mysql_find_rows.1 mysql_waitpid.1 mysqlaccess.1 mysqladmin.1 mysqlbinlog.1 mysqlcheck.1 diff --git a/tokuftdump.1 b/tokuftdump.1 new file mode 100644 index 0000000000000..3d9faae30cacc --- /dev/null +++ b/tokuftdump.1 @@ -0,0 +1,237 @@ +'\" t +.\" +.TH "\FBTOKUFTDUMP\FR" "1" "04/07/2016" "MariaDB 10\&.0" "MariaDB Database System" +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l +.\" ----------------------------------------------------------------- +.\" * MAIN CONTENT STARTS HERE * +.\" ----------------------------------------------------------------- +.\" tokuftdump +.\" upgrading MySQL +.SH "NAME" +tokuftdump \- look into the fractal tree file +.SH "SYNOPSIS" +.HP \w'\fBtokuftdump\ [\fR\fB\fIoptions\fR\fR\fB]\fR\ 'u +\fBtokuftdump [\fR\fB\fIoptions\fR\fR\fB]\fR +.SH "DESCRIPTION" +.PP +\fBtokuftdump\fR +Investigates and diagnoses the fractal tree\&. +.PP +\fBtokuftdump\fR +supports the following options for processing option files\&. +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: interactive option +.\" interactive option: tokuftdump +\fB\-\-interactive\fR +.sp +Interactive\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: support option +.\" support option: tokuftdump +\fB\-\-support \fI/path/to/fractal-tree/file\fR +.sp +An interactive way to see what messages and/or switch between FTs\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: json option +.\" json option: tokuftdump +\fB\-\-json \fI/path/to/fractal-tree/file [output_json_file]\fR +.sp +If the output json file is left empty, FT\&.json will be created automatically\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: nodata option +.\" nodata option: tokuftdump +\fB\-\-nodata\fR +.sp +Nodata\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: dumpdata option +.\" dumpdata option: tokuftdump +\fB\-\-dumpdata = \fR\fB\fI0|1\fR\fR +.sp +Dumpdata\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: header option +.\" header option: tokuftdump +\fB\-\-header\fR +.sp +Header\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: rootnode option +.\" rootnode option: tokuftdump +\fB\-\-rootnode\fR +.sp +Rootnode\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: node option +.\" node option: tokuftdump +\fB\-\-node \fIN\fR +.sp +Node\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: fragmentation option +.\" fragmentation option: tokuftdump +\fB\-\-fragmentation\fR +.sp +Fragmentation\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: garbage option +.\" garbage option: tokuftdump +\fB\-\-garbage\fR +.sp +Garbage\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: tsv option +.\" tsv option: tokuftdump +\fB\-\-tsv\fR +.sp +TSV\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: translation-table option +.\" translation-table option: tokuftdump +\fB\-\-translation\-table\fR +.sp +Translation table\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" tokuftdump: summary option +.\" summary option: tokuftdump +\fB\-\-summary\fR +.sp +Provide summary info\&. +.RE +.SH "COPYRIGHT" +.br +.PP +Copyright 2016 MariaDB Foundation +.PP +This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. +.PP +This documentation is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +.PP +You should have received a copy of the GNU General Public License along with the program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA or see http://www.gnu.org/licenses/. +.sp +.SH "SEE ALSO" +For more information, please refer to the MariaDB Knowledge Base, available online at https://mariadb.com/kb/ +.SH AUTHOR +MariaDB Foundation (http://www.mariadb.org/). From ed3998ae7cc286860670bc9a285aeb99c5edcced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 25 Oct 2016 15:46:10 +0200 Subject: [PATCH 134/295] Revert "Add tokuftdump man page" This reverts commit 1daf746e31e38a3ec1cdcb9427153b65f744dcda. Removed temporarily to make sure there are no legal problems. --- man/CMakeLists.txt | 2 +- tokuftdump.1 | 237 --------------------------------------------- 2 files changed, 1 insertion(+), 238 deletions(-) delete mode 100644 tokuftdump.1 diff --git a/man/CMakeLists.txt b/man/CMakeLists.txt index c6163fee5372d..c4383b31a1787 100644 --- a/man/CMakeLists.txt +++ b/man/CMakeLists.txt @@ -22,7 +22,7 @@ SET(MAN1_SERVER innochecksum.1 my_print_defaults.1 myisam_ftdump.1 myisamchk.1 mysql_tzinfo_to_sql.1 mysql_upgrade.1 mysql_zap.1 mysqld_multi.1 mysqld_safe.1 mysqldumpslow.1 mysqlhotcopy.1 mysqltest.1 perror.1 replace.1 resolve_stack_dump.1 - resolveip.1 mysqlbug.1 tokuftdump.1) + resolveip.1 mysqlbug.1) SET(MAN8_SERVER mysqld.8) SET(MAN1_CLIENT msql2mysql.1 mysql.1 mysql_find_rows.1 mysql_waitpid.1 mysqlaccess.1 mysqladmin.1 mysqlbinlog.1 mysqlcheck.1 diff --git a/tokuftdump.1 b/tokuftdump.1 deleted file mode 100644 index 3d9faae30cacc..0000000000000 --- a/tokuftdump.1 +++ /dev/null @@ -1,237 +0,0 @@ -'\" t -.\" -.TH "\FBTOKUFTDUMP\FR" "1" "04/07/2016" "MariaDB 10\&.0" "MariaDB Database System" -.\" ----------------------------------------------------------------- -.\" * set default formatting -.\" ----------------------------------------------------------------- -.\" disable hyphenation -.nh -.\" disable justification (adjust text to left margin only) -.ad l -.\" ----------------------------------------------------------------- -.\" * MAIN CONTENT STARTS HERE * -.\" ----------------------------------------------------------------- -.\" tokuftdump -.\" upgrading MySQL -.SH "NAME" -tokuftdump \- look into the fractal tree file -.SH "SYNOPSIS" -.HP \w'\fBtokuftdump\ [\fR\fB\fIoptions\fR\fR\fB]\fR\ 'u -\fBtokuftdump [\fR\fB\fIoptions\fR\fR\fB]\fR -.SH "DESCRIPTION" -.PP -\fBtokuftdump\fR -Investigates and diagnoses the fractal tree\&. -.PP -\fBtokuftdump\fR -supports the following options for processing option files\&. -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: interactive option -.\" interactive option: tokuftdump -\fB\-\-interactive\fR -.sp -Interactive\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: support option -.\" support option: tokuftdump -\fB\-\-support \fI/path/to/fractal-tree/file\fR -.sp -An interactive way to see what messages and/or switch between FTs\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: json option -.\" json option: tokuftdump -\fB\-\-json \fI/path/to/fractal-tree/file [output_json_file]\fR -.sp -If the output json file is left empty, FT\&.json will be created automatically\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: nodata option -.\" nodata option: tokuftdump -\fB\-\-nodata\fR -.sp -Nodata\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: dumpdata option -.\" dumpdata option: tokuftdump -\fB\-\-dumpdata = \fR\fB\fI0|1\fR\fR -.sp -Dumpdata\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: header option -.\" header option: tokuftdump -\fB\-\-header\fR -.sp -Header\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: rootnode option -.\" rootnode option: tokuftdump -\fB\-\-rootnode\fR -.sp -Rootnode\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: node option -.\" node option: tokuftdump -\fB\-\-node \fIN\fR -.sp -Node\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: fragmentation option -.\" fragmentation option: tokuftdump -\fB\-\-fragmentation\fR -.sp -Fragmentation\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: garbage option -.\" garbage option: tokuftdump -\fB\-\-garbage\fR -.sp -Garbage\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: tsv option -.\" tsv option: tokuftdump -\fB\-\-tsv\fR -.sp -TSV\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: translation-table option -.\" translation-table option: tokuftdump -\fB\-\-translation\-table\fR -.sp -Translation table\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} -.\" tokuftdump: summary option -.\" summary option: tokuftdump -\fB\-\-summary\fR -.sp -Provide summary info\&. -.RE -.SH "COPYRIGHT" -.br -.PP -Copyright 2016 MariaDB Foundation -.PP -This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. -.PP -This documentation is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -.PP -You should have received a copy of the GNU General Public License along with the program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA or see http://www.gnu.org/licenses/. -.sp -.SH "SEE ALSO" -For more information, please refer to the MariaDB Knowledge Base, available online at https://mariadb.com/kb/ -.SH AUTHOR -MariaDB Foundation (http://www.mariadb.org/). From d7dc03a26797f07625e8c44d2d1ac7f76e860bad Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 25 Oct 2016 17:01:37 +0200 Subject: [PATCH 135/295] 5.6.33-79.0 --- storage/xtradb/buf/buf0buf.cc | 4 +- storage/xtradb/buf/buf0dblwr.cc | 2 +- storage/xtradb/buf/buf0flu.cc | 5 + storage/xtradb/dict/dict0boot.cc | 4 + storage/xtradb/dict/dict0crea.cc | 583 ++++++++++++++++++++++ storage/xtradb/dict/dict0dict.cc | 158 ++++++ storage/xtradb/dict/dict0load.cc | 159 +++++- storage/xtradb/fil/fil0fil.cc | 2 + storage/xtradb/fts/fts0fts.cc | 31 ++ storage/xtradb/handler/ha_innodb.cc | 392 ++++++++++++++- storage/xtradb/handler/ha_innodb.h | 37 ++ storage/xtradb/handler/handler0alter.cc | 66 ++- storage/xtradb/handler/i_s.cc | 2 + storage/xtradb/handler/xtradb_i_s.cc | 350 +++++++++++++ storage/xtradb/handler/xtradb_i_s.h | 2 + storage/xtradb/include/data0type.h | 14 + storage/xtradb/include/data0type.ic | 16 + storage/xtradb/include/dict0boot.h | 32 ++ storage/xtradb/include/dict0crea.h | 91 ++++ storage/xtradb/include/dict0dict.h | 46 ++ storage/xtradb/include/dict0load.h | 29 ++ storage/xtradb/include/fts0fts.h | 10 + storage/xtradb/include/os0thread.h | 15 +- storage/xtradb/include/rem0types.h | 3 + storage/xtradb/include/row0mysql.h | 85 +++- storage/xtradb/include/srv0srv.h | 5 + storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0log.cc | 20 +- storage/xtradb/log/log0online.cc | 33 +- storage/xtradb/mach/mach0data.cc | 13 +- storage/xtradb/os/os0thread.cc | 24 +- storage/xtradb/rem/rem0rec.cc | 23 +- storage/xtradb/row/row0ftsort.cc | 2 +- storage/xtradb/row/row0log.cc | 14 +- storage/xtradb/row/row0merge.cc | 18 +- storage/xtradb/row/row0mysql.cc | 634 +++++++++++++++++++++++- storage/xtradb/row/row0sel.cc | 45 +- storage/xtradb/srv/srv0start.cc | 6 + 38 files changed, 2892 insertions(+), 85 deletions(-) diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 321a1d9f673a3..978d94f07ec76 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -4492,7 +4492,9 @@ buf_page_io_complete( recv_recover_page(TRUE, (buf_block_t*) bpage); } - if (uncompressed && !recv_no_ibuf_operations) { + if (uncompressed && !recv_no_ibuf_operations + && fil_page_get_type(frame) == FIL_PAGE_INDEX + && page_is_leaf(frame)) { buf_block_t* block; ibool update_ibuf_bitmap; diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index f4d1c637e3e6f..3c12d6da73f3f 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -521,7 +521,7 @@ buf_dblwr_process() if (buf_page_is_corrupted(true, read_buf, zip_size)) { fprintf(stderr, - "InnoDB: Warning: database page" + "InnoDB: Database page" " corruption or a failed\n" "InnoDB: file read of" " space %lu page %lu.\n" diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 14a5fbde7e820..5dd2efcf0c36b 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -2568,6 +2568,11 @@ page_cleaner_sleep_if_needed( ulint next_loop_time) /*!< in: time when next loop iteration should start */ { + /* No sleep if we are cleaning the buffer pool during the shutdown + with everything else finished */ + if (srv_shutdown_state == SRV_SHUTDOWN_FLUSH_PHASE) + return; + ulint cur_time = ut_time_ms(); if (next_loop_time > cur_time) { diff --git a/storage/xtradb/dict/dict0boot.cc b/storage/xtradb/dict/dict0boot.cc index 94a3af2852b39..c0bb0298bea72 100644 --- a/storage/xtradb/dict/dict0boot.cc +++ b/storage/xtradb/dict/dict0boot.cc @@ -272,6 +272,10 @@ dict_boot(void) ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_FOR_NAME == 2); ut_ad(DICT_NUM_COLS__SYS_FOREIGN_COLS == 4); ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_COLS == 6); + ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT == 3); + ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT == 5); + ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT_COLS == 3); + ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS == 5); mtr_start(&mtr); diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index a4fcf57c02844..9460ce5165977 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -38,6 +38,7 @@ Created 1/8/1996 Heikki Tuuri #include "que0que.h" #include "row0ins.h" #include "row0mysql.h" +#include "row0sel.h" #include "pars0pars.h" #include "trx0roll.h" #include "usr0sess.h" @@ -1790,6 +1791,135 @@ dict_create_or_check_sys_tablespace(void) return(err); } +/** Creates the zip_dict system table inside InnoDB +at server bootstrap or server start if it is not found or is +not of the right form. +@return DB_SUCCESS or error code */ +UNIV_INTERN +dberr_t +dict_create_or_check_sys_zip_dict(void) +{ + trx_t* trx; + my_bool srv_file_per_table_backup; + dberr_t err; + dberr_t sys_zip_dict_err; + dberr_t sys_zip_dict_cols_err; + + ut_a(srv_get_active_thread_type() == SRV_NONE); + + /* Note: The master thread has not been started at this point. */ + + sys_zip_dict_err = dict_check_if_system_table_exists( + "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); + sys_zip_dict_cols_err = dict_check_if_system_table_exists( + "SYS_ZIP_DICT_COLS", DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, + 1); + + if (sys_zip_dict_err == DB_SUCCESS && + sys_zip_dict_cols_err == DB_SUCCESS) + return (DB_SUCCESS); + + trx = trx_allocate_for_mysql(); + + trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); + + trx->op_info = "creating zip_dict and zip_dict_cols sys tables"; + + row_mysql_lock_data_dictionary(trx); + + /* Check which incomplete table definition to drop. */ + + if (sys_zip_dict_err == DB_CORRUPTION) { + ib_logf(IB_LOG_LEVEL_WARN, + "Dropping incompletely created " + "SYS_ZIP_DICT table."); + row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE); + } + if (sys_zip_dict_cols_err == DB_CORRUPTION) { + ib_logf(IB_LOG_LEVEL_WARN, + "Dropping incompletely created " + "SYS_ZIP_DICT_COLS table."); + row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE); + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Creating zip_dict and zip_dict_cols system tables."); + + /* We always want SYSTEM tables to be created inside the system + tablespace. */ + srv_file_per_table_backup = srv_file_per_table; + srv_file_per_table = 0; + + err = que_eval_sql( + NULL, + "PROCEDURE CREATE_SYS_ZIP_DICT_PROC () IS\n" + "BEGIN\n" + "CREATE TABLE SYS_ZIP_DICT(\n" + " ID INT UNSIGNED NOT NULL,\n" + " NAME CHAR(" + STRINGIFY_ARG(ZIP_DICT_MAX_NAME_LENGTH) + ") NOT NULL,\n" + " DATA BLOB NOT NULL\n" + ");\n" + "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_ID" + " ON SYS_ZIP_DICT (ID);\n" + "CREATE UNIQUE INDEX SYS_ZIP_DICT_NAME" + " ON SYS_ZIP_DICT (NAME);\n" + "CREATE TABLE SYS_ZIP_DICT_COLS(\n" + " TABLE_ID INT UNSIGNED NOT NULL,\n" + " COLUMN_POS INT UNSIGNED NOT NULL,\n" + " DICT_ID INT UNSIGNED NOT NULL\n" + ");\n" + "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_COLS_COMPOSITE" + " ON SYS_ZIP_DICT_COLS (TABLE_ID, COLUMN_POS);\n" + "END;\n", + FALSE, trx); + + if (err != DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Creation of SYS_ZIP_DICT and SYS_ZIP_DICT_COLS" + "has failed with error %lu. Tablespace is full. " + "Dropping incompletely created tables.", + (ulong) err); + + ut_a(err == DB_OUT_OF_FILE_SPACE + || err == DB_TOO_MANY_CONCURRENT_TRXS); + + row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE); + row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE); + + if (err == DB_OUT_OF_FILE_SPACE) { + err = DB_MUST_GET_MORE_FILE_SPACE; + } + } + + trx_commit_for_mysql(trx); + + row_mysql_unlock_data_dictionary(trx); + + trx_free_for_mysql(trx); + + srv_file_per_table = srv_file_per_table_backup; + + if (err == DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_INFO, + "zip_dict and zip_dict_cols system tables created."); + } + + /* Note: The master thread has not been started at this point. */ + /* Confirm and move to the non-LRU part of the table LRU list. */ + + sys_zip_dict_err = dict_check_if_system_table_exists( + "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); + ut_a(sys_zip_dict_err == DB_SUCCESS); + sys_zip_dict_cols_err = dict_check_if_system_table_exists( + "SYS_ZIP_DICT_COLS", + DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, 1); + ut_a(sys_zip_dict_cols_err == DB_SUCCESS); + + return(err); +} + /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -1843,3 +1973,456 @@ dict_create_add_tablespace_to_dictionary( return(error); } + +/** Add a single compression dictionary definition to the SYS_ZIP_DICT +InnoDB system table. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_add_zip_dict( + const char* name, /*!< in: dict name */ + ulint name_len, /*!< in: dict name length */ + const char* data, /*!< in: dict data */ + ulint data_len, /*!< in: dict data length */ + trx_t* trx) /*!< in/out: transaction */ +{ + ut_ad(name); + ut_ad(data); + + pars_info_t* info = pars_info_create(); + + pars_info_add_literal(info, "name", name, name_len, + DATA_VARCHAR, DATA_ENGLISH); + pars_info_add_literal(info, "data", data, data_len, + DATA_BLOB, DATA_BINARY_TYPE | DATA_NOT_NULL); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + " max_id INT;\n" + "DECLARE CURSOR cur IS\n" + " SELECT ID FROM SYS_ZIP_DICT\n" + " ORDER BY ID DESC;\n" + "BEGIN\n" + " max_id := 0;\n" + " OPEN cur;\n" + " FETCH cur INTO max_id;\n" + " IF (cur % NOTFOUND) THEN\n" + " max_id := 0;\n" + " END IF;\n" + " CLOSE cur;\n" + " INSERT INTO SYS_ZIP_DICT VALUES" + " (max_id + 1, :name, :data);\n" + "END;\n", + FALSE, trx); + + return error; +} + +/** Fetch callback, just stores extracted zip_dict id in the external +variable. +@return TRUE if all OK */ +static +ibool +dict_create_extract_int_aux( + void* row, /*!< in: sel_node_t* */ + void* user_arg) /*!< in: int32 id */ +{ + sel_node_t* node = static_cast(row); + dfield_t* dfield = que_node_get_val(node->select_list); + dtype_t* type = dfield_get_type(dfield); + ulint len = dfield_get_len(dfield); + + ut_a(dtype_get_mtype(type) == DATA_INT); + ut_a(len == sizeof(ib_uint32_t)); + + memcpy(user_arg, dfield_get_data(dfield), sizeof(ib_uint32_t)); + + return(TRUE); +} + +/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS +InnoDB system table. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_add_zip_dict_reference( + ulint table_id, /*!< in: table id */ + ulint column_pos, /*!< in: column position */ + ulint dict_id, /*!< in: dict id */ + trx_t* trx) /*!< in/out: transaction */ +{ + pars_info_t* info = pars_info_create(); + + pars_info_add_int4_literal(info, "table_id", table_id); + pars_info_add_int4_literal(info, "column_pos", column_pos); + pars_info_add_int4_literal(info, "dict_id", dict_id); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + "BEGIN\n" + " INSERT INTO SYS_ZIP_DICT_COLS VALUES" + " (:table_id, :column_pos, :dict_id);\n" + "END;\n", + FALSE, trx); + return error; +} + +/** Get a single compression dictionary id for the given +(table id, column pos) pair. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_get_zip_dict_id_by_reference( + ulint table_id, /*!< in: table id */ + ulint column_pos, /*!< in: column position */ + ulint* dict_id, /*!< out: dict id */ + trx_t* trx) /*!< in/out: transaction */ +{ + ut_ad(dict_id); + + pars_info_t* info = pars_info_create(); + + ib_uint32_t dict_id_buf; + mach_write_to_4(reinterpret_cast(&dict_id_buf ), + ULINT32_UNDEFINED); + + pars_info_add_int4_literal(info, "table_id", table_id); + pars_info_add_int4_literal(info, "column_pos", column_pos); + pars_info_bind_function( + info, "my_func", dict_create_extract_int_aux, &dict_id_buf); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + "DECLARE FUNCTION my_func;\n" + "DECLARE CURSOR cur IS\n" + " SELECT DICT_ID FROM SYS_ZIP_DICT_COLS\n" + " WHERE TABLE_ID = :table_id AND\n" + " COLUMN_POS = :column_pos;\n" + "BEGIN\n" + " OPEN cur;\n" + " FETCH cur INTO my_func();\n" + " CLOSE cur;\n" + "END;\n", + FALSE, trx); + if (error == DB_SUCCESS) { + ib_uint32_t local_dict_id = mach_read_from_4( + reinterpret_cast(&dict_id_buf)); + if (local_dict_id == ULINT32_UNDEFINED) + error = DB_RECORD_NOT_FOUND; + else + *dict_id = local_dict_id; + } + return error; +} + +/** Get compression dictionary id for the given name. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_get_zip_dict_id_by_name( + const char* dict_name, /*!< in: dict name */ + ulint dict_name_len, /*!< in: dict name length */ + ulint* dict_id, /*!< out: dict id */ + trx_t* trx) /*!< in/out: transaction */ +{ + ut_ad(dict_name); + ut_ad(dict_name_len); + ut_ad(dict_id); + + pars_info_t* info = pars_info_create(); + + pars_info_add_literal(info, "dict_name", dict_name, dict_name_len, + DATA_VARCHAR, DATA_ENGLISH); + + ib_uint32_t dict_id_buf; + mach_write_to_4(reinterpret_cast(&dict_id_buf), + ULINT32_UNDEFINED); + pars_info_bind_function( + info, "my_func", dict_create_extract_int_aux, &dict_id_buf); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + "DECLARE FUNCTION my_func;\n" + "DECLARE CURSOR cur IS\n" + " SELECT ID FROM SYS_ZIP_DICT\n" + " WHERE NAME = :dict_name;\n" + "BEGIN\n" + " OPEN cur;\n" + " FETCH cur INTO my_func();\n" + " CLOSE cur;\n" + "END;\n", + FALSE, trx); + if (error == DB_SUCCESS) { + ib_uint32_t local_dict_id = mach_read_from_4( + reinterpret_cast(&dict_id_buf)); + if (local_dict_id == ULINT32_UNDEFINED) + error = DB_RECORD_NOT_FOUND; + else + *dict_id = local_dict_id; + } + return error; +} + +/** Auxiliary enum used to indicate zip dict data extraction result code */ +enum zip_dict_info_aux_code { + zip_dict_info_success, /*!< success */ + zip_dict_info_not_found, /*!< zip dict record not found */ + zip_dict_info_oom, /*!< out of memory */ + zip_dict_info_corrupted_name, /*!< corrupted zip dict name */ + zip_dict_info_corrupted_data /*!< corrupted zip dict data */ +}; + +/** Auxiliary struct used to return zip dict info aling with result code */ +struct zip_dict_info_aux { + LEX_STRING name; /*!< zip dict name */ + LEX_STRING data; /*!< zip dict data */ + int code; /*!< result code (0 - success) */ +}; + +/** Fetch callback, just stores extracted zip_dict data in the external +variable. +@return always returns TRUE */ +static +ibool +dict_create_get_zip_dict_info_by_id_aux( + void* row, /*!< in: sel_node_t* */ + void* user_arg) /*!< in: pointer to zip_dict_info_aux* */ +{ + sel_node_t* node = static_cast(row); + zip_dict_info_aux* result = + static_cast(user_arg); + + result->code = zip_dict_info_success; + result->name.str = 0; + result->name.length = 0; + result->data.str = 0; + result->data.length = 0; + + /* NAME field */ + que_node_t* exp = node->select_list; + ut_a(exp != 0); + + dfield_t* dfield = que_node_get_val(exp); + dtype_t* type = dfield_get_type(dfield); + ut_a(dtype_get_mtype(type) == DATA_VARCHAR); + + ulint len = dfield_get_len(dfield); + void* data = dfield_get_data(dfield); + + + if (len == UNIV_SQL_NULL) { + result->code = zip_dict_info_corrupted_name; + } + else { + result->name.str = + static_cast(my_malloc(len + 1, MYF(0))); + if (result->name.str == 0) { + result->code = zip_dict_info_oom; + } + else { + memcpy(result->name.str, data, len); + result->name.str[len] = '\0'; + result->name.length = len; + } + } + + /* DATA field */ + exp = que_node_get_next(exp); + ut_a(exp != 0); + + dfield = que_node_get_val(exp); + type = dfield_get_type(dfield); + ut_a(dtype_get_mtype(type) == DATA_BLOB); + + len = dfield_get_len(dfield); + data = dfield_get_data(dfield); + + if (len == UNIV_SQL_NULL) { + result->code = zip_dict_info_corrupted_data; + } + else { + result->data.str = + static_cast(my_malloc( + len == 0 ? 1 : len, MYF(0))); + if (result->data.str == 0) { + result->code = zip_dict_info_oom; + } + else { + memcpy(result->data.str, data, len); + result->data.length = len; + } + } + + ut_ad(que_node_get_next(exp) == 0); + + if (result->code != zip_dict_info_success) { + if (result->name.str == 0) { + mem_free(result->name.str); + result->name.str = 0; + result->name.length = 0; + } + if (result->data.str == 0) { + mem_free(result->data.str); + result->data.str = 0; + result->data.length = 0; + } + } + + return TRUE; +} + +/** Get compression dictionary info (name and data) for the given id. +Allocates memory for name and data on success. +Must be freed with mem_free(). +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_get_zip_dict_info_by_id( + ulint dict_id, /*!< in: dict id */ + char** name, /*!< out: dict name */ + ulint* name_len, /*!< out: dict name length*/ + char** data, /*!< out: dict data */ + ulint* data_len, /*!< out: dict data length*/ + trx_t* trx) /*!< in/out: transaction */ +{ + ut_ad(name); + ut_ad(data); + + zip_dict_info_aux rec; + rec.code = zip_dict_info_not_found; + pars_info_t* info = pars_info_create(); + + pars_info_add_int4_literal(info, "id", dict_id); + pars_info_bind_function( + info, "my_func", dict_create_get_zip_dict_info_by_id_aux, + &rec); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + "DECLARE FUNCTION my_func;\n" + "DECLARE CURSOR cur IS\n" + " SELECT NAME, DATA FROM SYS_ZIP_DICT\n" + " WHERE ID = :id;\n" + "BEGIN\n" + " OPEN cur;\n" + " FETCH cur INTO my_func();\n" + " CLOSE cur;\n" + "END;\n", + FALSE, trx); + if (error == DB_SUCCESS) { + switch (rec.code) { + case zip_dict_info_success: + *name = rec.name.str; + *name_len = rec.name.length; + *data = rec.data.str; + *data_len = rec.data.length; + break; + case zip_dict_info_not_found: + error = DB_RECORD_NOT_FOUND; + break; + case zip_dict_info_oom: + error = DB_OUT_OF_MEMORY; + break; + case zip_dict_info_corrupted_name: + case zip_dict_info_corrupted_data: + error = DB_INVALID_NULL; + break; + default: + ut_error; + } + } + return error; +} + +/** Remove a single compression dictionary from the data dictionary +tables in the database. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_remove_zip_dict( + const char* name, /*!< in: dict name */ + ulint name_len, /*!< in: dict name length */ + trx_t* trx) /*!< in/out: transaction */ +{ + ut_ad(name); + + pars_info_t* info = pars_info_create(); + + ib_uint32_t dict_id_buf; + mach_write_to_4(reinterpret_cast(&dict_id_buf), + ULINT32_UNDEFINED); + ib_uint32_t counter_buf; + mach_write_to_4(reinterpret_cast(&counter_buf), + ULINT32_UNDEFINED); + + pars_info_add_literal(info, "name", name, name_len, + DATA_VARCHAR, DATA_ENGLISH); + pars_info_bind_int4_literal(info, "dict_id", &dict_id_buf); + pars_info_bind_function(info, "find_dict_func", + dict_create_extract_int_aux, &dict_id_buf); + pars_info_bind_function(info, "count_func", + dict_create_extract_int_aux, &counter_buf); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + "DECLARE FUNCTION find_dict_func;\n" + "DECLARE FUNCTION count_func;\n" + "DECLARE CURSOR dict_cur IS\n" + " SELECT ID FROM SYS_ZIP_DICT\n" + " WHERE NAME = :name\n" + " FOR UPDATE;\n" + "DECLARE CURSOR ref_cur IS\n" + " SELECT 1 FROM SYS_ZIP_DICT_COLS\n" + " WHERE DICT_ID = :dict_id;\n" + "BEGIN\n" + " OPEN dict_cur;\n" + " FETCH dict_cur INTO find_dict_func();\n" + " IF NOT (SQL % NOTFOUND) THEN\n" + " OPEN ref_cur;\n" + " FETCH ref_cur INTO count_func();\n" + " IF SQL % NOTFOUND THEN\n" + " DELETE FROM SYS_ZIP_DICT WHERE CURRENT OF dict_cur;\n" + " END IF;\n" + " CLOSE ref_cur;\n" + " END IF;\n" + " CLOSE dict_cur;\n" + "END;\n", + FALSE, trx); + if (error == DB_SUCCESS) { + ib_uint32_t local_dict_id = mach_read_from_4( + reinterpret_cast(&dict_id_buf)); + if (local_dict_id == ULINT32_UNDEFINED) { + error = DB_RECORD_NOT_FOUND; + } + else { + ib_uint32_t local_counter = mach_read_from_4( + reinterpret_cast(&counter_buf)); + if (local_counter != ULINT32_UNDEFINED) + error = DB_ROW_IS_REFERENCED; + } + } + return error; +} + +/** Remove all compression dictionary references for the given table ID from +the data dictionary tables in the database. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_remove_zip_dict_references_for_table( + ulint table_id, /*!< in: table id */ + trx_t* trx) /*!< in/out: transaction */ +{ + pars_info_t* info = pars_info_create(); + + pars_info_add_int4_literal(info, "table_id", table_id); + + dberr_t error = que_eval_sql(info, + "PROCEDURE P () IS\n" + "BEGIN\n" + " DELETE FROM SYS_ZIP_DICT_COLS\n" + " WHERE TABLE_ID = :table_id;\n" + "END;\n", + FALSE, trx); + return error; +} diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index f1fbf25c3a61e..57dd6cfa04dff 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -6781,3 +6781,161 @@ dict_tf_to_row_format_string( return(0); } #endif /* !UNIV_HOTBACKUP */ + +/** Insert a records into SYS_ZIP_DICT. +@retval DB_SUCCESS if OK +@retval dberr_t if the insert failed */ +UNIV_INTERN +dberr_t +dict_create_zip_dict( + const char* name, /*!< in: zip_dict name */ + ulint name_len, /*!< in: zip_dict name length*/ + const char* data, /*!< in: zip_dict data */ + ulint data_len) /*!< in: zip_dict data length */ +{ + dberr_t err = DB_SUCCESS; + trx_t* trx; + + ut_ad(name); + ut_ad(data); + + rw_lock_x_lock(&dict_operation_lock); + dict_mutex_enter_for_mysql(); + + trx = trx_allocate_for_background(); + trx->op_info = "insert zip_dict"; + trx->dict_operation_lock_mode = RW_X_LATCH; + trx_start_if_not_started(trx); + + err = dict_create_add_zip_dict(name, name_len, data, data_len, trx); + + if (err == DB_SUCCESS) { + trx_commit_for_mysql(trx); + } + else { + trx->op_info = "rollback of internal trx on zip_dict table"; + trx_rollback_to_savepoint(trx, NULL); + ut_a(trx->error_state == DB_SUCCESS); + } + trx->op_info = ""; + trx->dict_operation_lock_mode = 0; + trx_free_for_background(trx); + + dict_mutex_exit_for_mysql(); + rw_lock_x_unlock(&dict_operation_lock); + + return err; +} +/** Get single compression dictionary id for the given +(table id, column pos) pair. +@retval DB_SUCCESS if OK +@retval DB_RECORD_NOT_FOUND if not found */ +UNIV_INTERN +dberr_t +dict_get_dictionary_id_by_key( + ulint table_id, /*!< in: table id */ + ulint column_pos, /*!< in: column position */ + ulint* dict_id) /*!< out: zip_dict id */ +{ + dberr_t err = DB_SUCCESS; + trx_t* trx; + + rw_lock_s_lock(&dict_operation_lock); + dict_mutex_enter_for_mysql(); + + trx = trx_allocate_for_background(); + trx->op_info = "get zip dict id by composite key"; + trx->dict_operation_lock_mode = RW_S_LATCH; + trx_start_if_not_started(trx); + + err = dict_create_get_zip_dict_id_by_reference(table_id, column_pos, + dict_id, trx); + + trx_commit_for_mysql(trx); + trx->dict_operation_lock_mode = 0; + trx_free_for_background(trx); + + dict_mutex_exit_for_mysql(); + rw_lock_s_unlock(&dict_operation_lock); + + return err; +} +/** Get compression dictionary info (name and data) for the given id. +Allocates memory in name->str and data->str on success. +Must be freed with mem_free(). +@retval DB_SUCCESS if OK +@retval DB_RECORD_NOT_FOUND if not found */ +UNIV_INTERN +dberr_t +dict_get_dictionary_info_by_id( + ulint dict_id, /*!< in: table name */ + char** name, /*!< out: dictionary name */ + ulint* name_len, /*!< out: dictionary name length*/ + char** data, /*!< out: dictionary data */ + ulint* data_len) /*!< out: dictionary data length*/ +{ + dberr_t err = DB_SUCCESS; + trx_t* trx; + + rw_lock_s_lock(&dict_operation_lock); + dict_mutex_enter_for_mysql(); + + trx = trx_allocate_for_background(); + trx->op_info = "get zip dict name and data by id"; + trx->dict_operation_lock_mode = RW_S_LATCH; + trx_start_if_not_started(trx); + + err = dict_create_get_zip_dict_info_by_id(dict_id, name, name_len, + data, data_len, trx); + + trx_commit_for_mysql(trx); + trx->dict_operation_lock_mode = 0; + trx_free_for_background(trx); + + dict_mutex_exit_for_mysql(); + rw_lock_s_unlock(&dict_operation_lock); + + return err; +} +/** Delete a record in SYS_ZIP_DICT with the given name. +@retval DB_SUCCESS if OK +@retval DB_RECORD_NOT_FOUND if not found +@retval DB_ROW_IS_REFERENCED if in use */ +UNIV_INTERN +dberr_t +dict_drop_zip_dict( + const char* name, /*!< in: zip_dict name */ + ulint name_len) /*!< in: zip_dict name length*/ +{ + dberr_t err = DB_SUCCESS; + trx_t* trx; + + ut_ad(name); + + rw_lock_x_lock(&dict_operation_lock); + dict_mutex_enter_for_mysql(); + + trx = trx_allocate_for_background(); + trx->op_info = "delete zip_dict"; + trx->dict_operation_lock_mode = RW_X_LATCH; + trx_start_if_not_started(trx); + + err = dict_create_remove_zip_dict(name, name_len, trx); + + if (err == DB_SUCCESS) { + trx_commit_for_mysql(trx); + } + else { + trx->op_info = "rollback of internal trx on zip_dict table"; + trx_rollback_to_savepoint(trx, NULL); + ut_a(trx->error_state == DB_SUCCESS); + } + trx->op_info = ""; + trx->dict_operation_lock_mode = 0; + trx_free_for_background(trx); + + dict_mutex_exit_for_mysql(); + rw_lock_x_unlock(&dict_operation_lock); + + return err; +} diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 988351dbca55a..db2aa3239f5ab 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -56,7 +56,9 @@ static const char* SYSTEM_TABLE_NAME[] = { "SYS_FOREIGN", "SYS_FOREIGN_COLS", "SYS_TABLESPACES", - "SYS_DATAFILES" + "SYS_DATAFILES", + "SYS_ZIP_DICT", + "SYS_ZIP_DICT_COLS" }; /* If this flag is TRUE, then we will load the cluster index's (and tables') @@ -728,6 +730,161 @@ dict_process_sys_datafiles( return(NULL); } +/** This function parses a SYS_ZIP_DICT record, extracts necessary +information from the record and returns to caller. +@return error message, or NULL on success */ +UNIV_INTERN +const char* +dict_process_sys_zip_dict( + mem_heap_t* heap, /*!< in/out: heap memory */ + ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ + const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ + ulint* id, /*!< out: dict id */ + const char** name, /*!< out: dict name */ + const char** data, /*!< out: dict data */ + ulint* data_len) /*!< out: dict data length */ +{ + ulint len; + const byte* field; + + /* Initialize the output values */ + *id = ULINT_UNDEFINED; + *name = NULL; + *data = NULL; + *data_len = 0; + + if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { + return("delete-marked record in SYS_ZIP_DICT"); + } + + if (UNIV_UNLIKELY( + rec_get_n_fields_old(rec)!= DICT_NUM_FIELDS__SYS_ZIP_DICT)) { + return("wrong number of columns in SYS_ZIP_DICT record"); + } + + field = rec_get_nth_field_old( + rec, DICT_FLD__SYS_ZIP_DICT__ID, &len); + if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { + goto err_len; + } + *id = mach_read_from_4(field); + + rec_get_nth_field_offs_old( + rec, DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID, &len); + if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { + goto err_len; + } + + rec_get_nth_field_offs_old( + rec, DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR, &len); + if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { + goto err_len; + } + + field = rec_get_nth_field_old( + rec, DICT_FLD__SYS_ZIP_DICT__NAME, &len); + if (UNIV_UNLIKELY(len == 0 || len == UNIV_SQL_NULL)) { + goto err_len; + } + *name = mem_heap_strdupl(heap, (char*) field, len); + + field = rec_get_nth_field_old( + rec, DICT_FLD__SYS_ZIP_DICT__DATA, &len); + if (UNIV_UNLIKELY(len == UNIV_SQL_NULL)) { + goto err_len; + } + + if (rec_get_1byte_offs_flag(rec) == 0 && + rec_2_is_field_extern(rec, DICT_FLD__SYS_ZIP_DICT__DATA)) { + ut_a(len >= BTR_EXTERN_FIELD_REF_SIZE); + + if (UNIV_UNLIKELY + (!memcmp(field + len - BTR_EXTERN_FIELD_REF_SIZE, + field_ref_zero, + BTR_EXTERN_FIELD_REF_SIZE))) { + goto err_len; + } + *data = reinterpret_cast( + btr_copy_externally_stored_field(data_len, field, + zip_size, len, heap)); + } + else { + *data_len = len; + *data = static_cast(mem_heap_dup(heap, field, len)); + } + + return(NULL); + +err_len: + return("incorrect column length in SYS_ZIP_DICT"); +} + +/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary +information from the record and returns to caller. +@return error message, or NULL on success */ +UNIV_INTERN +const char* +dict_process_sys_zip_dict_cols( + mem_heap_t* heap, /*!< in/out: heap memory */ + const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ + ulint* table_id, /*!< out: table id */ + ulint* column_pos, /*!< out: column position */ + ulint* dict_id) /*!< out: dict id */ +{ + ulint len; + const byte* field; + + /* Initialize the output values */ + *table_id = ULINT_UNDEFINED; + *column_pos = ULINT_UNDEFINED; + *dict_id = ULINT_UNDEFINED; + + if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { + return("delete-marked record in SYS_ZIP_DICT_COLS"); + } + + if (UNIV_UNLIKELY(rec_get_n_fields_old(rec) != + DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS)) { + return("wrong number of columns in SYS_ZIP_DICT_COLS" + " record"); + } + + field = rec_get_nth_field_old( + rec, DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID, &len); + if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { +err_len: + return("incorrect column length in SYS_ZIP_DICT_COLS"); + } + *table_id = mach_read_from_4(field); + + field = rec_get_nth_field_old( + rec, DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS, &len); + if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { + goto err_len; + } + *column_pos = mach_read_from_4(field); + + rec_get_nth_field_offs_old( + rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID, &len); + if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { + goto err_len; + } + + rec_get_nth_field_offs_old( + rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR, &len); + if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { + goto err_len; + } + + field = rec_get_nth_field_old( + rec, DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID, &len); + if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { + goto err_len; + } + *dict_id = mach_read_from_4(field); + + return(NULL); +} /********************************************************************//** Determine the flags of a table as stored in SYS_TABLES.TYPE and N_COLS. @return ULINT_UNDEFINED if error, else a valid dict_table_t::flags. */ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index c1dbb5f91b996..57e415ae939fe 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -489,6 +489,8 @@ fil_space_get_by_id( ut_ad(space->magic_n == FIL_SPACE_MAGIC_N), space->id == id); + /* The system tablespace must always be found */ + ut_ad(space || id != 0 || srv_is_being_started); return(space); } diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index 25059db96b078..a0f0fab5566b0 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -108,6 +108,7 @@ UNIV_INTERN mysql_pfs_key_t fts_pll_tokenize_mutex_key; /** variable to record innodb_fts_internal_tbl_name for information schema table INNODB_FTS_INSERTED etc. */ UNIV_INTERN char* fts_internal_tbl_name = NULL; +UNIV_INTERN char* fts_internal_tbl_name2 = NULL; /** InnoDB default stopword list: There are different versions of stopwords, the stop words listed @@ -6569,6 +6570,36 @@ fts_check_corrupt_index( return(0); } +/* Get parent table name if it's a fts aux table +@param[in] aux_table_name aux table name +@param[in] aux_table_len aux table length +@return parent table name, or NULL */ +char* +fts_get_parent_table_name( + const char* aux_table_name, + ulint aux_table_len) +{ + fts_aux_table_t aux_table; + char* parent_table_name = NULL; + + if (fts_is_aux_table_name(&aux_table, aux_table_name, aux_table_len)) { + dict_table_t* parent_table; + + parent_table = dict_table_open_on_id( + aux_table.parent_id, TRUE, DICT_TABLE_OP_NORMAL); + + if (parent_table != NULL) { + parent_table_name = mem_strdupl( + parent_table->name, + strlen(parent_table->name)); + + dict_table_close(parent_table, TRUE, FALSE); + } + } + + return(parent_table_name); +} + /** Check the validity of the parent table. @param[in] aux_table auxiliary table @return true if it is a valid table or false if it is not */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index f00d11bd870c4..58d638d0b0c30 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1361,6 +1361,29 @@ normalize_table_name_low( ibool set_lower_case); /* in: TRUE if we want to set name to lower case */ +/** Creates a new compression dictionary. */ +static +handler_create_zip_dict_result +innobase_create_zip_dict( + handlerton* hton, /*!< in: innobase handlerton */ + THD* thd, /*!< in: handle to the MySQL thread */ + const char* name, /*!< in: zip dictionary name */ + ulint* name_len, + /*!< in/out: zip dictionary name length */ + const char* data, /*!< in: zip dictionary data */ + ulint* data_len); + /*!< in/out: zip dictionary data length */ + +/** Drops a existing compression dictionary. */ +static +handler_drop_zip_dict_result +innobase_drop_zip_dict( + handlerton* hton, /*!< in: innobase handlerton */ + THD* thd, /*!< in: handle to the MySQL thread */ + const char* name, /*!< in: zip dictionary name */ + ulint* name_len); + /*!< in/out: zip dictionary name length */ + /*************************************************************//** Checks if buffer pool is big enough to enable backoff algorithm. InnoDB empty free list algorithm backoff requires free pages @@ -3422,6 +3445,9 @@ innobase_init( innobase_hton->kill_connection = innobase_kill_connection; + innobase_hton->create_zip_dict = innobase_create_zip_dict; + innobase_hton->drop_zip_dict = innobase_drop_zip_dict; + ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); #ifndef DBUG_OFF @@ -4100,6 +4126,89 @@ innobase_purge_changed_page_bitmaps( return (my_bool)log_online_purge_changed_page_bitmaps(lsn); } +/** Creates a new compression dictionary. */ +static +handler_create_zip_dict_result +innobase_create_zip_dict( + handlerton* hton, /*!< in: innobase handlerton */ + THD* thd, /*!< in: handle to the MySQL thread */ + const char* name, /*!< in: zip dictionary name */ + ulint* name_len, + /*!< in/out: zip dictionary name length */ + const char* data, /*!< in: zip dictionary data */ + ulint* data_len) + /*!< in/out: zip dictionary data length */ +{ + handler_create_zip_dict_result result = + HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; + + DBUG_ENTER("innobase_create_zip_dict"); + DBUG_ASSERT(hton == innodb_hton_ptr); + + if (UNIV_UNLIKELY(high_level_read_only)) { + DBUG_RETURN(HA_CREATE_ZIP_DICT_READ_ONLY); + } + + if (UNIV_UNLIKELY(*name_len > ZIP_DICT_MAX_NAME_LENGTH)) { + *name_len = ZIP_DICT_MAX_NAME_LENGTH; + DBUG_RETURN(HA_CREATE_ZIP_DICT_NAME_TOO_LONG); + } + + if (UNIV_UNLIKELY(*data_len > ZIP_DICT_MAX_DATA_LENGTH)) { + *data_len = ZIP_DICT_MAX_DATA_LENGTH; + DBUG_RETURN(HA_CREATE_ZIP_DICT_DATA_TOO_LONG); + } + + switch (dict_create_zip_dict(name, *name_len, data, *data_len)) { + case DB_SUCCESS: + result = HA_CREATE_ZIP_DICT_OK; + break; + case DB_DUPLICATE_KEY: + result = HA_CREATE_ZIP_DICT_ALREADY_EXISTS; + break; + default: + ut_ad(0); + result = HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; + } + DBUG_RETURN(result); +} + +/** Drops a existing compression dictionary. */ +static +handler_drop_zip_dict_result +innobase_drop_zip_dict( + handlerton* hton, /*!< in: innobase handlerton */ + THD* thd, /*!< in: handle to the MySQL thread */ + const char* name, /*!< in: zip dictionary name */ + ulint* name_len) + /*!< in/out: zip dictionary name length */ +{ + handler_drop_zip_dict_result result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; + + DBUG_ENTER("innobase_drop_zip_dict"); + DBUG_ASSERT(hton == innodb_hton_ptr); + + if (UNIV_UNLIKELY(high_level_read_only)) { + DBUG_RETURN(HA_DROP_ZIP_DICT_READ_ONLY); + } + + switch (dict_drop_zip_dict(name, *name_len)) { + case DB_SUCCESS: + result = HA_DROP_ZIP_DICT_OK; + break; + case DB_RECORD_NOT_FOUND: + result = HA_DROP_ZIP_DICT_DOES_NOT_EXIST; + break; + case DB_ROW_IS_REFERENCED: + result = HA_DROP_ZIP_DICT_IS_REFERENCED; + break; + default: + ut_ad(0); + result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; + } + DBUG_RETURN(result); +} + /*****************************************************************//** Check whether this is a fake change transaction. @return TRUE if a fake change transaction */ @@ -5460,6 +5569,86 @@ innobase_build_index_translation( DBUG_RETURN(ret); } +/** This function checks if all the compression dictionaries referenced +in table->fields exist in SYS_ZIP_DICT InnoDB system table. +@return true if all referenced dictionaries exist */ +UNIV_INTERN +bool +innobase_check_zip_dicts( + const TABLE* table, /*!< in: table in MySQL data + dictionary */ + ulint* dict_ids, /*!< out: identified zip dict ids + (at least n_fields long) */ + trx_t* trx, /*!< in: transaction */ + const char** err_dict_name) /*!< out: the name of the + zip_dict which does not exist. */ +{ + DBUG_ENTER("innobase_check_zip_dicts"); + + bool res = true; + dberr_t err = DB_SUCCESS; + const size_t n_fields = table->s->fields; + + Field* field_ptr; + for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; + ++field_idx) + { + field_ptr = table->field[field_idx]; + if (field_ptr->has_associated_compression_dictionary()) { + err = dict_create_get_zip_dict_id_by_name( + field_ptr->zip_dict_name.str, + field_ptr->zip_dict_name.length, + &dict_ids[field_idx], + trx); + ut_a(err == DB_SUCCESS || err == DB_RECORD_NOT_FOUND); + } + else { + dict_ids[field_idx] = ULINT_UNDEFINED; + } + + } + + if (err != DB_SUCCESS) { + res = false; + *err_dict_name = field_ptr->zip_dict_name.str; + } + + DBUG_RETURN(res); +} + +/** This function creates compression dictionary references in +SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info +in table->fields and provided zip dict ids. */ +UNIV_INTERN +void +innobase_create_zip_dict_references( + const TABLE* table, /*!< in: table in MySQL data + dictionary */ + table_id_t ib_table_id, /*!< in: table ID in Innodb data + dictionary */ + ulint* zip_dict_ids, /*!< in: zip dict ids + (at least n_fields long) */ + trx_t* trx) /*!< in: transaction */ +{ + DBUG_ENTER("innobase_create_zip_dict_references"); + + dberr_t err = DB_SUCCESS; + const size_t n_fields = table->s->fields; + + for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; + ++field_idx) + { + if (zip_dict_ids[field_idx] != ULINT_UNDEFINED) { + err = dict_create_add_zip_dict_reference(ib_table_id, + table->field[field_idx]->field_index, + zip_dict_ids[field_idx], trx); + ut_a(err == DB_SUCCESS); + } + } + + DBUG_VOID_RETURN; +} + /*******************************************************************//** This function uses index translation table to quickly locate the requested index structure. @@ -6749,7 +6938,12 @@ ha_innobase::store_key_val_for_row( blob_data = row_mysql_read_blob_ref(&blob_len, (byte*) (record + (ulint) get_field_offset(table, field)), - (ulint) field->pack_length()); + (ulint) field->pack_length(), + field->column_format() == + COLUMN_FORMAT_TYPE_COMPRESSED, + reinterpret_cast( + field->zip_dict_data.str), + field->zip_dict_data.length, prebuilt); true_len = blob_len; @@ -7004,6 +7198,9 @@ build_template_field( templ->mbminlen = dict_col_get_mbminlen(col); templ->mbmaxlen = dict_col_get_mbmaxlen(col); templ->is_unsigned = col->prtype & DATA_UNSIGNED; + templ->compressed = (field->column_format() + == COLUMN_FORMAT_TYPE_COMPRESSED); + templ->zip_dict_data = field->zip_dict_data; if (!dict_index_is_clust(index) && templ->rec_field_no == ULINT_UNDEFINED) { @@ -7761,8 +7958,11 @@ calc_row_difference( switch (col_type) { case DATA_BLOB: - o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len); - n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len); + /* Do not compress blob column while comparing*/ + o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len, + false, 0, 0, prebuilt); + n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len, + false, 0, 0, prebuilt); break; @@ -7832,7 +8032,13 @@ calc_row_difference( TRUE, new_mysql_row_col, col_pack_len, - dict_table_is_comp(prebuilt->table)); + dict_table_is_comp(prebuilt->table), + field->column_format() == + COLUMN_FORMAT_TYPE_COMPRESSED, + reinterpret_cast( + field->zip_dict_data.str), + field->zip_dict_data.length, + prebuilt); dfield_copy(&ufield->new_val, &dfield); } else { dfield_set_null(&ufield->new_val); @@ -9503,6 +9709,7 @@ create_table_def( ulint unsigned_type; ulint binary_type; ulint long_true_varchar; + ulint compressed; ulint charset_no; ulint i; ulint doc_id_col = 0; @@ -9649,6 +9856,13 @@ create_table_def( } } + /* Check if the the field has COMPRESSED attribute */ + compressed = 0; + if (field->column_format() == + COLUMN_FORMAT_TYPE_COMPRESSED) { + compressed = DATA_COMPRESSED; + } + /* First check whether the column to be added has a system reserved name. */ if (dict_col_name_is_reserved(field->field_name)){ @@ -9669,7 +9883,8 @@ create_table_def( dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type - | binary_type | long_true_varchar, + | binary_type | long_true_varchar + | compressed, charset_no), col_len); } @@ -10505,6 +10720,10 @@ ha_innobase::create( const char* stmt; size_t stmt_len; + mem_heap_t* heap = 0; + ulint* zip_dict_ids = 0; + const char* err_zip_dict_name = 0; + DBUG_ENTER("ha_innobase::create"); DBUG_ASSERT(thd != NULL); @@ -10595,6 +10814,18 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); + heap = mem_heap_create(form->s->fields * sizeof(ulint)); + zip_dict_ids = static_cast( + mem_heap_alloc(heap, form->s->fields * sizeof(ulint))); + + if (!innobase_check_zip_dicts(form, zip_dict_ids, + trx, &err_zip_dict_name)) { + error = -1; + my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, + MYF(0), err_zip_dict_name); + goto cleanup; + } + error = create_table_def(trx, form, norm_name, temp_path, remote_path, flags, flags2); if (error) { @@ -10702,6 +10933,22 @@ ha_innobase::create( dict_table_get_all_fts_indexes(innobase_table, fts->indexes); } + /* + Adding compression dictionary <-> compressed table column links + to the SYS_ZIP_DICT_COLS table. + */ + ut_a(zip_dict_ids != 0); + { + dict_table_t* local_table = dict_table_open_on_name( + norm_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); + + ut_a(local_table); + table_id_t table_id = local_table->id; + dict_table_close(local_table, TRUE, FALSE); + innobase_create_zip_dict_references(form, + table_id, zip_dict_ids, trx); + } + stmt = innobase_get_stmt(thd, &stmt_len); if (stmt) { @@ -10818,6 +11065,9 @@ ha_innobase::create( trx_free_for_mysql(trx); + if (heap != 0) + mem_heap_free(heap); + DBUG_RETURN(0); cleanup: @@ -10827,6 +11077,9 @@ ha_innobase::create( trx_free_for_mysql(trx); + if (heap != 0) + mem_heap_free(heap); + DBUG_RETURN(error); } @@ -11904,6 +12157,14 @@ ha_innobase::info_low( if (dict_stats_is_persistent_enabled(ib_table)) { if (is_analyze) { + + /* If this table is already queued for + background analyze, remove it from the + queue as we are about to do the same */ + dict_mutex_enter_for_mysql(); + dict_stats_recalc_pool_del(ib_table); + dict_mutex_exit_for_mysql(); + opt = DICT_STATS_RECALC_PERSISTENT; } else { /* This is e.g. 'SHOW INDEXES', fetch @@ -13050,6 +13311,11 @@ ha_innobase::extra( if (prebuilt->blob_heap) { row_mysql_prebuilt_free_blob_heap(prebuilt); } + + if (prebuilt->compress_heap) { + row_mysql_prebuilt_free_compress_heap(prebuilt); + } + break; case HA_EXTRA_RESET_STATE: reset_template(); @@ -13101,6 +13367,10 @@ ha_innobase::reset() row_mysql_prebuilt_free_blob_heap(prebuilt); } + if (prebuilt->compress_heap) { + row_mysql_prebuilt_free_compress_heap(prebuilt); + } + reset_template(); ds_mrr.reset(); @@ -13300,7 +13570,11 @@ ha_innobase::external_lock( && lock_type == F_WRLCK) || thd_sql_command(thd) == SQLCOM_CREATE_INDEX || thd_sql_command(thd) == SQLCOM_DROP_INDEX - || thd_sql_command(thd) == SQLCOM_DELETE)) { + || thd_sql_command(thd) == SQLCOM_DELETE + || thd_sql_command(thd) == + SQLCOM_CREATE_COMPRESSION_DICTIONARY + || thd_sql_command(thd) == + SQLCOM_DROP_COMPRESSION_DICTIONARY)) { if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { @@ -14062,7 +14336,9 @@ ha_innobase::store_lock( && lock_type <= TL_WRITE)) || sql_command == SQLCOM_CREATE_INDEX || sql_command == SQLCOM_DROP_INDEX - || sql_command == SQLCOM_DELETE)) { + || sql_command == SQLCOM_DELETE + || sql_command == SQLCOM_CREATE_COMPRESSION_DICTIONARY + || sql_command == SQLCOM_DROP_COMPRESSION_DICTIONARY)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); @@ -15001,6 +15277,82 @@ ha_innobase::check_if_incompatible_data( return(COMPATIBLE_DATA_YES); } +/** This function reads zip dict-related info from SYS_ZIP_DICT +and SYS_ZIP_DICT_COLS for all columns marked with +COLUMN_FORMAT_TYPE_COMPRESSED flag and updates +zip_dict_name / zip_dict_data for those which have associated +compression dictionaries. +*/ +UNIV_INTERN +void +ha_innobase::update_field_defs_with_zip_dict_info() +{ + DBUG_ENTER("update_field_defs_with_zip_dict_info"); + ut_ad(!mutex_own(&dict_sys->mutex)); + + char norm_name[FN_REFLEN]; + normalize_table_name(norm_name, table_share->normalized_path.str); + + dict_table_t* ib_table = dict_table_open_on_name( + norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); + + /* if dict_table_open_on_name() returns NULL, then it means that + TABLE_SHARE is populated for a table being created and we can + skip filling zip dict info here */ + if (ib_table == 0) + DBUG_VOID_RETURN; + + table_id_t ib_table_id = ib_table->id; + dict_table_close(ib_table, FALSE, FALSE); + Field* field; + for (uint i = 0; i < table_share->fields; ++i) { + field = table_share->field[i]; + if (field->column_format() == + COLUMN_FORMAT_TYPE_COMPRESSED) { + bool reference_found = false; + ulint dict_id = 0; + switch (dict_get_dictionary_id_by_key(ib_table_id, i, + &dict_id)) { + case DB_SUCCESS: + reference_found = true; + break; + case DB_RECORD_NOT_FOUND: + reference_found = false; + break; + default: + ut_error; + } + if (reference_found) { + char* local_name = 0; + ulint local_name_len = 0; + char* local_data = 0; + ulint local_data_len = 0; + if (dict_get_dictionary_info_by_id(dict_id, + &local_name, &local_name_len, + &local_data, &local_data_len) != + DB_SUCCESS) { + ut_error; + } + else { + field->zip_dict_name.str = + local_name; + field->zip_dict_name.length = + local_name_len; + field->zip_dict_data.str = + local_data; + field->zip_dict_data.length = + local_data_len; + } + } + else { + field->zip_dict_name = null_lex_cstr; + field->zip_dict_data = null_lex_cstr; + } + } + } + DBUG_VOID_RETURN; +} + /****************************************************************//** Update the system variable innodb_io_capacity_max using the "saved" value. This function is registered as a callback with MySQL. */ @@ -15555,7 +15907,12 @@ innodb_internal_table_update( my_free(old); } - fts_internal_tbl_name = *(char**) var_ptr; + fts_internal_tbl_name2 = *(char**) var_ptr; + if (fts_internal_tbl_name2 == NULL) { + fts_internal_tbl_name = const_cast("default"); + } else { + fts_internal_tbl_name = fts_internal_tbl_name2; + } } /****************************************************************//** @@ -17888,7 +18245,7 @@ static MYSQL_SYSVAR_BOOL(disable_sort_file_cache, srv_disable_sort_file_cache, "Whether to disable OS system file cache for sort I/O", NULL, NULL, FALSE); -static MYSQL_SYSVAR_STR(ft_aux_table, fts_internal_tbl_name, +static MYSQL_SYSVAR_STR(ft_aux_table, fts_internal_tbl_name2, PLUGIN_VAR_NOCMDARG, "FTS internal auxiliary table to be checked", innodb_internal_table_validate, @@ -18340,6 +18697,19 @@ static MYSQL_SYSVAR_BOOL(locking_fake_changes, srv_fake_changes_locks, "not take any locks at all.", NULL, NULL, TRUE); +static MYSQL_SYSVAR_UINT(compressed_columns_zip_level, + srv_compressed_columns_zip_level, + PLUGIN_VAR_RQCMDARG, + "Compression level used for compressed columns. 0 is no compression" + ", 1 is fastest and 9 is best compression. Default is 6.", + NULL, NULL, DEFAULT_COMPRESSION_LEVEL, 0, 9, 0); + +static MYSQL_SYSVAR_ULONG(compressed_columns_threshold, + srv_compressed_columns_threshold, + PLUGIN_VAR_RQCMDARG, + "Compress column data if its length exceeds this value. Default is 96", + NULL, NULL, 96, 1, ~0UL, 0); + static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), MYSQL_SYSVAR(additional_mem_pool_size), @@ -18537,6 +18907,8 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(fake_changes), MYSQL_SYSVAR(locking_fake_changes), MYSQL_SYSVAR(tmpdir), + MYSQL_SYSVAR(compressed_columns_zip_level), + MYSQL_SYSVAR(compressed_columns_threshold), NULL }; @@ -18559,6 +18931,8 @@ mysql_declare_plugin(innobase) i_s_xtradb_read_view, i_s_xtradb_internal_hash_tables, i_s_xtradb_rseg, +i_s_xtradb_zip_dict, +i_s_xtradb_zip_dict_cols, i_s_innodb_trx, i_s_innodb_locks, i_s_innodb_lock_waits, diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index c9f9cfabc1f71..609787bd6a1a7 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -287,6 +287,15 @@ class ha_innobase: public handler /** @} */ bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); + + /** This function reads zip dict-related info from SYS_ZIP_DICT + and SYS_ZIP_DICT_COLS for all columns marked with + COLUMN_FORMAT_TYPE_COMPRESSED flag and updates + zip_dict_name / zip_dict_data for those which have associated + compression dictionaries. + */ + virtual void update_field_defs_with_zip_dict_info(); + private: /** Builds a 'template' to the prebuilt struct. @@ -665,3 +674,31 @@ innobase_build_index_translation( INNOBASE_SHARE* share); /*!< in/out: share structure where index translation table will be constructed in. */ + +/** This function checks if all the compression dictionaries referenced +in table->fields exist in SYS_ZIP_DICT InnoDB system table. +@return true if all referenced dictionaries exist */ +UNIV_INTERN +bool +innobase_check_zip_dicts( + const TABLE* table, /*!< in: table in MySQL data + dictionary */ + ulint* dict_ids, /*!< out: identified zip dict ids + (at least n_fields long) */ + trx_t* trx, /*!< in: transaction */ + const char** err_dict_name); /*!< out: the name of the + zip_dict which does not exist. */ + +/** This function creates compression dictionary references in +SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info +in table->fields and provided zip dict ids. */ +UNIV_INTERN +void +innobase_create_zip_dict_references( + const TABLE* table, /*!< in: table in MySQL data + dictionary */ + table_id_t ib_table_id, /*!< in: table ID in Innodb data + dictionary */ + ulint* zip_dict_ids, /*!< in: zip dict ids + (at least n_fields long) */ + trx_t* trx); /*!< in: transaction */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 1a39f70614d62..291ed06a9559d 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -201,7 +201,10 @@ innobase_need_rebuild( /*==================*/ const Alter_inplace_info* ha_alter_info) { - if (ha_alter_info->handler_flags + Alter_inplace_info::HA_ALTER_FLAGS alter_inplace_flags = + ha_alter_info->handler_flags & ~(INNOBASE_INPLACE_IGNORE); + + if (alter_inplace_flags == Alter_inplace_info::CHANGE_CREATE_OPTION && !(ha_alter_info->create_info->used_fields & (HA_CREATE_USED_ROW_FORMAT @@ -1069,6 +1072,15 @@ innobase_col_to_mysql( field->reset(); if (field->type() == MYSQL_TYPE_VARCHAR) { + if (field->column_format() == + COLUMN_FORMAT_TYPE_COMPRESSED) { + /* Skip compressed varchar column when + reporting an erroneous row + during index creation or table rebuild. */ + field->set_null(); + break; + } + /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2328,7 +2340,8 @@ innobase_build_col_map_add( mem_heap_t* heap, dfield_t* dfield, const Field* field, - ulint comp) + ulint comp, + row_prebuilt_t* prebuilt) { if (field->is_real_null()) { dfield_set_null(dfield); @@ -2340,7 +2353,10 @@ innobase_build_col_map_add( byte* buf = static_cast(mem_heap_alloc(heap, size)); row_mysql_store_col_in_innobase_format( - dfield, buf, TRUE, field->ptr, size, comp); + dfield, buf, TRUE, field->ptr, size, comp, + field->column_format() == COLUMN_FORMAT_TYPE_COMPRESSED, + reinterpret_cast(field->zip_dict_data.str), + field->zip_dict_data.length, prebuilt); } /** Construct the translation table for reordering, dropping or @@ -2365,7 +2381,8 @@ innobase_build_col_map( const dict_table_t* new_table, const dict_table_t* old_table, dtuple_t* add_cols, - mem_heap_t* heap) + mem_heap_t* heap, + row_prebuilt_t* prebuilt) { DBUG_ENTER("innobase_build_col_map"); DBUG_ASSERT(altered_table != table); @@ -2404,7 +2421,7 @@ innobase_build_col_map( innobase_build_col_map_add( heap, dtuple_get_nth_field(add_cols, i), altered_table->field[i], - dict_table_is_comp(new_table)); + dict_table_is_comp(new_table), prebuilt); found_col: i++; } @@ -2567,7 +2584,8 @@ prepare_inplace_alter_table_dict( ulint flags2, ulint fts_doc_id_col, bool add_fts_doc_id, - bool add_fts_doc_id_idx) + bool add_fts_doc_id_idx, + row_prebuilt_t* prebuilt) { bool dict_locked = false; ulint* add_key_nums; /* MySQL key numbers */ @@ -2578,6 +2596,7 @@ prepare_inplace_alter_table_dict( dberr_t error; ulint num_fts_index; ha_innobase_inplace_ctx*ctx; + ulint* zip_dict_ids = 0; DBUG_ENTER("prepare_inplace_alter_table_dict"); @@ -2712,6 +2731,18 @@ prepare_inplace_alter_table_dict( ctx->new_table->id); ulint n_cols; dtuple_t* add_cols; + const char* err_zip_dict_name = 0; + + zip_dict_ids = static_cast( + mem_heap_alloc(ctx->heap, + altered_table->s->fields * sizeof(ulint))); + + if (!innobase_check_zip_dicts(altered_table, zip_dict_ids, + ctx->trx, &err_zip_dict_name)) { + my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, + MYF(0), err_zip_dict_name); + goto new_clustered_failed; + } if (innobase_check_foreigns( ha_alter_info, altered_table, old_table, @@ -2815,6 +2846,12 @@ prepare_inplace_alter_table_dict( } } + if (field->column_format() == + COLUMN_FORMAT_TYPE_COMPRESSED) { + field_type |= DATA_COMPRESSED; + } + + if (dict_col_name_is_reserved(field->field_name)) { dict_mem_table_free(ctx->new_table); my_error(ER_WRONG_COLUMN_NAME, MYF(0), @@ -2894,7 +2931,7 @@ prepare_inplace_alter_table_dict( ctx->col_map = innobase_build_col_map( ha_alter_info, altered_table, old_table, ctx->new_table, user_table, - add_cols, ctx->heap); + add_cols, ctx->heap, prebuilt); ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info)); @@ -3072,6 +3109,15 @@ prepare_inplace_alter_table_dict( DBUG_ASSERT(error == DB_SUCCESS); + /* + Adding compression dictionary <-> compressed table column links + to the SYS_ZIP_DICT_COLS table. + */ + if (zip_dict_ids != 0) { + innobase_create_zip_dict_references(altered_table, + ctx->trx->table_id, zip_dict_ids, ctx->trx); + } + /* Commit the data dictionary transaction in order to release the table locks on the system tables. This means that if MySQL crashes while creating a new primary key inside @@ -3767,7 +3813,7 @@ ha_innobase::prepare_inplace_alter_table( } if (!(ha_alter_info->handler_flags & INNOBASE_ALTER_DATA) - || (ha_alter_info->handler_flags + || ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) == Alter_inplace_info::CHANGE_CREATE_OPTION && !innobase_need_rebuild(ha_alter_info))) { @@ -3893,7 +3939,7 @@ ha_innobase::prepare_inplace_alter_table( table_share->table_name.str, flags, flags2, fts_doc_col_no, add_fts_doc_id, - add_fts_doc_id_idx)); + add_fts_doc_id_idx, prebuilt)); } /** Alter the table structure in-place with operations @@ -3933,7 +3979,7 @@ ha_innobase::inplace_alter_table( DBUG_RETURN(false); } - if (ha_alter_info->handler_flags + if ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) == Alter_inplace_info::CHANGE_CREATE_OPTION && !innobase_need_rebuild(ha_alter_info)) { goto ok_exit; diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc index dfdad55ec3bef..b351e464a1eac 100644 --- a/storage/xtradb/handler/i_s.cc +++ b/storage/xtradb/handler/i_s.cc @@ -4050,6 +4050,8 @@ i_s_fts_config_fill( DBUG_RETURN(0); } + DEBUG_SYNC_C("i_s_fts_config_fille_check"); + fields = table->field; /* Prevent DDL to drop fts aux tables. */ diff --git a/storage/xtradb/handler/xtradb_i_s.cc b/storage/xtradb/handler/xtradb_i_s.cc index 213e3c1aa53d9..9176378447608 100644 --- a/storage/xtradb/handler/xtradb_i_s.cc +++ b/storage/xtradb/handler/xtradb_i_s.cc @@ -32,9 +32,11 @@ this program; if not, write to the Free Software Foundation, Inc., #include #include #include "srv0start.h" /* for srv_was_started */ +#include /* btr_pcur_t */ #include /* btr_search_sys */ #include /* recv_sys */ #include +#include /* for ZIP_DICT_MAX_* constants */ /* for XTRADB_RSEG table */ #include "trx0trx.h" /* for TRX_QUE_STATE_STR_MAX_LEN */ @@ -130,6 +132,28 @@ field_store_string( return(ret); } +/** Auxiliary function to store (char*, len) value in MYSQL_TYPE_BLOB +field. +@return 0 on success */ +static +int +field_store_blob( + Field* field, /*!< in/out: target field for storage */ + const char* data, /*!< in: pointer to data, or NULL */ + uint data_len) /*!< in: data length */ +{ + int ret; + + if (data != NULL) { + ret = field->store(data, data_len, system_charset_info); + field->set_notnull(); + } else { + ret = 0; /* success */ + field->set_null(); + } + + return(ret); +} static int @@ -603,3 +627,329 @@ UNIV_INTERN struct st_mysql_plugin i_s_xtradb_rseg = STRUCT_FLD(__reserved1, NULL), STRUCT_FLD(flags, 0UL), }; + + +/************************************************************************/ +enum zip_dict_field_type +{ + zip_dict_field_id, + zip_dict_field_name, + zip_dict_field_zip_dict +}; + +static ST_FIELD_INFO xtradb_sys_zip_dict_fields_info[] = +{ + { STRUCT_FLD(field_name, "id"), + STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), + STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), + STRUCT_FLD(value, 0), + STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), + STRUCT_FLD(old_name, ""), + STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, + + { STRUCT_FLD(field_name, "name"), + STRUCT_FLD(field_length, ZIP_DICT_MAX_NAME_LENGTH), + STRUCT_FLD(field_type, MYSQL_TYPE_STRING), + STRUCT_FLD(value, 0), + STRUCT_FLD(field_flags, 0), + STRUCT_FLD(old_name, ""), + STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, + + { STRUCT_FLD(field_name, "zip_dict"), + STRUCT_FLD(field_length, ZIP_DICT_MAX_DATA_LENGTH), + STRUCT_FLD(field_type, MYSQL_TYPE_BLOB), + STRUCT_FLD(value, 0), + STRUCT_FLD(field_flags, 0), + STRUCT_FLD(old_name, ""), + STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, + + END_OF_ST_FIELD_INFO +}; + +/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT with information +collected by scanning SYS_ZIP_DICT table. +@return 0 on success */ +static +int +xtradb_i_s_dict_fill_sys_zip_dict( + THD* thd, /*!< in: thread */ + ulint id, /*!< in: dict ID */ + const char* name, /*!< in: dict name */ + const char* data, /*!< in: dict data */ + ulint data_len, /*!< in: dict data length */ + TABLE* table_to_fill) /*!< in/out: fill this table */ +{ + DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict"); + + Field** fields = table_to_fill->field; + + OK(field_store_ulint(fields[zip_dict_field_id], id)); + OK(field_store_string(fields[zip_dict_field_name], name)); + OK(field_store_blob(fields[zip_dict_field_zip_dict], data, + data_len)); + + OK(schema_table_store_record(thd, table_to_fill)); + + DBUG_RETURN(0); +} + +/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. +Loop through each record in SYS_ZIP_DICT, and extract the column +information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. +@return 0 on success */ +static +int +xtradb_i_s_sys_zip_dict_fill_table( + THD* thd, /*!< in: thread */ + TABLE_LIST* tables, /*!< in/out: tables to fill */ + Item* ) /*!< in: condition (not used) */ +{ + btr_pcur_t pcur; + const rec_t* rec; + mem_heap_t* heap; + mtr_t mtr; + + DBUG_ENTER("xtradb_i_s_sys_zip_dict_fill_table"); + RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); + + /* deny access to user without SUPER_ACL privilege */ + if (check_global_access(thd, SUPER_ACL)) { + DBUG_RETURN(0); + } + + heap = mem_heap_create(1000); + mutex_enter(&dict_sys->mutex); + mtr_start(&mtr); + + rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT); + ulint zip_size = dict_table_zip_size(pcur.btr_cur.index->table); + + while (rec) { + const char* err_msg; + ulint id; + const char* name; + const char* data; + ulint data_len; + + /* Extract necessary information from a SYS_ZIP_DICT row */ + err_msg = dict_process_sys_zip_dict( + heap, zip_size, rec, &id, &name, &data, &data_len); + + mtr_commit(&mtr); + mutex_exit(&dict_sys->mutex); + + if (!err_msg) { + xtradb_i_s_dict_fill_sys_zip_dict( + thd, id, name, data, data_len, + tables->table); + } else { + push_warning_printf(thd, + Sql_condition::WARN_LEVEL_WARN, + ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); + } + + mem_heap_empty(heap); + + /* Get the next record */ + mutex_enter(&dict_sys->mutex); + mtr_start(&mtr); + rec = dict_getnext_system(&pcur, &mtr); + } + + mtr_commit(&mtr); + mutex_exit(&dict_sys->mutex); + mem_heap_free(heap); + + DBUG_RETURN(0); +} + +static int i_s_xtradb_zip_dict_init(void* p) +{ + DBUG_ENTER("i_s_xtradb_zip_dict_init"); + + ST_SCHEMA_TABLE* schema = static_cast(p); + + schema->fields_info = xtradb_sys_zip_dict_fields_info; + schema->fill_table = xtradb_i_s_sys_zip_dict_fill_table; + + DBUG_RETURN(0); +} + +UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict = +{ + STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), + STRUCT_FLD(info, &i_s_info), + STRUCT_FLD(name, "XTRADB_ZIP_DICT"), + STRUCT_FLD(author, PLUGIN_AUTHOR), + STRUCT_FLD(descr, "InnoDB compression dictionaries information"), + STRUCT_FLD(license, PLUGIN_LICENSE_GPL), + STRUCT_FLD(init, i_s_xtradb_zip_dict_init), + STRUCT_FLD(deinit, i_s_common_deinit), + STRUCT_FLD(version, INNODB_VERSION_SHORT), + STRUCT_FLD(status_vars, NULL), + STRUCT_FLD(system_vars, NULL), + STRUCT_FLD(__reserved1, NULL), + STRUCT_FLD(flags, 0UL), +}; + +enum zip_dict_cols_field_type +{ + zip_dict_cols_field_table_id, + zip_dict_cols_field_column_pos, + zip_dict_cols_field_dict_id +}; + +static ST_FIELD_INFO xtradb_sys_zip_dict_cols_fields_info[] = +{ + { STRUCT_FLD(field_name, "table_id"), + STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), + STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), + STRUCT_FLD(value, 0), + STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), + STRUCT_FLD(old_name, ""), + STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, + + { STRUCT_FLD(field_name, "column_pos"), + STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), + STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), + STRUCT_FLD(value, 0), + STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), + STRUCT_FLD(old_name, ""), + STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, + + { STRUCT_FLD(field_name, "dict_id"), + STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), + STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), + STRUCT_FLD(value, 0), + STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), + STRUCT_FLD(old_name, ""), + STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, + + END_OF_ST_FIELD_INFO +}; + +/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS with information +collected by scanning SYS_ZIP_DICT_COLS table. +@return 0 on success */ +static +int +xtradb_i_s_dict_fill_sys_zip_dict_cols( + THD* thd, /*!< in: thread */ + ulint table_id, /*!< in: table ID */ + ulint column_pos, /*!< in: column position */ + ulint dict_id, /*!< in: dict ID */ + TABLE* table_to_fill) /*!< in/out: fill this table */ +{ + DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict_cols"); + + Field** fields = table_to_fill->field; + + OK(field_store_ulint(fields[zip_dict_cols_field_table_id], + table_id)); + OK(field_store_ulint(fields[zip_dict_cols_field_column_pos], + column_pos)); + OK(field_store_ulint(fields[zip_dict_cols_field_dict_id], + dict_id)); + + OK(schema_table_store_record(thd, table_to_fill)); + + DBUG_RETURN(0); +} + +/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. +Loop through each record in SYS_ZIP_DICT_COLS, and extract the column +information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. +@return 0 on success */ +static +int +xtradb_i_s_sys_zip_dict_cols_fill_table( + THD* thd, /*!< in: thread */ + TABLE_LIST* tables, /*!< in/out: tables to fill */ + Item* ) /*!< in: condition (not used) */ +{ + btr_pcur_t pcur; + const rec_t* rec; + mem_heap_t* heap; + mtr_t mtr; + + DBUG_ENTER("xtradb_i_s_sys_zip_dict_cols_fill_table"); + RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); + + /* deny access to user without SUPER_ACL privilege */ + if (check_global_access(thd, SUPER_ACL)) { + DBUG_RETURN(0); + } + + heap = mem_heap_create(1000); + mutex_enter(&dict_sys->mutex); + mtr_start(&mtr); + + rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT_COLS); + + while (rec) { + const char* err_msg; + ulint table_id; + ulint column_pos; + ulint dict_id; + + /* Extract necessary information from a SYS_ZIP_DICT_COLS + row */ + err_msg = dict_process_sys_zip_dict_cols( + heap, rec, &table_id, &column_pos, &dict_id); + + mtr_commit(&mtr); + mutex_exit(&dict_sys->mutex); + + if (!err_msg) { + xtradb_i_s_dict_fill_sys_zip_dict_cols( + thd, table_id, column_pos, dict_id, + tables->table); + } else { + push_warning_printf(thd, + Sql_condition::WARN_LEVEL_WARN, + ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); + } + + mem_heap_empty(heap); + + /* Get the next record */ + mutex_enter(&dict_sys->mutex); + mtr_start(&mtr); + rec = dict_getnext_system(&pcur, &mtr); + } + + mtr_commit(&mtr); + mutex_exit(&dict_sys->mutex); + mem_heap_free(heap); + + DBUG_RETURN(0); +} + +static int i_s_xtradb_zip_dict_cols_init(void* p) +{ + DBUG_ENTER("i_s_xtradb_zip_dict_cols_init"); + + ST_SCHEMA_TABLE* schema = static_cast(p); + + schema->fields_info = xtradb_sys_zip_dict_cols_fields_info; + schema->fill_table = xtradb_i_s_sys_zip_dict_cols_fill_table; + + DBUG_RETURN(0); +} + +UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict_cols = +{ + STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), + STRUCT_FLD(info, &i_s_info), + STRUCT_FLD(name, "XTRADB_ZIP_DICT_COLS"), + STRUCT_FLD(author, PLUGIN_AUTHOR), + STRUCT_FLD(descr, "InnoDB compressed columns information"), + STRUCT_FLD(license, PLUGIN_LICENSE_GPL), + STRUCT_FLD(init, i_s_xtradb_zip_dict_cols_init), + STRUCT_FLD(deinit, i_s_common_deinit), + STRUCT_FLD(version, INNODB_VERSION_SHORT), + STRUCT_FLD(status_vars, NULL), + STRUCT_FLD(system_vars, NULL), + STRUCT_FLD(__reserved1, NULL), + STRUCT_FLD(flags, 0UL), +}; diff --git a/storage/xtradb/handler/xtradb_i_s.h b/storage/xtradb/handler/xtradb_i_s.h index 2f7552c565adc..905d84587affd 100644 --- a/storage/xtradb/handler/xtradb_i_s.h +++ b/storage/xtradb/handler/xtradb_i_s.h @@ -22,5 +22,7 @@ this program; if not, write to the Free Software Foundation, Inc., extern struct st_mysql_plugin i_s_xtradb_read_view; extern struct st_mysql_plugin i_s_xtradb_internal_hash_tables; extern struct st_mysql_plugin i_s_xtradb_rseg; +extern struct st_mysql_plugin i_s_xtradb_zip_dict; +extern struct st_mysql_plugin i_s_xtradb_zip_dict_cols; #endif /* XTRADB_I_S_H */ diff --git a/storage/xtradb/include/data0type.h b/storage/xtradb/include/data0type.h index 111664b0b527f..f269c266efb9b 100644 --- a/storage/xtradb/include/data0type.h +++ b/storage/xtradb/include/data0type.h @@ -170,6 +170,9 @@ be less than 256 */ type when the column is true VARCHAR where MySQL uses 2 bytes to store the data len; for shorter VARCHARs MySQL uses only 1 byte */ +#define DATA_COMPRESSED 16384 /* this is ORed to the precise data + type when the column has COLUMN_FORMAT = + COMPRESSED attribute*/ /*-------------------------------------------*/ /* This many bytes we need to store the type information affecting the @@ -500,6 +503,17 @@ dtype_print( /*========*/ const dtype_t* type); /*!< in: type */ +/** +Calculates the number of extra bytes needed for compression header +depending on precise column type. +@reval 0 if prtype does not include DATA_COMPRESSED flag +@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag +*/ +UNIV_INLINE +ulint +prtype_get_compression_extra( + ulint prtype); /*!< in: precise type */ + /* Structure for an SQL data type. If you add fields to this structure, be sure to initialize them everywhere. This structure is initialized in the following functions: diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index d489bef89a8bb..29dc480a19c3e 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -26,6 +26,7 @@ Created 1/16/1996 Heikki Tuuri #include /* strlen() */ #include "mach0data.h" +#include "rem0types.h" /* ZIP_COLUMN_HEADER_LENGTH */ #ifndef UNIV_HOTBACKUP # include "ha_prototypes.h" @@ -709,3 +710,18 @@ dtype_get_sql_null_size( 0, 0)); #endif /* !UNIV_HOTBACKUP */ } + +/** +Calculates the number of extra bytes needed for compression header +depending on precise column type. +@reval 0 if prtype does not include DATA_COMPRESSED flag +@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag +*/ +UNIV_INLINE +ulint +prtype_get_compression_extra( + ulint prtype) /*!< in: precise type */ +{ + return (prtype & DATA_COMPRESSED) != 0 ? + ZIP_COLUMN_HEADER_LENGTH : 0; +} diff --git a/storage/xtradb/include/dict0boot.h b/storage/xtradb/include/dict0boot.h index 477e1150f437b..d5bee886cbf43 100644 --- a/storage/xtradb/include/dict0boot.h +++ b/storage/xtradb/include/dict0boot.h @@ -324,6 +324,38 @@ enum dict_fld_sys_datafiles_enum { DICT_FLD__SYS_DATAFILES__PATH = 3, DICT_NUM_FIELDS__SYS_DATAFILES = 4 }; +/* The columns in SYS_DICT */ +enum dict_col_sys_zip_dict_enum { + DICT_COL__SYS_ZIP_DICT__ID = 0, + DICT_COL__SYS_ZIP_DICT__NAME = 1, + DICT_COL__SYS_ZIP_DICT__DATA = 2, + DICT_NUM_COLS__SYS_ZIP_DICT = 3 +}; +/* The field numbers in the SYS_DICT clustered index */ +enum dict_fld_sys_zip_dict_enum { + DICT_FLD__SYS_ZIP_DICT__ID = 0, + DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID = 1, + DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR = 2, + DICT_FLD__SYS_ZIP_DICT__NAME = 3, + DICT_FLD__SYS_ZIP_DICT__DATA = 4, + DICT_NUM_FIELDS__SYS_ZIP_DICT = 5 +}; +/* The columns in SYS_DICT_COLS */ +enum dict_col_sys_zip_dict_cols_enum { + DICT_COL__SYS_ZIP_DICT_COLS__TABLE_ID = 0, + DICT_COL__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, + DICT_COL__SYS_ZIP_DICT_COLS__DICT_ID = 2, + DICT_NUM_COLS__SYS_ZIP_DICT_COLS = 3 +}; +/* The field numbers in the SYS_DICT_COLS clustered index */ +enum dict_fld_sys_zip_dict_cols_enum { + DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID = 0, + DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, + DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID = 2, + DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR = 3, + DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID = 4, + DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS = 5 +}; /* A number of the columns above occur in multiple tables. These are the length of thos fields. */ diff --git a/storage/xtradb/include/dict0crea.h b/storage/xtradb/include/dict0crea.h index 6146917469a84..33877b678345f 100644 --- a/storage/xtradb/include/dict0crea.h +++ b/storage/xtradb/include/dict0crea.h @@ -152,6 +152,19 @@ UNIV_INTERN dberr_t dict_create_or_check_sys_tablespace(void); /*=====================================*/ + +#define ZIP_DICT_MAX_NAME_LENGTH 64 +/* Max window size (2^15) minus 262 */ +#define ZIP_DICT_MAX_DATA_LENGTH 32506 + +/** Creates the zip_dict system table inside InnoDB +at server bootstrap or server start if it is not found or is +not of the right form. +@return DB_SUCCESS or error code */ +UNIV_INTERN +dberr_t +dict_create_or_check_sys_zip_dict(void); + /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -167,6 +180,84 @@ dict_create_add_tablespace_to_dictionary( trx_t* trx, /*!< in: transaction */ bool commit); /*!< in: if true then commit the transaction */ + +/** Add a single compression dictionary definition to the SYS_ZIP_DICT +InnoDB system table. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_add_zip_dict( + const char* name, /*!< in: dict name */ + ulint name_len, /*!< in: dict name length */ + const char* data, /*!< in: dict data */ + ulint data_len, /*!< in: dict data length */ + trx_t* trx); /*!< in/out: transaction */ + +/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS +InnoDB system table. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_add_zip_dict_reference( + ulint table_id, /*!< in: table id */ + ulint column_pos, /*!< in: column position */ + ulint dict_id, /*!< in: dict id */ + trx_t* trx); /*!< in/out: transaction */ + +/** Get a single compression dictionary id for the given +(table id, column pos) pair. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_get_zip_dict_id_by_reference( + ulint table_id, /*!< in: table id */ + ulint column_pos, /*!< in: column position */ + ulint* dict_id, /*!< out: dict id */ + trx_t* trx); /*!< in/out: transaction */ + +/** Get compression dictionary id for the given name. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_get_zip_dict_id_by_name( + const char* dict_name, /*!< in: dict name */ + ulint dict_name_len, /*!< in: dict name length */ + ulint* dict_id, /*!< out: dict id */ + trx_t* trx); /*!< in/out: transaction */ + +/** Get compression dictionary info (name and data) for the given id. +Allocates memory for name and data on success. +Must be freed with mem_free(). +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_get_zip_dict_info_by_id( + ulint dict_id, /*!< in: dict id */ + char** name, /*!< out: dict name */ + ulint* name_len, /*!< out: dict name length */ + char** data, /*!< out: dict data */ + ulint* data_len, /*!< out: dict data length */ + trx_t* trx); /*!< in/out: transaction */ + +/** Remove a single compression dictionary from the data dictionary +tables in the database. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_remove_zip_dict( + const char* name, /*!< in: dict name */ + ulint name_len, /*!< in: dict name length */ + trx_t* trx); /*!< in/out: transaction */ + +/** Remove all compression dictionary references for the given table ID from +the data dictionary tables in the database. +@return error code or DB_SUCCESS */ +UNIV_INTERN +dberr_t +dict_create_remove_zip_dict_references_for_table( + ulint table_id, /*!< in: table id */ + trx_t* trx); /*!< in/out: transaction */ + /********************************************************************//** Add a foreign key definition to the data dictionary tables. @return error code or DB_SUCCESS */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index f539f62960bfe..870b142ba3261 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -1845,6 +1845,52 @@ dict_table_set_corrupt_by_space( ulint space_id, ibool need_mutex); +/** Insert a records into SYS_ZIP_DICT. +@retval DB_SUCCESS if OK +@retval dberr_t if the insert failed */ +UNIV_INTERN +dberr_t +dict_create_zip_dict( + const char* name, /*!< in: zip_dict name */ + ulint name_len, /*!< in: zip_dict name length*/ + const char* data, /*!< in: zip_dict data */ + ulint data_len); /*!< in: zip_dict data length */ + +/** Get single compression dictionary id for the given +(table id, column pos) pair. +@retval DB_SUCCESS if OK +@retval DB_RECORD_NOT_FOUND if not found */ +UNIV_INTERN +dberr_t +dict_get_dictionary_id_by_key( + ulint table_id, /*!< in: table id */ + ulint column_pos, /*!< in: column position */ + ulint* dict_id); /*!< out: zip_dict id */ + +/** Get compression dictionary info (name and data) for the given id. +Allocates memory in name->str and data->str on success. +Must be freed with mem_free(). +@retval DB_SUCCESS if OK +@retval DB_RECORD_NOT_FOUND if not found */ +UNIV_INTERN +dberr_t +dict_get_dictionary_info_by_id( + ulint dict_id, /*!< in: table name */ + char** name, /*!< out: dictionary name */ + ulint* name_len, /*!< out: dictionary name length*/ + char** data, /*!< out: dictionary data */ + ulint* data_len); /*!< out: dictionary data length*/ + +/** Delete a record in SYS_ZIP_DICT with the given name. +@retval DB_SUCCESS if OK +@retval DB_RECORD_NOT_FOUND if not found +@retval DB_ROW_IS_REFERENCED if in use */ +UNIV_INTERN +dberr_t +dict_drop_zip_dict( + const char* name, /*!< in: zip_dict name */ + ulint name_len); /*!< in: zip_dict name length*/ + #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/xtradb/include/dict0load.h b/storage/xtradb/include/dict0load.h index dcbc3de8e942f..85e3e5656371b 100644 --- a/storage/xtradb/include/dict0load.h +++ b/storage/xtradb/include/dict0load.h @@ -44,6 +44,8 @@ enum dict_system_id_t { SYS_FOREIGN_COLS, SYS_TABLESPACES, SYS_DATAFILES, + SYS_ZIP_DICT, + SYS_ZIP_DICT_COLS, /* This must be last item. Defines the number of system tables. */ SYS_NUM_SYSTEM_TABLES @@ -386,6 +388,33 @@ dict_process_sys_datafiles( const rec_t* rec, /*!< in: current SYS_DATAFILES rec */ ulint* space, /*!< out: pace id */ const char** path); /*!< out: datafile path */ + +/** This function parses a SYS_ZIP_DICT record, extracts necessary +information from the record and returns to caller. +@return error message, or NULL on success */ +UNIV_INTERN +const char* +dict_process_sys_zip_dict( + mem_heap_t* heap, /*!< in/out: heap memory */ + ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ + const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ + ulint* id, /*!< out: dict id */ + const char** name, /*!< out: dict name */ + const char** data, /*!< out: dict data */ + ulint* data_len); /*!< out: dict data length */ + +/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary +information from the record and returns to caller. +@return error message, or NULL on success */ +UNIV_INTERN +const char* +dict_process_sys_zip_dict_cols( + mem_heap_t* heap, /*!< in/out: heap memory */ + const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ + ulint* table_id, /*!< out: table id */ + ulint* column_pos, /*!< out: column position */ + ulint* dict_id); /*!< out: dict id */ + /********************************************************************//** Get the filepath for a spaceid from SYS_DATAFILES. This function provides a temporary heap which is used for the table lookup, but not for the path. diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h index 87b5787d416be..3e2f359bbebef 100644 --- a/storage/xtradb/include/fts0fts.h +++ b/storage/xtradb/include/fts0fts.h @@ -375,6 +375,7 @@ extern bool fts_need_sync; /** Variable specifying the table that has Fulltext index to display its content through information schema table */ extern char* fts_internal_tbl_name; +extern char* fts_internal_tbl_name2; #define fts_que_graph_free(graph) \ do { \ @@ -823,6 +824,15 @@ void fts_drop_orphaned_tables(void); /*==========================*/ +/* Get parent table name if it's a fts aux table +@param[in] aux_table_name aux table name +@param[in] aux_table_len aux table length +@return parent table name, or NULL */ +char* +fts_get_parent_table_name( + const char* aux_table_name, + ulint aux_table_len); + /******************************************************************//** Since we do a horizontal split on the index table, we need to drop all the split tables. diff --git a/storage/xtradb/include/os0thread.h b/storage/xtradb/include/os0thread.h index e36f836e0be34..44ff5e6757e4a 100644 --- a/storage/xtradb/include/os0thread.h +++ b/storage/xtradb/include/os0thread.h @@ -131,14 +131,27 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ +/** +Waits until the specified thread completes and joins it. Its return value is +ignored. + +@param thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread); + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach = true) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** Returns the thread identifier of current thread. diff --git a/storage/xtradb/include/rem0types.h b/storage/xtradb/include/rem0types.h index f8133f77466d9..5da96066f8851 100644 --- a/storage/xtradb/include/rem0types.h +++ b/storage/xtradb/include/rem0types.h @@ -71,4 +71,7 @@ enum rec_format_enum { }; typedef enum rec_format_enum rec_format_t; +/** Compressed field header size in bytes */ +#define ZIP_COLUMN_HEADER_LENGTH 2 + #endif diff --git a/storage/xtradb/include/row0mysql.h b/storage/xtradb/include/row0mysql.h index fc1846b76f312..27d3adfc7f07c 100644 --- a/storage/xtradb/include/row0mysql.h +++ b/storage/xtradb/include/row0mysql.h @@ -41,6 +41,9 @@ struct SysIndexCallback; extern ibool row_rollback_on_timeout; +extern uint srv_compressed_columns_zip_level; +extern ulong srv_compressed_columns_threshold; + struct row_prebuilt_t; /*******************************************************************//** @@ -51,6 +54,49 @@ row_mysql_prebuilt_free_blob_heap( /*==============================*/ row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a ha_innobase:: table handle */ + +/** Frees the compress heap in prebuilt when no longer needed. */ +UNIV_INTERN +void +row_mysql_prebuilt_free_compress_heap( + row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a + ha_innobase:: table handle */ + +/** Uncompress blob/text/varchar column using zlib +@return pointer to the uncompressed data */ +const byte* +row_decompress_column( + const byte* data, /*!< in: data in innodb(compressed) format */ + ulint *len, /*!< in: data length; out: length of + decompressed data*/ + const byte* dict_data, + /*!< in: optional dictionary data used for + decompression */ + ulint dict_data_len, + /*!< in: optional dictionary data length */ + row_prebuilt_t* prebuilt); + /*!< in: use prebuilt->compress_heap only + here*/ + +/** Compress blob/text/varchar column using zlib +@return pointer to the compressed data */ +byte* +row_compress_column( + const byte* data, /*!< in: data in mysql(uncompressed) + format */ + ulint *len, /*!< in: data length; out: length of + compressed data*/ + ulint lenlen, /*!< in: bytes used to store the length of + data */ + const byte* dict_data, + /*!< in: optional dictionary data used for + compression */ + ulint dict_data_len, + /*!< in: optional dictionary data length */ + row_prebuilt_t* prebuilt); + /*!< in: use prebuilt->compress_heap only + here*/ + /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -89,10 +135,21 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len); /*!< in: BLOB length; if the value to store + ulint len, /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ + bool need_decompression, + /*!< in: if the data need to be compressed*/ + const byte* dict_data, + /*!< in: optional compression dictionary + data */ + ulint dict_data_len, + /*!< in: optional compression dictionary data + length */ + row_prebuilt_t* prebuilt); + /*compress_heap only + here */ /*******************************************************************//** Reads a reference to a BLOB in the MySQL format. @return pointer to BLOB data */ @@ -103,8 +160,17 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len); /*!< in: BLOB reference length + ulint col_len, /*!< in: BLOB reference length (not BLOB length) */ + bool need_compression, + /*!< in: if the data need to be + compressed*/ + const byte* dict_data, /*!< in: optional compression + dictionary data */ + ulint dict_data_len, /*!< in: optional compression + dictionary data length */ + row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap + only here */ /**************************************************************//** Pad a column with spaces. */ UNIV_INTERN @@ -152,7 +218,16 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp); /*!< in: nonzero=compact format */ + ulint comp, /*!< in: nonzero=compact format */ + bool need_compression, + /*!< in: if the data need to be + compressed */ + const byte* dict_data, /*!< in: optional compression + dictionary data */ + ulint dict_data_len, /*!< in: optional compression + dictionary data length */ + row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap + only here */ /****************************************************************//** Handles user errors and lock waits detected by the database engine. @return true if it was a lock wait and we should continue running the @@ -643,6 +718,8 @@ struct mysql_row_templ_t { ulint is_unsigned; /*!< if a column type is an integer type and this field is != 0, then it is an unsigned integer type */ + bool compressed; /*!< if column format is compressed */ + LEX_CSTRING zip_dict_data; /*!< associated compression dictionary */ }; #define MYSQL_FETCH_CACHE_SIZE 8 @@ -839,6 +916,8 @@ struct row_prebuilt_t { in fetch_cache */ mem_heap_t* blob_heap; /*!< in SELECTS BLOB fields are copied to this heap */ + mem_heap_t* compress_heap; /*!< memory heap used to compress + /decompress blob column*/ mem_heap_t* old_vers_heap; /*!< memory heap where a previous version is built in consistent read */ bool in_fts_query; /*!< Whether we are in a FTS query */ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 692d339608a2e..09f305091c2b0 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -487,6 +487,9 @@ extern ibool srv_priority_boost; extern ulint srv_truncated_status_writes; extern ulint srv_available_undo_logs; +extern ulint srv_column_compressed; +extern ulint srv_column_decompressed; + extern ulint srv_mem_pool_size; extern ulint srv_lock_table_size; @@ -1079,6 +1082,8 @@ struct export_var_t{ ulint innodb_purge_view_trx_id_age; /*!< rw_max_trx_id - purged view's min trx_id */ #endif /* UNIV_DEBUG */ + ulint innodb_column_compressed; /*!< srv_column_compressed */ + ulint innodb_column_decompressed; /*!< srv_column_decompressed */ }; /** Thread slot in the thread table. */ diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 4d64e3249c0b4..296c04d9f621b 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -47,7 +47,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_BUGFIX MYSQL_VERSION_PATCH #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 78.1 +#define PERCONA_INNODB_VERSION 79.0 #endif /* Enable UNIV_LOG_ARCHIVE in XtraDB */ diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 0768bb6bb003a..7784e8538b7c5 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -975,6 +975,7 @@ log_init(void) log_sys->next_checkpoint_no = 0; log_sys->last_checkpoint_lsn = log_sys->lsn; + log_sys->next_checkpoint_lsn = log_sys->lsn; log_sys->n_pending_checkpoint_writes = 0; @@ -1891,6 +1892,7 @@ log_complete_checkpoint(void) log_sys->next_checkpoint_no++; + ut_ad(log_sys->next_checkpoint_lsn >= log_sys->last_checkpoint_lsn); log_sys->last_checkpoint_lsn = log_sys->next_checkpoint_lsn; MONITOR_SET(MONITOR_LSN_CHECKPOINT_AGE, log_sys->lsn - log_sys->last_checkpoint_lsn); @@ -1978,11 +1980,17 @@ log_group_checkpoint( ulint i; ut_ad(!srv_read_only_mode); + ut_ad(srv_shutdown_state != SRV_SHUTDOWN_LAST_PHASE); ut_ad(mutex_own(&(log_sys->mutex))); ut_a(LOG_CHECKPOINT_SIZE <= OS_FILE_LOG_BLOCK_SIZE); buf = group->checkpoint_buf; +#ifdef UNIV_DEBUG + lsn_t old_next_checkpoint_lsn + = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + ut_ad(old_next_checkpoint_lsn <= log_sys->next_checkpoint_lsn); +#endif /* UNIV_DEBUG */ mach_write_to_8(buf + LOG_CHECKPOINT_NO, log_sys->next_checkpoint_no); mach_write_to_8(buf + LOG_CHECKPOINT_LSN, log_sys->next_checkpoint_lsn); @@ -2242,6 +2250,7 @@ log_checkpoint( return(FALSE); } + ut_ad(oldest_lsn >= log_sys->next_checkpoint_lsn); log_sys->next_checkpoint_lsn = oldest_lsn; #ifdef UNIV_DEBUG @@ -3490,13 +3499,15 @@ logs_empty_and_mark_files_at_shutdown(void) before proceeding further. */ srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; count = 0; - while (buf_page_cleaner_is_active) { - ++count; - os_thread_sleep(100000); - if (srv_print_verbose_log && count > 600) { + while (buf_page_cleaner_is_active || buf_lru_manager_is_active) { + if (srv_print_verbose_log && count == 0) { ib_logf(IB_LOG_LEVEL_INFO, "Waiting for page_cleaner to " "finish flushing of buffer pool"); + } + ++count; + os_thread_sleep(100000); + if (count > 600) { count = 0; } } @@ -3664,6 +3675,7 @@ logs_empty_and_mark_files_at_shutdown(void) ut_a(freed); ut_a(lsn == log_sys->lsn); + ut_ad(lsn == log_sys->last_checkpoint_lsn); if (lsn < srv_start_lsn) { ib_logf(IB_LOG_LEVEL_ERROR, diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index d80cb2ad4471d..46f544178d2ff 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -441,6 +441,7 @@ log_online_track_missing_on_startup( current server startup */ { ut_ad(last_tracked_lsn != tracking_start_lsn); + ut_ad(srv_track_changed_pages); ib_logf(IB_LOG_LEVEL_WARN, "last tracked LSN in \'%s\' is " LSN_PF ", but the last checkpoint LSN is " LSN_PF ". This might be " @@ -623,6 +624,8 @@ log_online_read_init(void) compile_time_assert(MODIFIED_PAGE_BLOCK_BITMAP % 8 == 0); compile_time_assert(MODIFIED_PAGE_BLOCK_BITMAP_LEN % 8 == 0); + ut_ad(srv_track_changed_pages); + log_bmp_sys = static_cast (ut_malloc(sizeof(*log_bmp_sys))); log_bmp_sys->read_buf_ptr = static_cast @@ -1097,10 +1100,15 @@ log_online_write_bitmap_page( { ibool success; + ut_ad(srv_track_changed_pages); ut_ad(mutex_own(&log_bmp_sys->mutex)); /* Simulate a write error */ - DBUG_EXECUTE_IF("bitmap_page_write_error", return FALSE;); + DBUG_EXECUTE_IF("bitmap_page_write_error", + ib_logf(IB_LOG_LEVEL_ERROR, + "simulating bitmap write error in " + "log_online_write_bitmap_page"); + return FALSE;); success = os_file_write(log_bmp_sys->out.name, log_bmp_sys->out.file, block, log_bmp_sys->out.offset, @@ -1190,7 +1198,9 @@ log_online_write_bitmap(void) rbt_next(log_bmp_sys->modified_pages, bmp_tree_node); DBUG_EXECUTE_IF("bitmap_page_2_write_error", - DBUG_SET("+d,bitmap_page_write_error");); + ut_ad(bmp_tree_node); /* 2nd page must exist */ + DBUG_SET("+d,bitmap_page_write_error"); + DBUG_SET("-d,bitmap_page_2_write_error");); } rbt_reset(log_bmp_sys->modified_pages); @@ -1211,15 +1221,11 @@ log_online_follow_redo_log(void) log_group_t* group; ibool result; - mutex_enter(&log_bmp_sys->mutex); - - if (!srv_track_changed_pages) { - mutex_exit(&log_bmp_sys->mutex); - return FALSE; - } - + ut_ad(srv_track_changed_pages); ut_ad(!srv_read_only_mode); + mutex_enter(&log_bmp_sys->mutex); + /* Grab the LSN of the last checkpoint, we will parse up to it */ mutex_enter(&(log_sys->mutex)); log_bmp_sys->end_lsn = log_sys->last_checkpoint_lsn; @@ -1562,9 +1568,12 @@ log_online_diagnose_bitmap_eof( /* It's a "Warning" here because it's not a fatal error for the whole server */ ib_logf(IB_LOG_LEVEL_WARN, - "changed page bitmap file \'%s\' does not " - "contain a complete run at the end.", - bitmap_file->name); + "changed page bitmap file \'%s\', size " + UINT64PF " bytes, does not " + "contain a complete run at the next read " + "offset " UINT64PF, + bitmap_file->name, bitmap_file->size, + bitmap_file->offset); return FALSE; } } diff --git a/storage/xtradb/mach/mach0data.cc b/storage/xtradb/mach/mach0data.cc index df68aab8a1874..206434dc5ab37 100644 --- a/storage/xtradb/mach/mach0data.cc +++ b/storage/xtradb/mach/mach0data.cc @@ -56,7 +56,18 @@ mach_parse_compressed( *val = flag; return(ptr + 1); - } else if (flag < 0xC0UL) { + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) + asm volatile("": : :"memory"); +#endif + + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } diff --git a/storage/xtradb/os/os0thread.cc b/storage/xtradb/os/os0thread.cc index 1d417f9823c47..93f45e060f8fd 100644 --- a/storage/xtradb/os/os0thread.cc +++ b/storage/xtradb/os/os0thread.cc @@ -210,14 +210,33 @@ os_thread_create_func( #endif } +/** +Waits until the specified thread completes and joins it. Its return value is +ignored. + +@param thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread) +{ + int ret MY_ATTRIBUTE((unused)) = pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed */ + ut_ad(ret == 0 || ret == ESRCH); +} + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ { #ifdef UNIV_DEBUG_THREAD_CREATION fprintf(stderr, "Thread exits, id %lu\n", @@ -233,7 +252,8 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - pthread_detach(pthread_self()); + if (detach) + pthread_detach(pthread_self()); pthread_exit(exit_value); #endif } diff --git a/storage/xtradb/rem/rem0rec.cc b/storage/xtradb/rem/rem0rec.cc index a95e9c236139e..09cd810cd7b08 100644 --- a/storage/xtradb/rem/rem0rec.cc +++ b/storage/xtradb/rem/rem0rec.cc @@ -320,7 +320,8 @@ rec_init_offsets_comp_ordinary( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (UNIV_UNLIKELY(col->len > 255) + if (UNIV_UNLIKELY(col->len > 255 - + prtype_get_compression_extra(col->prtype)) || UNIV_UNLIKELY(col->mtype == DATA_BLOB)) { if (len & 0x80) { @@ -841,8 +842,12 @@ rec_get_converted_size_comp_prefix_low( continue; } - ut_ad(len <= col->len || col->mtype == DATA_BLOB - || (col->len == 0 && col->mtype == DATA_VARCHAR)); + ut_ad(len <= col->len || col->mtype == DATA_BLOB || + ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY + || col->mtype == DATA_VARMYSQL) + && (col->len == 0 + || len <= col->len + + prtype_get_compression_extra(col->prtype)))); fixed_len = field->fixed_len; if (temp && fixed_len @@ -874,7 +879,9 @@ rec_get_converted_size_comp_prefix_low( ut_ad(col->len >= 256 || col->mtype == DATA_BLOB); extra_size += 2; } else if (len < 128 - || (col->len < 256 && col->mtype != DATA_BLOB)) { + || (col->len < 256 - + prtype_get_compression_extra(col->prtype) + && col->mtype != DATA_BLOB)) { extra_size++; } else { /* For variable-length columns, we look up the @@ -1269,12 +1276,16 @@ rec_convert_dtuple_to_rec_comp( *lens-- = (byte) (len >> 8) | 0xc0; *lens-- = (byte) len; } else { - ut_ad(len <= dtype_get_len(type) + ut_ad(len <= dtype_get_len(type) + + prtype_get_compression_extra( + dtype_get_prtype(type)) || dtype_get_mtype(type) == DATA_BLOB || !strcmp(index->name, FTS_INDEX_TABLE_IND_NAME)); if (len < 128 - || (dtype_get_len(type) < 256 + || (dtype_get_len(type) < 256 - + prtype_get_compression_extra( + dtype_get_prtype(type)) && dtype_get_mtype(type) != DATA_BLOB)) { *lens-- = (byte) len; diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 6fac6c0d317e5..97f2b8d4b5d54 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -960,7 +960,7 @@ fts_parallel_merge( CloseHandle(psort_info->thread_hdl); #endif /*__WIN__ */ - os_thread_exit(NULL); + os_thread_exit(NULL, false); OS_THREAD_DUMMY_RETURN; } diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index a6751b208f74c..54183759e8d31 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -613,7 +613,7 @@ row_log_table_delete( &old_pk_extra_size); ut_ad(old_pk_extra_size < 0x100); - mrec_size = 4 + old_pk_size; + mrec_size = 6 + old_pk_size; /* Log enough prefix of the BLOB unless both the old and new table are in COMPACT or REDUNDANT format, @@ -643,8 +643,8 @@ row_log_table_delete( *b++ = static_cast(old_pk_extra_size); /* Log the size of external prefix we saved */ - mach_write_to_2(b, ext_size); - b += 2; + mach_write_to_4(b, ext_size); + b += 4; rec_convert_dtuple_to_temp( b + old_pk_extra_size, new_index, @@ -2268,14 +2268,14 @@ row_log_table_apply_op( break; case ROW_T_DELETE: - /* 1 (extra_size) + 2 (ext_size) + at least 1 (payload) */ - if (mrec + 4 >= mrec_end) { + /* 1 (extra_size) + 4 (ext_size) + at least 1 (payload) */ + if (mrec + 6 >= mrec_end) { return(NULL); } extra_size = *mrec++; - ext_size = mach_read_from_2(mrec); - mrec += 2; + ext_size = mach_read_from_4(mrec); + mrec += 4; ut_ad(mrec < mrec_end); /* We assume extra_size < 0x100 for the PRIMARY KEY prefix. diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index feb18c82ab62a..3f50504bec8d6 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -523,7 +523,12 @@ row_merge_buf_add( dfield_set_len(field, len); } - ut_ad(len <= col->len || col->mtype == DATA_BLOB); + ut_ad(len <= col->len || col->mtype == DATA_BLOB || + ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY + || col->mtype == DATA_VARMYSQL) + && (col->len == 0 + || len <= col->len + + prtype_get_compression_extra(col->prtype)))); fixed_len = ifield->fixed_len; if (fixed_len && !dict_table_is_comp(index->table) @@ -552,7 +557,9 @@ row_merge_buf_add( } else if (dfield_is_ext(field)) { extra_size += 2; } else if (len < 128 - || (col->len < 256 && col->mtype != DATA_BLOB)) { + || (col->len < 256 - + prtype_get_compression_extra(col->prtype) + && col->mtype != DATA_BLOB)) { extra_size++; } else { /* For variable-length columns, we look up the @@ -3780,6 +3787,13 @@ row_merge_build_indexes( " exited when creating FTS" " index '%s'", indexes[i]->name); + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 466ff113127ad..d54ac222137b7 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -65,11 +65,54 @@ Created 9/17/2000 Heikki Tuuri #include "m_string.h" #include "my_sys.h" #include "ha_prototypes.h" +#include "zlib.h" #include /** Provide optional 4.x backwards compatibility for 5.0 and above */ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; +/** +Z_NO_COMPRESSION = 0 +Z_BEST_SPEED = 1 +Z_BEST_COMPRESSION = 9 +Z_DEFAULT_COMPRESSION = -1 +Compression level to be used by zlib for compressed-blob columns. +Settable by user. +*/ +UNIV_INTERN uint srv_compressed_columns_zip_level = DEFAULT_COMPRESSION_LEVEL; +/** +(Z_FILTERED | Z_HUFFMAN_ONLY | Z_RLE | Z_FIXED | Z_DEFAULT_STRATEGY) + +The strategy parameter is used to tune the compression algorithm. Use the +value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a +filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only +(no string match), or Z_RLE to limit match distances to one +(run-length encoding). Filtered data consists mostly of small values with a +somewhat random distribution. In this case, the compression algorithm is +tuned to compress them better. +The effect of Z_FILTERED is to force more Huffman coding and less string +matching; it is somewhat intermediate between Z_DEFAULT_STRATEGY and +Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as Z_HUFFMAN_ONLY, +but give better compression for PNG image data. The strategy parameter only +affects the compression ratio but not the correctness of the compressed +output even if it is not set appropriately. Z_FIXED prevents the use of +dynamic Huffman codes, allowing for a simpler decoder for special +applications. +*/ +const uint srv_compressed_columns_zlib_strategy = Z_DEFAULT_STRATEGY; +/** Compress the column if the data length exceeds this value. */ +UNIV_INTERN ulong srv_compressed_columns_threshold = 96; +/** +Determine if zlib needs to compute adler32 value for the compressed data. +This variables is similar to page_zip_zlib_wrap, but only used by +compressed blob columns. +*/ +const bool srv_compressed_columns_zlib_wrap = true; +/** +Determine if zlib will use custom memory allocation functions based on +InnoDB memory heap routines (mem_heap_t*). +*/ +const bool srv_compressed_columns_zlib_use_heap = false; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ char* table_name; /*!< table name */ @@ -173,6 +216,17 @@ row_mysql_prebuilt_free_blob_heap( prebuilt->blob_heap = NULL; } +/** Frees the compress heap in prebuilt when no longer needed. */ +UNIV_INTERN +void +row_mysql_prebuilt_free_compress_heap( + row_prebuilt_t* prebuilt) /*!< in: prebuilt struct of a + ha_innobase:: table handle */ +{ + mem_heap_free(prebuilt->compress_heap); + prebuilt->compress_heap = NULL; +} + /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -229,6 +283,425 @@ row_mysql_read_true_varchar( return(field + 1); } +/** + Compressed BLOB header format: + --------------------------------------------------------------- + | reserved | wrap | algorithm | len-len | compressed | unused | + | [1] | [1] | [5] | [3] | [1] | [5] | + --------------------------------------------------------------- + | 0 0 | 1 1 | 2 6 | 7 9 | 10 10 | 11 15 | + --------------------------------------------------------------- + * 'reserved' bit is planned to be used in future versions of the BLOB + header. In this version it must always be + 'default_zip_column_reserved_value' (0). + * 'wrap' identifies if compression algorithm calculated a checksum + (adler32 in case of zlib) and appended it to the compressed data. + * 'algorithm' identifies which algoritm was used to compress this BLOB. + Currently, the only value 'default_zip_column_algorithm_value' (0) is + supported. + * 'len-len' field identifies the length of the column length data portion + followed by this header (see below). + * If 'compressed' bit is set to 1, then this header is immediately followed + by 1..8 bytes (depending on the value of 'len-len' bitfield) which + determine original (uncompressed) block size. These 'len-len' bytes are + followed by compressed representation of the original data. + * If 'compressed' bit is set to 0, every other bitfield ('wrap', + 'algorithm' and 'le-len') must be ignored. In this case the header is + immediately followed by uncompressed (original) data. +*/ + +/** + Currently the only supported value for the 'reserved' field is + false (0). +*/ +static const bool default_zip_column_reserved_value = false; + +/** + Currently the only supported value for the 'algorithm' field is 0, which + means 'zlib'. +*/ +static const uint default_zip_column_algorithm_value = 0; + +static const size_t zip_column_prefix_max_length = + ZIP_COLUMN_HEADER_LENGTH + 8; +static const size_t zip_column_header_length = ZIP_COLUMN_HEADER_LENGTH; + +/* 'reserved', bit 0 */ +static const uint zip_column_reserved = 0; +/* 0000 0000 0000 0001 */ +static const uint zip_column_reserved_mask = 0x0001; + +/* 'wrap', bit 1 */ +static const uint zip_column_wrap = 1; +/* 0000 0000 0000 0010 */ +static const uint zip_column_wrap_mask = 0x0002; + +/* 'algorithm', bit 2,3,4,5,6 */ +static const uint zip_column_algorithm = 2; +/* 0000 0000 0111 1100 */ +static const uint zip_column_algorithm_mask = 0x007C; + +/* 'len-len', bit 7,8,9 */ +static const uint zip_column_data_length = 7; +/* 0000 0011 1000 0000 */ +static const uint zip_column_data_length_mask = 0x0380; + +/* 'compressed', bit 10 */ +static const uint zip_column_compressed = 10; +/* 0000 0100 0000 0000 */ +static const uint zip_column_compressed_mask = 0x0400; + +/** Updates compressed block header with the given components */ +static void +column_set_compress_header( + byte* data, + bool compressed, + ulint lenlen, + uint alg, + bool wrap, + bool reserved) +{ + ulint header = 0; + header |= (compressed << zip_column_compressed); + header |= (lenlen << zip_column_data_length); + header |= (alg << zip_column_algorithm); + header |= (wrap << zip_column_wrap); + header |= (reserved << zip_column_reserved); + mach_write_to_2(data, header); +} + +/** Parse compressed block header into components */ +static void +column_get_compress_header( + const byte* data, + bool* compressed, + ulint* lenlen, + uint* alg, + bool* wrap, + bool* reserved +) +{ + ulint header = mach_read_from_2(data); + *compressed = ((header & zip_column_compressed_mask) >> + zip_column_compressed); + *lenlen = ((header & zip_column_data_length_mask) >> + zip_column_data_length); + *alg = ((header & zip_column_algorithm_mask) >> + zip_column_algorithm); + *wrap = ((header & zip_column_wrap_mask) >> + zip_column_wrap); + *reserved = ((header & zip_column_reserved_mask) >> + zip_column_reserved); +} + +/** Allocate memory for zlib. */ +static +void* +column_zip_zalloc( + void* opaque, /*!< in/out: memory heap */ + uInt items, /*!< in: number of items to allocate */ + uInt size) /*!< in: size of an item in bytes */ +{ + return(mem_heap_zalloc(static_cast(opaque), + items * size)); +} + +/** Deallocate memory for zlib. */ +static +void +column_zip_free( + void* opaque MY_ATTRIBUTE((unused)), /*!< in: memory heap */ + void* address MY_ATTRIBUTE((unused))) /*!< in: object to free */ +{ +} + +/** Configure the zlib allocator to use the given memory heap. */ +UNIV_INTERN +void +column_zip_set_alloc( + void* stream, /*!< in/out: zlib stream */ + mem_heap_t* heap) /*!< in: memory heap to use */ +{ + z_stream* strm = static_cast(stream); + + if (srv_compressed_columns_zlib_use_heap) { + strm->zalloc = column_zip_zalloc; + strm->zfree = column_zip_free; + strm->opaque = heap; + } else { + strm->zalloc = (alloc_func)0; + strm->zfree = (free_func)0; + strm->opaque = (voidpf)0; + } +} + +/** Compress blob/text/varchar column using zlib +@return pointer to the compressed data */ +byte* +row_compress_column( + const byte* data, /*!< in: data in mysql(uncompressed) + format */ + ulint *len, /*!< in: data length; out: length of + compressed data*/ + ulint lenlen, /*!< in: bytes used to store the length of + data */ + const byte* dict_data, + /*!< in: optional dictionary data used for + compression */ + ulint dict_data_len, + /*!< in: optional dictionary data length */ + row_prebuilt_t* prebuilt) + /*!< in: use prebuilt->compress_heap only + here*/ +{ + int err = 0; + ulint comp_len = *len; + ulint buf_len = *len + zip_column_prefix_max_length; + byte* buf; + byte* ptr; + z_stream c_stream; + bool wrap = srv_compressed_columns_zlib_wrap; + + int window_bits = wrap ? MAX_WBITS : -MAX_WBITS; + + if (!prebuilt->compress_heap) { + prebuilt->compress_heap = + mem_heap_create(max(UNIV_PAGE_SIZE, buf_len)); + } + + buf = static_cast(mem_heap_zalloc( + prebuilt->compress_heap,buf_len)); + + if (*len < srv_compressed_columns_threshold || + srv_compressed_columns_zip_level == Z_NO_COMPRESSION) + goto do_not_compress; + + ptr = buf + zip_column_header_length + lenlen; + + /*init deflate object*/ + c_stream.next_in = const_cast(data); + c_stream.avail_in = *len; + c_stream.next_out = ptr; + c_stream.avail_out = comp_len; + + column_zip_set_alloc(&c_stream, prebuilt->compress_heap); + + err = deflateInit2(&c_stream, srv_compressed_columns_zip_level, + Z_DEFLATED, window_bits, MAX_MEM_LEVEL, + srv_compressed_columns_zlib_strategy); + ut_a(err == Z_OK); + + if (dict_data != 0 && dict_data_len != 0) { + err = deflateSetDictionary(&c_stream, dict_data, + dict_data_len); + ut_a(err == Z_OK); + } + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + deflateEnd(&c_stream); + if (err == Z_OK) + err = Z_BUF_ERROR; + } else { + comp_len = c_stream.total_out; + err = deflateEnd(&c_stream); + } + + switch (err) { + case Z_OK: + break; + case Z_BUF_ERROR: + /* data after compress is larger than uncompressed data*/ + break; + default: + ib_logf(IB_LOG_LEVEL_ERROR, + "failed to compress the column, error: %d\n", err); + } + + /* make sure the compressed data size is smaller than + uncompressed data */ + if (err == Z_OK && + *len > (comp_len + zip_column_header_length + lenlen)) { + column_set_compress_header(buf, true, lenlen - 1, + default_zip_column_algorithm_value, wrap, + default_zip_column_reserved_value); + ptr = buf + zip_column_header_length; + /*store the uncompressed data length*/ + switch (lenlen) { + case 1: + mach_write_to_1(ptr, *len); + break; + case 2: + mach_write_to_2(ptr, *len); + break; + case 3: + mach_write_to_3(ptr, *len); + break; + case 4: + mach_write_to_4(ptr, *len); + break; + default: + ut_error; + } + + *len = comp_len + zip_column_header_length + lenlen; + return buf; + } + +do_not_compress: + ptr = buf; + column_set_compress_header(ptr, false, 0, + default_zip_column_algorithm_value, false, + default_zip_column_reserved_value); + ptr += zip_column_header_length; + memcpy(ptr, data, *len); + *len += zip_column_header_length; + return buf; +} + +/** Uncompress blob/text/varchar column using zlib +@return pointer to the uncompressed data */ +const byte* +row_decompress_column( + const byte* data, /*!< in: data in innodb(compressed) format */ + ulint *len, /*!< in: data length; out: length of + decompressed data*/ + const byte* dict_data, + /*!< in: optional dictionary data used for + decompression */ + ulint dict_data_len, + /*!< in: optional dictionary data length */ + row_prebuilt_t* prebuilt) + /*!< in: use prebuilt->compress_heap only + here*/ +{ + ulint buf_len = 0; + byte* buf; + int err = 0; + int window_bits = 0; + z_stream d_stream; + bool is_compressed = false; + bool wrap = false; + bool reserved = false; + ulint lenlen = 0; + uint alg = 0; + + ut_ad(*len != ULINT_UNDEFINED); + ut_ad(*len >= zip_column_header_length); + + column_get_compress_header(data, &is_compressed, &lenlen, &alg, + &wrap, &reserved); + + if (reserved != default_zip_column_reserved_value) { + ib_logf(IB_LOG_LEVEL_FATAL, + "unsupported compressed BLOB header format\n"); + } + + if (alg != default_zip_column_algorithm_value) { + ib_logf(IB_LOG_LEVEL_FATAL, + "unsupported 'algorithm' value in the" + " compressed BLOB header\n"); + } + + ut_a(lenlen < 4); + + data += zip_column_header_length; + if (!is_compressed) { /* column not compressed */ + *len -= zip_column_header_length; + return data; + } + + lenlen++; + + ulint comp_len = *len - zip_column_header_length - lenlen; + + ulint uncomp_len = 0; + switch (lenlen) { + case 1: + uncomp_len = mach_read_from_1(data); + break; + case 2: + uncomp_len = mach_read_from_2(data); + break; + case 3: + uncomp_len = mach_read_from_3(data); + break; + case 4: + uncomp_len = mach_read_from_4(data); + break; + default: + ut_error; + } + + data += lenlen; + + /* data is compressed, decompress it*/ + if (!prebuilt->compress_heap) { + prebuilt->compress_heap = + mem_heap_create(max(UNIV_PAGE_SIZE, uncomp_len)); + } + + buf_len = uncomp_len; + buf = static_cast(mem_heap_zalloc( + prebuilt->compress_heap, buf_len)); + + /* init d_stream */ + d_stream.next_in = const_cast(data); + d_stream.avail_in = comp_len; + d_stream.next_out = buf; + d_stream.avail_out = buf_len; + + column_zip_set_alloc(&d_stream, prebuilt->compress_heap); + + window_bits = wrap ? MAX_WBITS : -MAX_WBITS; + err = inflateInit2(&d_stream, window_bits); + ut_a(err == Z_OK); + + err = inflate(&d_stream, Z_FINISH); + if (err == Z_NEED_DICT) { + ut_a(dict_data != 0 && dict_data_len != 0); + err = inflateSetDictionary(&d_stream, dict_data, + dict_data_len); + ut_a(err == Z_OK); + err = inflate(&d_stream, Z_FINISH); + } + + if (err != Z_STREAM_END) { + inflateEnd(&d_stream); + if (err == Z_BUF_ERROR && d_stream.avail_in == 0) + err = Z_DATA_ERROR; + } else { + buf_len = d_stream.total_out; + err = inflateEnd(&d_stream); + } + + switch (err) { + case Z_OK: + break; + case Z_BUF_ERROR: + ib_logf(IB_LOG_LEVEL_FATAL, + "zlib buf error, this shouldn't happen\n"); + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "failed to decompress column, error: %d\n", err); + } + + if (err == Z_OK) { + if (buf_len != uncomp_len) { + ib_logf(IB_LOG_LEVEL_FATAL, + "failed to decompress blob column, may" + " be corrupted\n"); + } + *len = buf_len; + return buf; + } + + *len -= (zip_column_header_length + lenlen); + return data; +} + + /*******************************************************************//** Stores a reference to a BLOB in the MySQL format. */ UNIV_INTERN @@ -242,10 +715,21 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len) /*!< in: BLOB length; if the value to store + ulint len, /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ + bool need_decompression, + /*!< in: if the data need to be compressed*/ + const byte* dict_data, + /*!< in: optional compression dictionary + data */ + ulint dict_data_len, + /*!< in: optional compression dictionary data + length */ + row_prebuilt_t* prebuilt) + /*compress_heap only + here */ { /* MySQL might assume the field is set to zero except the length and the pointer fields */ @@ -257,13 +741,28 @@ row_mysql_store_blob_ref( In 32-bit architectures we only use the first 4 bytes of the pointer slot. */ - ut_a(col_len - 8 > 1 || len < 256); - ut_a(col_len - 8 > 2 || len < 256 * 256); - ut_a(col_len - 8 > 3 || len < 256 * 256 * 256); + ut_a(col_len - 8 > 1 || + len < 256 + + (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); + ut_a(col_len - 8 > 2 || + len < 256 * 256 + + (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); + ut_a(col_len - 8 > 3 || + len < 256 * 256 * 256 + + (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - mach_write_to_n_little_endian(dest, col_len - 8, len); + const byte *ptr = NULL; - memcpy(dest + col_len - 8, &data, sizeof data); + if (need_decompression) + ptr = row_decompress_column((const byte*)data, &len, + dict_data, dict_data_len, prebuilt); + + if (ptr) + memcpy(dest + col_len - 8, &ptr, sizeof ptr); + else + memcpy(dest + col_len - 8, &data, sizeof data); + + mach_write_to_n_little_endian(dest, col_len - 8, len); } /*******************************************************************//** @@ -276,15 +775,32 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len) /*!< in: BLOB reference length + ulint col_len, /*!< in: BLOB reference length (not BLOB length) */ + bool need_compression, + /*!< in: if the data need to be + compressed*/ + const byte* dict_data, /*!< in: optional compression + dictionary data */ + ulint dict_data_len, /*!< in: optional compression + dictionary data length */ + row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap + only here */ { - byte* data; + byte* data = NULL; + byte* ptr = NULL; *len = mach_read_from_n_little_endian(ref, col_len - 8); memcpy(&data, ref + col_len - 8, sizeof data); + if (need_compression) { + ptr = row_compress_column(data, len, col_len - 8, dict_data, + dict_data_len, prebuilt); + if (ptr) + data = ptr; + } + return(data); } @@ -367,7 +883,16 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp) /*!< in: nonzero=compact format */ + ulint comp, /*!< in: nonzero=compact format */ + bool need_compression, + /*!< in: if the data need to be + compressed*/ + const byte* dict_data, /*!< in: optional compression + dictionary data */ + ulint dict_data_len, /*!< in: optional compression + dictionary data length */ + row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap + only here */ { const byte* ptr = mysql_data; const dtype_t* dtype; @@ -420,8 +945,14 @@ row_mysql_store_col_in_innobase_format( lenlen = 2; } - ptr = row_mysql_read_true_varchar(&col_len, mysql_data, - lenlen); + const byte* tmp_ptr = row_mysql_read_true_varchar( + &col_len, mysql_data, lenlen); + if (need_compression) + ptr = row_compress_column(tmp_ptr, &col_len, + lenlen, dict_data, dict_data_len, + prebuilt); + else + ptr = tmp_ptr; } else { /* Remove trailing spaces from old style VARCHAR columns. */ @@ -503,7 +1034,9 @@ row_mysql_store_col_in_innobase_format( } } else if (type == DATA_BLOB && row_format_col) { - ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len); + ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len, + need_compression, dict_data, dict_data_len, + prebuilt); } dfield_set_data(dfield, ptr, col_len); @@ -561,7 +1094,11 @@ row_mysql_convert_row_to_innobase( TRUE, /* MySQL row format data */ mysql_rec + templ->mysql_col_offset, templ->mysql_col_len, - dict_table_is_comp(prebuilt->table)); + dict_table_is_comp(prebuilt->table), + templ->compressed, + reinterpret_cast( + templ->zip_dict_data.str), + templ->zip_dict_data.length, prebuilt); next_column: ; } @@ -907,6 +1444,10 @@ row_prebuilt_free( mem_heap_free(prebuilt->blob_heap); } + if (prebuilt->compress_heap) { + mem_heap_free(prebuilt->compress_heap); + } + if (prebuilt->old_vers_heap) { mem_heap_free(prebuilt->old_vers_heap); } @@ -1333,6 +1874,9 @@ row_insert_for_mysql( return(DB_READ_ONLY); } + if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) + mem_heap_empty(prebuilt->compress_heap); + trx->op_info = "inserting"; row_mysql_delay_if_needed(); @@ -2693,6 +3237,10 @@ row_drop_tables_for_mysql_in_background(void) return(n_tables + n_tables_dropped); } + DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", + os_thread_sleep(5000000); + ); + table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); @@ -2703,6 +3251,16 @@ row_drop_tables_for_mysql_in_background(void) goto already_dropped; } + if (!table->to_be_dropped) { + /* There is a scenario: the old table is dropped + just after it's added into drop list, and new + table with the same name is created, then we try + to drop the new table in background. */ + dict_table_close(table, FALSE, FALSE); + + goto already_dropped; + } + ut_a(!table->can_be_evicted); dict_table_close(table, FALSE, FALSE); @@ -2833,6 +3391,12 @@ row_mysql_table_id_reassign( pars_info_add_ull_literal(info, "old_id", table->id); pars_info_add_ull_literal(info, "new_id", *new_id); + /* As micro-SQL does not support int4 == int8 comparisons, + old and new IDs are added again under different names as + int4 values*/ + pars_info_add_int4_literal(info, "old_id_narrow", table->id); + pars_info_add_int4_literal(info, "new_id_narrow", *new_id); + err = que_eval_sql( info, "PROCEDURE RENUMBER_TABLE_PROC () IS\n" @@ -2843,6 +3407,8 @@ row_mysql_table_id_reassign( " WHERE TABLE_ID = :old_id;\n" "UPDATE SYS_INDEXES SET TABLE_ID = :new_id\n" " WHERE TABLE_ID = :old_id;\n" + "UPDATE SYS_ZIP_DICT_COLS SET TABLE_ID = :new_id_narrow\n" + " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n", FALSE, trx); return(err); @@ -3609,6 +4175,12 @@ row_truncate_table_for_mysql( pars_info_add_ull_literal(info, "old_id", table->id); pars_info_add_ull_literal(info, "new_id", new_id); + /* As micro-SQL does not support int4 == int8 comparisons, + old and new IDs are added again under different names as + int4 values*/ + pars_info_add_int4_literal(info, "old_id_narrow", table->id); + pars_info_add_int4_literal(info, "new_id_narrow", new_id); + err = que_eval_sql(info, "PROCEDURE RENUMBER_TABLE_ID_PROC () IS\n" "BEGIN\n" @@ -3620,6 +4192,9 @@ row_truncate_table_for_mysql( "UPDATE SYS_INDEXES" " SET TABLE_ID = :new_id, SPACE = :new_space\n" " WHERE TABLE_ID = :old_id;\n" + "UPDATE SYS_ZIP_DICT_COLS\n" + " SET TABLE_ID = :new_id_narrow\n" + " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n" , FALSE, trx); @@ -3962,6 +4537,13 @@ row_drop_table_for_mysql( } } + + DBUG_EXECUTE_IF("row_drop_table_add_to_background", + row_add_table_to_background_drop_list(table->name); + err = DB_SUCCESS; + goto funct_exit; + ); + /* TODO: could we replace the counter n_foreign_key_checks_running with lock checks on the table? Acquire here an exclusive lock on the table, and rewrite lock0lock.cc and the lock wait in srv0srv.cc so that @@ -4232,6 +4814,19 @@ row_drop_table_for_mysql( filepath = fil_make_ibd_name(tablename, false); } + /* Remove all compression dictionary references for the + table */ + err = dict_create_remove_zip_dict_references_for_table( + table->id, trx); + if (err != DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_ERROR, "Error: (%s) not " + "able to remove compression dictionary " + "references for table %s", ut_strerr(err), + tablename); + + goto funct_exit; + } + if (dict_table_has_fts_index(table) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) { ut_ad(table->n_ref_count == 0); @@ -4578,6 +5173,19 @@ row_drop_database_for_mysql( row_mysql_lock_data_dictionary(trx); while ((table_name = dict_get_first_table_name_in_db(name))) { + /* Drop parent table if it is a fts aux table, to + avoid accessing dropped fts aux tables in information + scheam when parent table still exists. + Note: Drop parent table will drop fts aux tables. */ + char* parent_table_name; + parent_table_name = fts_get_parent_table_name( + table_name, strlen(table_name)); + + if (parent_table_name != NULL) { + mem_free(table_name); + table_name = parent_table_name; + } + ut_a(memcmp(table_name, name, namelen) == 0); table = dict_table_open_on_name( diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index 74579687a9b5e..d2821abdc2ed4 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2460,9 +2460,11 @@ row_sel_convert_mysql_key_to_innobase( if (UNIV_LIKELY(!is_null)) { buf = row_mysql_store_col_in_innobase_format( dfield, buf, - FALSE, /* MySQL key value format col */ + /* MySQL key value format col */ + FALSE, key_ptr + data_offset, data_len, - dict_table_is_comp(index->table)); + dict_table_is_comp(index->table), + false, 0, 0 ,0); ut_a(buf <= original_buf + buf_len); } @@ -2555,12 +2557,16 @@ row_sel_store_row_id_to_prebuilt( #ifdef UNIV_DEBUG /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ -# define row_sel_field_store_in_mysql_format(dest,templ,idx,field,src,len) \ - row_sel_field_store_in_mysql_format_func(dest,templ,idx,field,src,len) +# define row_sel_field_store_in_mysql_format( \ + dest,templ,idx,field,src,len,prebuilt) \ + row_sel_field_store_in_mysql_format_func \ + (dest,templ,idx,field,src,len, prebuilt) #else /* UNIV_DEBUG */ /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ -# define row_sel_field_store_in_mysql_format(dest,templ,idx,field,src,len) \ - row_sel_field_store_in_mysql_format_func(dest,templ,src,len) +# define row_sel_field_store_in_mysql_format( \ + dest,templ,idx,field,src,len,prebuilt) \ + row_sel_field_store_in_mysql_format_func \ + (dest,templ,src,len, prebuilt) #endif /* UNIV_DEBUG */ /**************************************************************//** @@ -2590,7 +2596,10 @@ row_sel_field_store_in_mysql_format_func( templ->icp_rec_field_no */ #endif /* UNIV_DEBUG */ const byte* data, /*!< in: data to store */ - ulint len) /*!< in: length of the data */ + ulint len, /*!< in: length of the data */ + row_prebuilt_t* prebuilt) + /*!< in: use prebuilt->compress_heap + only here */ { byte* ptr; #ifdef UNIV_DEBUG @@ -2634,6 +2643,15 @@ row_sel_field_store_in_mysql_format_func( field_end = dest + templ->mysql_col_len; if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) { + /* If this is a compressed column, + decompress it first */ + if (templ->compressed) + data = row_decompress_column(data, &len, + reinterpret_cast( + templ->zip_dict_data.str), + templ->zip_dict_data.length, + prebuilt); + /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2684,7 +2702,11 @@ row_sel_field_store_in_mysql_format_func( already copied to the buffer in row_sel_store_mysql_rec */ row_mysql_store_blob_ref(dest, templ->mysql_col_len, data, - len); + len, templ->compressed, + reinterpret_cast( + templ->zip_dict_data.str), + templ->zip_dict_data.length, + prebuilt); break; case DATA_MYSQL: @@ -2837,7 +2859,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len); + templ, index, field_no, data, len, prebuilt); if (heap != prebuilt->blob_heap) { mem_heap_free(heap); @@ -2887,7 +2909,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len); + templ, index, field_no, data, len, prebuilt); } ut_ad(len != UNIV_SQL_NULL); @@ -2935,6 +2957,9 @@ row_sel_store_mysql_rec( prebuilt->blob_heap = NULL; } + if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) + mem_heap_empty(prebuilt->compress_heap); + for (i = 0; i < prebuilt->n_template; i++) { const mysql_row_templ_t*templ = &prebuilt->mysql_template[i]; const ulint field_no diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 930694ac0af23..8f1d341ad1b2c 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2714,6 +2714,12 @@ innobase_start_or_create_for_mysql(void) return(err); } + /* Create the SYS_ZIP_DICT system table */ + err = dict_create_or_check_sys_zip_dict(); + if (err != DB_SUCCESS) { + return(err); + } + srv_is_being_started = FALSE; ut_a(trx_purge_state() == PURGE_STATE_INIT); From d9787aa29af3e77c5cd04defe0331c721542cff6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 25 Oct 2016 17:03:23 +0200 Subject: [PATCH 136/295] 5.6.33-79.0 --- storage/tokudb/CMakeLists.txt | 2 +- .../tokudb/PerconaFT/buildheader/make_tdb.cc | 3 + .../ft/cachetable/cachetable-internal.h | 2 + .../PerconaFT/ft/cachetable/cachetable.cc | 16 ++ .../PerconaFT/ft/cachetable/cachetable.h | 6 + storage/tokudb/PerconaFT/ft/ft-ops.cc | 151 ++++++++++++++- storage/tokudb/PerconaFT/ft/ft-ops.h | 2 + storage/tokudb/PerconaFT/ft/ft.cc | 14 +- storage/tokudb/PerconaFT/ft/ft.h | 6 + .../tokudb/PerconaFT/ft/logger/logformat.cc | 9 + storage/tokudb/PerconaFT/ft/logger/recover.cc | 78 ++++++++ .../PerconaFT/ft/serialize/rbtree_mhs.h | 16 +- .../test-rbtree-insert-remove-without-mhs.cc | 7 +- storage/tokudb/PerconaFT/ft/txn/roll.cc | 118 +++++++++++- storage/tokudb/PerconaFT/portability/file.cc | 6 + .../tokudb/PerconaFT/portability/memory.cc | 9 + storage/tokudb/PerconaFT/portability/memory.h | 4 +- .../PerconaFT/portability/toku_portability.h | 2 + .../tokudb/PerconaFT/src/tests/CMakeLists.txt | 42 ++-- .../src/tests/recovery_fileops_unit.cc | 155 ++++++++------- storage/tokudb/PerconaFT/src/ydb-internal.h | 3 +- storage/tokudb/PerconaFT/src/ydb.cc | 50 ++++- storage/tokudb/PerconaFT/src/ydb_db.cc | 99 +++++++--- storage/tokudb/PerconaFT/src/ydb_db.h | 16 ++ storage/tokudb/hatoku_hton.cc | 1 + .../r/dir-per-db-with-custom-data-dir.result | 10 + .../mysql-test/tokudb/r/dir_per_db.result | 180 ++++++++++++++++++ .../r/i_s_tokudb_lock_waits_released.result | 12 ++ .../mysql-test/tokudb/r/row_format.result | 51 +++++ ...dir-per-db-with-custom-data-dir-master.opt | 1 + .../t/dir-per-db-with-custom-data-dir.test | 16 ++ .../mysql-test/tokudb/t/dir_per_db.test | 76 ++++++++ .../tokudb/t/dir_per_db_show_table_files.inc | 9 + .../t/i_s_tokudb_lock_waits_released.test | 29 ++- .../mysql-test/tokudb/t/row_format.test | 41 ++++ .../mysql-test/tokudb_bugs/r/db938.result | 1 + .../mysql-test/tokudb_bugs/t/db938.test | 3 + .../t/partition_debug_sync_tokudb.test | 4 +- storage/tokudb/tokudb_sysvars.cc | 14 ++ storage/tokudb/tokudb_sysvars.h | 1 + 40 files changed, 1105 insertions(+), 160 deletions(-) create mode 100644 storage/tokudb/mysql-test/tokudb/r/dir-per-db-with-custom-data-dir.result create mode 100644 storage/tokudb/mysql-test/tokudb/r/dir_per_db.result create mode 100644 storage/tokudb/mysql-test/tokudb/r/row_format.result create mode 100644 storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir-master.opt create mode 100644 storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir.test create mode 100644 storage/tokudb/mysql-test/tokudb/t/dir_per_db.test create mode 100644 storage/tokudb/mysql-test/tokudb/t/dir_per_db_show_table_files.inc create mode 100644 storage/tokudb/mysql-test/tokudb/t/row_format.test diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index fbb02582f4da6..ad30e6d40eb4f 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(TOKUDB_VERSION 5.6.32-78.1) +SET(TOKUDB_VERSION 5.6.33-79.0) # PerconaFT only supports x86-64 and cmake-2.8.9+ IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT CMAKE_VERSION VERSION_LESS "2.8.9") diff --git a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc index 576f902f6aee8..7ede78b3c0db3 100644 --- a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc +++ b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc @@ -422,6 +422,9 @@ static void print_db_env_struct (void) { "int (*set_checkpoint_pool_threads)(DB_ENV *, uint32_t)", "void (*set_check_thp)(DB_ENV *, bool new_val)", "bool (*get_check_thp)(DB_ENV *)", + "bool (*set_dir_per_db)(DB_ENV *, bool new_val)", + "bool (*get_dir_per_db)(DB_ENV *)", + "const char *(*get_data_dir)(DB_ENV *env)", NULL}; sort_and_dump_fields("db_env", true, extra); diff --git a/storage/tokudb/PerconaFT/ft/cachetable/cachetable-internal.h b/storage/tokudb/PerconaFT/ft/cachetable/cachetable-internal.h index dc6aec9226d8f..05fb771de0871 100644 --- a/storage/tokudb/PerconaFT/ft/cachetable/cachetable-internal.h +++ b/storage/tokudb/PerconaFT/ft/cachetable/cachetable-internal.h @@ -138,6 +138,8 @@ struct cachefile { // nor attempt to open any cachefile with the same fname (dname) // until this cachefile has been fully closed and unlinked. bool unlink_on_close; + // If set then fclose will not be logged in recovery log. + bool skip_log_recover_on_close; int fd; /* Bug: If a file is opened read-only, then it is stuck in read-only. If it is opened read-write, then subsequent writers can write to it too. */ CACHETABLE cachetable; struct fileid fileid; diff --git a/storage/tokudb/PerconaFT/ft/cachetable/cachetable.cc b/storage/tokudb/PerconaFT/ft/cachetable/cachetable.cc index 5bba977de1a33..6d753805fa9fa 100644 --- a/storage/tokudb/PerconaFT/ft/cachetable/cachetable.cc +++ b/storage/tokudb/PerconaFT/ft/cachetable/cachetable.cc @@ -467,6 +467,10 @@ toku_cachefile_fname_in_env (CACHEFILE cf) { return cf->fname_in_env; } +void toku_cachefile_set_fname_in_env(CACHEFILE cf, char *new_fname_in_env) { + cf->fname_in_env = new_fname_in_env; +} + int toku_cachefile_get_fd (CACHEFILE cf) { return cf->fd; @@ -2903,6 +2907,18 @@ bool toku_cachefile_is_unlink_on_close(CACHEFILE cf) { return cf->unlink_on_close; } +void toku_cachefile_skip_log_recover_on_close(CACHEFILE cf) { + cf->skip_log_recover_on_close = true; +} + +void toku_cachefile_do_log_recover_on_close(CACHEFILE cf) { + cf->skip_log_recover_on_close = false; +} + +bool toku_cachefile_is_skip_log_recover_on_close(CACHEFILE cf) { + return cf->skip_log_recover_on_close; +} + uint64_t toku_cachefile_size(CACHEFILE cf) { int64_t file_size; int fd = toku_cachefile_get_fd(cf); diff --git a/storage/tokudb/PerconaFT/ft/cachetable/cachetable.h b/storage/tokudb/PerconaFT/ft/cachetable/cachetable.h index 148326562ab81..3b3cb0a2d4691 100644 --- a/storage/tokudb/PerconaFT/ft/cachetable/cachetable.h +++ b/storage/tokudb/PerconaFT/ft/cachetable/cachetable.h @@ -500,12 +500,18 @@ int toku_cachefile_get_fd (CACHEFILE); // Return the filename char * toku_cachefile_fname_in_env (CACHEFILE cf); +void toku_cachefile_set_fname_in_env(CACHEFILE cf, char *new_fname_in_env); + // Make it so when the cachefile closes, the underlying file is unlinked void toku_cachefile_unlink_on_close(CACHEFILE cf); // is this cachefile marked as unlink on close? bool toku_cachefile_is_unlink_on_close(CACHEFILE cf); +void toku_cachefile_skip_log_recover_on_close(CACHEFILE cf); +void toku_cachefile_do_log_recover_on_close(CACHEFILE cf); +bool toku_cachefile_is_skip_log_recover_on_close(CACHEFILE cf); + // Return the logger associated with the cachefile struct tokulogger *toku_cachefile_logger(CACHEFILE cf); diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index f131668889e9e..30a8710d7aa1e 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -149,22 +149,23 @@ basement nodes, bulk fetch, and partial fetch: #include "ft/cachetable/checkpoint.h" #include "ft/cursor.h" -#include "ft/ft.h" #include "ft/ft-cachetable-wrappers.h" #include "ft/ft-flusher.h" #include "ft/ft-internal.h" -#include "ft/msg.h" +#include "ft/ft.h" #include "ft/leafentry.h" #include "ft/logger/log-internal.h" +#include "ft/msg.h" #include "ft/node.h" #include "ft/serialize/block_table.h" -#include "ft/serialize/sub_block.h" #include "ft/serialize/ft-serialize.h" #include "ft/serialize/ft_layout_version.h" #include "ft/serialize/ft_node-serialize.h" +#include "ft/serialize/sub_block.h" #include "ft/txn/txn_manager.h" -#include "ft/ule.h" #include "ft/txn/xids.h" +#include "ft/ule.h" +#include "src/ydb-internal.h" #include @@ -179,6 +180,7 @@ basement nodes, bulk fetch, and partial fetch: #include +#include /* Status is intended for display to humans to help understand system behavior. * It does not need to be perfectly thread-safe. */ @@ -2593,12 +2595,104 @@ static inline int ft_open_maybe_direct(const char *filename, int oflag, int mode static const mode_t file_mode = S_IRUSR+S_IWUSR+S_IRGRP+S_IWGRP+S_IROTH+S_IWOTH; +inline bool toku_file_is_root(const char *path, const char *last_slash) { + return last_slash == path; +} + +static std::unique_ptr toku_file_get_parent_dir( + const char *path) { + std::unique_ptr result(nullptr, &toku_free); + + bool has_trailing_slash = false; + + /* Find the offset of the last slash */ + const char *last_slash = strrchr(path, OS_PATH_SEPARATOR); + + if (!last_slash) { + /* No slash in the path, return NULL */ + return result; + } + + /* Ok, there is a slash. Is there anything after it? */ + if (static_cast(last_slash - path + 1) == strlen(path)) { + has_trailing_slash = true; + } + + /* Reduce repetative slashes. */ + while (last_slash > path && last_slash[-1] == OS_PATH_SEPARATOR) { + last_slash--; + } + + /* Check for the root of a drive. */ + if (toku_file_is_root(path, last_slash)) { + return result; + } + + /* If a trailing slash prevented the first strrchr() from trimming + the last component of the path, trim that component now. */ + if (has_trailing_slash) { + /* Back up to the previous slash. */ + last_slash--; + while (last_slash > path && last_slash[0] != OS_PATH_SEPARATOR) { + last_slash--; + } + + /* Reduce repetative slashes. */ + while (last_slash > path && last_slash[-1] == OS_PATH_SEPARATOR) { + last_slash--; + } + } + + /* Check for the root of a drive. */ + if (toku_file_is_root(path, last_slash)) { + return result; + } + + result.reset(toku_strndup(path, last_slash - path)); + return result; +} + +static bool toku_create_subdirs_if_needed(const char *path) { + static const mode_t dir_mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | + S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH; + + toku_struct_stat stat; + bool subdir_exists = true; + auto subdir = toku_file_get_parent_dir(path); + + if (!subdir.get()) + return true; + + if (toku_stat(subdir.get(), &stat) == -1) { + if (ENOENT == get_error_errno()) + subdir_exists = false; + else + return false; + } + + if (subdir_exists) { + if (!S_ISDIR(stat.st_mode)) + return false; + return true; + } + + if (!toku_create_subdirs_if_needed(subdir.get())) + return false; + + if (toku_os_mkdir(subdir.get(), dir_mode)) + return false; + + return true; +} + // open a file for use by the ft // Requires: File does not exist. static int ft_create_file(FT_HANDLE UU(ft_handle), const char *fname, int *fdp) { int r; int fd; int er; + if (!toku_create_subdirs_if_needed(fname)) + return get_error_errno(); fd = ft_open_maybe_direct(fname, O_RDWR | O_BINARY, file_mode); assert(fd==-1); if ((er = get_maybe_error_errno()) != ENOENT) { @@ -4427,6 +4521,55 @@ void toku_ft_unlink(FT_HANDLE handle) { toku_cachefile_unlink_on_close(cf); } +int toku_ft_rename_iname(DB_TXN *txn, + const char *data_dir, + const char *old_iname, + const char *new_iname, + CACHETABLE ct) { + int r = 0; + + std::unique_ptr new_iname_full(nullptr, + &toku_free); + std::unique_ptr old_iname_full(nullptr, + &toku_free); + + new_iname_full.reset(toku_construct_full_name(2, data_dir, new_iname)); + old_iname_full.reset(toku_construct_full_name(2, data_dir, old_iname)); + + if (txn) { + BYTESTRING bs_old_name = {static_cast(strlen(old_iname) + 1), + const_cast(old_iname)}; + BYTESTRING bs_new_name = {static_cast(strlen(new_iname) + 1), + const_cast(new_iname)}; + FILENUM filenum = FILENUM_NONE; + { + CACHEFILE cf; + r = toku_cachefile_of_iname_in_env(ct, old_iname, &cf); + if (r != ENOENT) { + char *old_fname_in_cf = toku_cachefile_fname_in_env(cf); + toku_cachefile_set_fname_in_env(cf, toku_xstrdup(new_iname)); + toku_free(old_fname_in_cf); + filenum = toku_cachefile_filenum(cf); + } + } + toku_logger_save_rollback_frename( + db_txn_struct_i(txn)->tokutxn, &bs_old_name, &bs_new_name); + toku_log_frename(db_txn_struct_i(txn)->tokutxn->logger, + (LSN *)0, + 0, + toku_txn_get_txnid(db_txn_struct_i(txn)->tokutxn), + bs_old_name, + filenum, + bs_new_name); + } + + r = toku_os_rename(old_iname_full.get(), new_iname_full.get()); + if (r != 0) + return r; + r = toku_fsync_directory(new_iname_full.get()); + return r; +} + int toku_ft_get_fragmentation(FT_HANDLE ft_handle, TOKU_DB_FRAGMENTATION report) { int fd = toku_cachefile_get_fd(ft_handle->ft->cf); toku_ft_lock(ft_handle->ft); diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.h b/storage/tokudb/PerconaFT/ft/ft-ops.h index 313a74628ea17..70cf045d43c1c 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.h +++ b/storage/tokudb/PerconaFT/ft/ft-ops.h @@ -48,6 +48,8 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "ft/msg.h" #include "util/dbt.h" +#define OS_PATH_SEPARATOR '/' + typedef struct ft_handle *FT_HANDLE; int toku_open_ft_handle (const char *fname, int is_create, FT_HANDLE *, int nodesize, int basementnodesize, enum toku_compression_method compression_method, CACHETABLE, TOKUTXN, int(*)(DB *,const DBT*,const DBT*)) __attribute__ ((warn_unused_result)); diff --git a/storage/tokudb/PerconaFT/ft/ft.cc b/storage/tokudb/PerconaFT/ft/ft.cc index 699fcc5760375..7c94b4c59d345 100644 --- a/storage/tokudb/PerconaFT/ft/ft.cc +++ b/storage/tokudb/PerconaFT/ft/ft.cc @@ -253,7 +253,19 @@ static void ft_close(CACHEFILE cachefile, int fd, void *header_v, bool oplsn_val char* fname_in_env = toku_cachefile_fname_in_env(cachefile); assert(fname_in_env); BYTESTRING bs = {.len=(uint32_t) strlen(fname_in_env), .data=fname_in_env}; - toku_log_fclose(logger, &lsn, ft->h->dirty, bs, toku_cachefile_filenum(cachefile)); // flush the log on close (if new header is being written), otherwise it might not make it out. + if (!toku_cachefile_is_skip_log_recover_on_close(cachefile)) { + toku_log_fclose( + logger, + &lsn, + ft->h->dirty, + bs, + toku_cachefile_filenum(cachefile)); // flush the log on + // close (if new header + // is being written), + // otherwise it might + // not make it out. + toku_cachefile_do_log_recover_on_close(cachefile); + } } } if (ft->h->dirty) { // this is the only place this bit is tested (in currentheader) diff --git a/storage/tokudb/PerconaFT/ft/ft.h b/storage/tokudb/PerconaFT/ft/ft.h index d600e093bdcf0..7a3c4fa783cb9 100644 --- a/storage/tokudb/PerconaFT/ft/ft.h +++ b/storage/tokudb/PerconaFT/ft/ft.h @@ -53,6 +53,12 @@ typedef struct ft_options *FT_OPTIONS; void toku_ft_unlink(FT_HANDLE handle); void toku_ft_unlink_on_commit(FT_HANDLE handle, TOKUTXN txn); +int toku_ft_rename_iname(DB_TXN *txn, + const char *data_dir, + const char *old_iname, + const char *new_iname, + CACHETABLE ct); + void toku_ft_init_reflock(FT ft); void toku_ft_destroy_reflock(FT ft); void toku_ft_grab_reflock(FT ft); diff --git a/storage/tokudb/PerconaFT/ft/logger/logformat.cc b/storage/tokudb/PerconaFT/ft/logger/logformat.cc index 6f3baa81c86b2..49b611388038d 100644 --- a/storage/tokudb/PerconaFT/ft/logger/logformat.cc +++ b/storage/tokudb/PerconaFT/ft/logger/logformat.cc @@ -90,6 +90,10 @@ const struct logtype rollbacks[] = { {"fcreate", 'F', FA{{"FILENUM", "filenum", 0}, {"BYTESTRING", "iname", 0}, NULLFIELD}, LOG_BEGIN_ACTION_NA}, + //rename file + {"frename", 'n', FA{{"BYTESTRING", "old_iname", 0}, + {"BYTESTRING", "new_iname", 0}, + NULLFIELD}, LOG_BEGIN_ACTION_NA}, // cmdinsert is used to insert a key-value pair into a DB. For rollback we don't need the data. {"cmdinsert", 'i', FA{ {"FILENUM", "filenum", 0}, @@ -195,6 +199,11 @@ const struct logtype logtypes[] = { {"fdelete", 'U', FA{{"TXNID_PAIR", "xid", 0}, {"FILENUM", "filenum", 0}, NULLFIELD}, SHOULD_LOG_BEGIN}, + {"frename", 'n', FA{{"TXNID_PAIR", "xid", 0}, + {"BYTESTRING", "old_iname", 0}, + {"FILENUM", "old_filenum", 0}, + {"BYTESTRING", "new_iname", 0}, + NULLFIELD}, IGNORE_LOG_BEGIN}, {"enq_insert", 'I', FA{{"FILENUM", "filenum", 0}, {"TXNID_PAIR", "xid", 0}, {"BYTESTRING", "key", 0}, diff --git a/storage/tokudb/PerconaFT/ft/logger/recover.cc b/storage/tokudb/PerconaFT/ft/logger/recover.cc index 38f29773bd630..a9c30c0e37a91 100644 --- a/storage/tokudb/PerconaFT/ft/logger/recover.cc +++ b/storage/tokudb/PerconaFT/ft/logger/recover.cc @@ -36,6 +36,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." +#include #include "ft/cachetable/cachetable.h" #include "ft/cachetable/checkpoint.h" #include "ft/ft.h" @@ -935,6 +936,83 @@ static int toku_recover_backward_fdelete (struct logtype_fdelete *UU(l), RECOVER return 0; } +static int toku_recover_frename(struct logtype_frename *l, RECOVER_ENV renv) { + assert(renv); + assert(renv->env); + + toku_struct_stat stat; + const char *data_dir = renv->env->get_data_dir(renv->env); + bool old_exist = true; + bool new_exist = true; + + assert(data_dir); + + struct file_map_tuple *tuple; + + std::unique_ptr old_iname_full( + toku_construct_full_name(2, data_dir, l->old_iname.data), &toku_free); + std::unique_ptr new_iname_full( + toku_construct_full_name(2, data_dir, l->new_iname.data), &toku_free); + + if (toku_stat(old_iname_full.get(), &stat) == -1) { + if (ENOENT == errno) + old_exist = false; + else + return 1; + } + + if (toku_stat(new_iname_full.get(), &stat) == -1) { + if (ENOENT == errno) + new_exist = false; + else + return 1; + } + + // Both old and new files can exist if: + // - rename() is not completed + // - fcreate was replayed during recovery + // 'Stalled cachefiles' container cachefile_list::m_stale_fileid contains + // closed but not yet evicted cachefiles and the key of this container is + // fs-dependent file id - (device id, inode number) pair. As it is supposed + // new file have not yet created during recovery process the 'stalled + // cachefile' container can contain only cache file of old file. + // To preserve the old cachefile file's id and keep it in + // 'stalled cachefiles' container the new file is removed + // and the old file is renamed. + if (old_exist && new_exist && + (toku_os_unlink(new_iname_full.get()) == -1 || + toku_os_rename(old_iname_full.get(), new_iname_full.get()) == -1 || + toku_fsync_directory(old_iname_full.get()) == -1 || + toku_fsync_directory(new_iname_full.get()) == -1)) + return 1; + + if (old_exist && !new_exist && + (toku_os_rename(old_iname_full.get(), new_iname_full.get()) == -1 || + toku_fsync_directory(old_iname_full.get()) == -1 || + toku_fsync_directory(new_iname_full.get()) == -1)) + return 1; + + if (file_map_find(&renv->fmap, l->old_filenum, &tuple) != DB_NOTFOUND) { + if (tuple->iname) + toku_free(tuple->iname); + tuple->iname = toku_xstrdup(l->new_iname.data); + } + + TOKUTXN txn = NULL; + toku_txnid2txn(renv->logger, l->xid, &txn); + + if (txn) + toku_logger_save_rollback_frename(txn, &l->old_iname, &l->new_iname); + + return 0; +} + +static int toku_recover_backward_frename(struct logtype_frename *UU(l), + RECOVER_ENV UU(renv)) { + // nothing + return 0; +} + static int toku_recover_enq_insert (struct logtype_enq_insert *l, RECOVER_ENV renv) { int r; TOKUTXN txn = NULL; diff --git a/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h b/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h index 92f1e278e1a01..eb8c953b08c13 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h +++ b/storage/tokudb/PerconaFT/ft/serialize/rbtree_mhs.h @@ -106,6 +106,7 @@ namespace MhsRbTree { static const uint64_t MHS_MAX_VAL = 0xffffffffffffffff; OUUInt64() : _value(0) {} OUUInt64(uint64_t s) : _value(s) {} + OUUInt64(const OUUInt64& o) : _value(o._value) {} bool operator<(const OUUInt64 &r) const { invariant(!(_value == MHS_MAX_VAL && r.ToInt() == MHS_MAX_VAL)); return _value < r.ToInt(); @@ -182,15 +183,18 @@ namespace MhsRbTree { class Node { public: - struct BlockPair { + class BlockPair { + public: OUUInt64 _offset; OUUInt64 _size; BlockPair() : _offset(0), _size(0) {} BlockPair(uint64_t o, uint64_t s) : _offset(o), _size(s) {} - BlockPair(OUUInt64 o, OUUInt64 s) : _offset(o), _size(s) {} - int operator<(const struct BlockPair &rhs) const { + BlockPair(const BlockPair &o) + : _offset(o._offset), _size(o._size) {} + + int operator<(const BlockPair &rhs) const { return _offset < rhs._offset; } int operator<(const uint64_t &o) const { return _offset < o; } @@ -203,15 +207,15 @@ namespace MhsRbTree { }; EColor _color; - struct BlockPair _hole; - struct Pair _label; + BlockPair _hole; + Pair _label; Node *_left; Node *_right; Node *_parent; Node(EColor c, Node::BlockPair h, - struct Pair lb, + Pair lb, Node *l, Node *r, Node *p) diff --git a/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc b/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc index 85f29ce9813ef..cefe66335a6cb 100644 --- a/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc +++ b/storage/tokudb/PerconaFT/ft/tests/test-rbtree-insert-remove-without-mhs.cc @@ -53,9 +53,10 @@ static void generate_random_input() { std::srand(unsigned(std::time(0))); // set some values: - for (uint64_t i = 1; i < N; ++i) { - input_vector.push_back({i, 0}); - old_vector[i] = {i, 0}; + for (uint64_t i = 0; i < N; ++i) { + MhsRbTree::Node::BlockPair bp = {i+1, 0}; + input_vector.push_back(bp); + old_vector[i] = bp; } // using built-in random generator: std::random_shuffle(input_vector.begin(), input_vector.end(), myrandom); diff --git a/storage/tokudb/PerconaFT/ft/txn/roll.cc b/storage/tokudb/PerconaFT/ft/txn/roll.cc index 90eee1e580a80..9f3977743a049 100644 --- a/storage/tokudb/PerconaFT/ft/txn/roll.cc +++ b/storage/tokudb/PerconaFT/ft/txn/roll.cc @@ -38,13 +38,13 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. /* rollback and rollforward routines. */ - -#include "ft/ft.h" +#include #include "ft/ft-ops.h" +#include "ft/ft.h" #include "ft/log_header.h" #include "ft/logger/log-internal.h" -#include "ft/txn/xids.h" #include "ft/txn/rollback-apply.h" +#include "ft/txn/xids.h" // functionality provided by roll.c is exposed by an autogenerated // header file, logheader.h @@ -162,10 +162,122 @@ toku_rollback_fcreate (FILENUM filenum, // directory row lock for its dname) and we would not get this // far if there were other live handles. toku_cachefile_unlink_on_close(cf); + toku_cachefile_skip_log_recover_on_close(cf); done: return 0; } +int toku_commit_frename(BYTESTRING /* old_name */, + BYTESTRING /* new_iname */, + TOKUTXN /* txn */, + LSN UU(oplsn)) { + return 0; +} + +int toku_rollback_frename(BYTESTRING old_iname, + BYTESTRING new_iname, + TOKUTXN txn, + LSN UU(oplsn)) { + assert(txn); + assert(txn->logger); + assert(txn->logger->ct); + + CACHETABLE cachetable = txn->logger->ct; + + toku_struct_stat stat; + bool old_exist = true; + bool new_exist = true; + + std::unique_ptr old_iname_full( + toku_cachetable_get_fname_in_cwd(cachetable, old_iname.data), + &toku_free); + std::unique_ptr new_iname_full( + toku_cachetable_get_fname_in_cwd(cachetable, new_iname.data), + &toku_free); + + if (toku_stat(old_iname_full.get(), &stat) == -1) { + if (ENOENT == errno) + old_exist = false; + else + return 1; + } + + if (toku_stat(new_iname_full.get(), &stat) == -1) { + if (ENOENT == errno) + new_exist = false; + else + return 1; + } + + // Both old and new files can exist if: + // - rename() is not completed + // - fcreate was replayed during recovery + // 'Stalled cachefiles' container cachefile_list::m_stale_fileid contains + // closed but not yet evicted cachefiles and the key of this container is + // fs-dependent file id - (device id, inode number) pair. To preserve the + // new cachefile + // file's id and keep it in 'stalled cachefiles' container the old file is + // removed + // and the new file is renamed. + if (old_exist && new_exist && + (toku_os_unlink(old_iname_full.get()) == -1 || + toku_os_rename(new_iname_full.get(), old_iname_full.get()) == -1 || + toku_fsync_directory(new_iname_full.get()) == -1 || + toku_fsync_directory(old_iname_full.get()) == -1)) + return 1; + + if (!old_exist && new_exist && + (toku_os_rename(new_iname_full.get(), old_iname_full.get()) == -1 || + toku_fsync_directory(new_iname_full.get()) == -1 || + toku_fsync_directory(old_iname_full.get()) == -1)) + return 1; + + // it's ok if both files do not exist on recovery + if (!old_exist && !new_exist) + assert(txn->for_recovery); + + CACHEFILE cf; + int r = toku_cachefile_of_iname_in_env(cachetable, new_iname.data, &cf); + if (r != ENOENT) { + char *old_fname_in_cf = toku_cachefile_fname_in_env(cf); + toku_cachefile_set_fname_in_env(cf, toku_xstrdup(old_iname.data)); + toku_free(old_fname_in_cf); + // There is at least one case when fclose logging cause error: + // 1) start transaction + // 2) create ft 'a'(write "fcreate" in recovery log) + // 3) rename ft 'a' to 'b'(write "frename" in recovery log) + // 4) abort transaction: + // a) rollback rename ft (renames 'b' to 'a') + // b) rollback create ft (removes 'a'): + // invokes toku_cachefile_unlink_on_close - lazy unlink on file + // close, + // it just sets corresponding flag in cachefile object + // c) write "unlink" for 'a' in recovery log + // (when transaction is aborted all locks are released, + // when file lock is released the file is closed and unlinked if + // corresponding flag is set in cachefile object) + // 5) crash + // + // After this we have the following records in recovery log: + // - create ft 'a', + // - rename 'a' to 'b', + // - unlink 'a' + // + // On recovery: + // - create 'a' + // - rename 'a' to 'b' + // - unlink 'a' - as 'a' file does not exist we have crash on assert + // here + // + // There is no need to write "unlink" in recovery log in (4a) because + // 'a' will be removed + // on transaction rollback on recovery. + toku_cachefile_skip_log_recover_on_close(cf); + } + + return 0; +} + int find_ft_from_filenum (const FT &ft, const FILENUM &filenum); int find_ft_from_filenum (const FT &ft, const FILENUM &filenum) { FILENUM thisfnum = toku_cachefile_filenum(ft->cf); diff --git a/storage/tokudb/PerconaFT/portability/file.cc b/storage/tokudb/PerconaFT/portability/file.cc index 5332a2dff5553..0e3efc1a12afc 100644 --- a/storage/tokudb/PerconaFT/portability/file.cc +++ b/storage/tokudb/PerconaFT/portability/file.cc @@ -356,6 +356,12 @@ toku_os_close(int fd) { // if EINTR, retry until success return r; } +int toku_os_rename(const char *old_name, const char *new_name) { + return rename(old_name, new_name); +} + +int toku_os_unlink(const char *path) { return unlink(path); } + ssize_t toku_os_read(int fd, void *buf, size_t count) { ssize_t r; diff --git a/storage/tokudb/PerconaFT/portability/memory.cc b/storage/tokudb/PerconaFT/portability/memory.cc index 2de12699c61f9..5430ff84b7059 100644 --- a/storage/tokudb/PerconaFT/portability/memory.cc +++ b/storage/tokudb/PerconaFT/portability/memory.cc @@ -313,6 +313,15 @@ toku_strdup(const char *s) { return (char *) toku_memdup(s, strlen(s)+1); } +char *toku_strndup(const char *s, size_t n) { + size_t s_size = strlen(s); + size_t bytes_to_copy = n > s_size ? s_size : n; + ++bytes_to_copy; + char *result = (char *)toku_memdup(s, bytes_to_copy); + result[bytes_to_copy - 1] = 0; + return result; +} + void toku_free(void *p) { if (p) { diff --git a/storage/tokudb/PerconaFT/portability/memory.h b/storage/tokudb/PerconaFT/portability/memory.h index 7780536f279ec..5ae652d39fc52 100644 --- a/storage/tokudb/PerconaFT/portability/memory.h +++ b/storage/tokudb/PerconaFT/portability/memory.h @@ -125,7 +125,9 @@ size_t toku_malloc_usable_size(void *p) __attribute__((__visibility__("default") void *toku_memdup (const void *v, size_t len); /* Toku-version of strdup. Use this so that it calls toku_malloc() */ char *toku_strdup (const char *s) __attribute__((__visibility__("default"))); - +/* Toku-version of strndup. Use this so that it calls toku_malloc() */ +char *toku_strndup(const char *s, size_t n) + __attribute__((__visibility__("default"))); /* Copy memory. Analogous to strdup() Crashes instead of returning NULL */ void *toku_xmemdup (const void *v, size_t len) __attribute__((__visibility__("default"))); /* Toku-version of strdup. Use this so that it calls toku_xmalloc() Crashes instead of returning NULL */ diff --git a/storage/tokudb/PerconaFT/portability/toku_portability.h b/storage/tokudb/PerconaFT/portability/toku_portability.h index 921d3a309f6c0..f127b0fe172dc 100644 --- a/storage/tokudb/PerconaFT/portability/toku_portability.h +++ b/storage/tokudb/PerconaFT/portability/toku_portability.h @@ -246,6 +246,8 @@ int toku_os_open(const char *path, int oflag, int mode); int toku_os_open_direct(const char *path, int oflag, int mode); int toku_os_close(int fd); int toku_os_fclose(FILE * stream); +int toku_os_rename(const char *old_name, const char *new_name); +int toku_os_unlink(const char *path); ssize_t toku_os_read(int fd, void *buf, size_t count); ssize_t toku_os_pread(int fd, void *buf, size_t count, off_t offset); void toku_os_recursive_delete(const char *path); diff --git a/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt index 47f6aa44a75e8..c01a8f0d62870 100644 --- a/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt @@ -108,11 +108,11 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) foreach(ov c d r) if (ov STREQUAL c) - set(gset 0) set(hset 0) + set(iset 0) else () - set(gset 0 1 2 3 4 5) - set(hset 0 1) + set(hset 0 1 2 3 4 5) + set(iset 0 1) endif () foreach(av 0 1) @@ -130,25 +130,27 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) foreach(dv ${dset}) foreach(ev ${eset}) foreach(fv 0 1) - foreach(gv ${gset}) + foreach(gv 0 1) foreach(hv ${hset}) - - if ((NOT ov STREQUAL c) AND (NOT cv) AND ((NOT bv) OR (NOT ev) OR (dv))) - set(iset 0 1) - else () - set(iset 0) - endif () - foreach(iv ${iset}) - set(testname "ydb/recovery_fileops_unit.${ov}${av}${bv}${cv}${dv}${ev}${fv}${gv}${hv}${iv}") - set(envdir "recovery_fileops_unit_dir/${ov}${av}${bv}${cv}${dv}${ev}${fv}${gv}${hv}${iv}") - set(errfile "recovery_fileops_unit_dir/${ov}${av}${bv}${cv}${dv}${ev}${fv}${gv}${hv}${iv}.ctest-errors") - add_test(NAME ${testname} - COMMAND run_recovery_fileops_unit.sh $ ${errfile} 137 - -O ${ov} -A ${av} -B ${bv} -C ${cv} -D ${dv} -E ${ev} -F ${fv} -G ${gv} -H ${hv} -I ${iv} - ) - setup_toku_test_properties(${testname} ${envdir}) - set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${errfile}") + + if ((NOT ov STREQUAL c) AND (NOT cv) AND ((NOT bv) OR (NOT ev) OR (dv))) + set(jset 0 1) + else () + set(jset 0) + endif () + + foreach(jv ${jset}) + set(testname "ydb/recovery_fileops_unit.${ov}${av}${bv}${cv}${dv}${ev}${fv}${gv}${hv}${iv}${jv}") + set(envdir "recovery_fileops_unit_dir/${ov}${av}${bv}${cv}${dv}${ev}${fv}${gv}${hv}${iv}${jv}") + set(errfile "recovery_fileops_unit_dir/${ov}${av}${bv}${cv}${dv}${ev}${fv}${gv}${hv}${iv}${jv}.ctest-errors") + add_test(NAME ${testname} + COMMAND run_recovery_fileops_unit.sh $ ${errfile} 137 + -O ${ov} -A ${av} -B ${bv} -C ${cv} -D ${dv} -E ${ev} -F ${fv} -G ${gv} -H ${hv} -I ${iv} -J ${jv} + ) + setup_toku_test_properties(${testname} ${envdir}) + set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${errfile}") + endforeach(jv) endforeach(iv) endforeach(hv) endforeach(gv) diff --git a/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc b/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc index 2c905c5ff122e..cc99ab560d85e 100644 --- a/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc +++ b/storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc @@ -36,17 +36,17 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#include "test.h" -#include "toku_pthread.h" #include -#include #include - +#include +#include "ft/logger/logger.h" +#include "test.h" +#include "toku_pthread.h" static int do_recover; static int do_crash; static char fileop; -static int choices['I'-'A'+1]; +static int choices['J' - 'A' + 1]; const int num_choices = sizeof(choices)/sizeof(choices[0]); static DB_TXN *txn; const char *oldname = "oldfoo"; @@ -58,11 +58,14 @@ static char *cmd; static void usage(void) { - fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] (-c|-r) -O fileop -A# -B# -C# -D# -E# -F# [-G# -H# -I#]\n" - " fileop = c/r/d (create/rename/delete)\n" - " Where # is a single digit number > 0.\n" - " A-F are required for fileop=create\n" - " A-I are required for fileop=delete, fileop=rename\n", cmd); + fprintf(stderr, + "Usage:\n%s [-v|-q]* [-h] (-c|-r) -O fileop -A# -B# -C# -D# -E# " + "-F# -G# [-H# -I# -J#]\n" + " fileop = c/r/d (create/rename/delete)\n" + " Where # is a single digit number > 0.\n" + " A-G are required for fileop=create\n" + " A-I are required for fileop=delete, fileop=rename\n", + cmd); exit(1); } @@ -129,19 +132,18 @@ get_choice_flush_log_before_crash(void) { return get_bool_choice('F'); } -static int -get_choice_create_type(void) { - return get_x_choice('G', 6); -} +static int get_choice_dir_per_db(void) { return get_bool_choice('G'); } + +static int get_choice_create_type(void) { return get_x_choice('H', 6); } static int get_choice_txn_does_open_close_before_fileop(void) { - return get_bool_choice('H'); + return get_bool_choice('I'); } static int get_choice_lock_table_split_fcreate(void) { - int choice = get_bool_choice('I'); + int choice = get_bool_choice('J'); if (choice) assert(fileop_did_commit()); return choice; @@ -156,63 +158,65 @@ do_args(int argc, char * const argv[]) { choices[i] = -1; } - int c; - while ((c = getopt(argc, argv, "vqhcrO:A:B:C:D:E:F:G:H:I:X:")) != -1) { - switch(c) { - case 'v': - verbose++; - break; - case 'q': - verbose--; - if (verbose<0) verbose=0; - break; - case 'h': - case '?': - usage(); - break; - case 'c': - do_crash = 1; - break; - case 'r': - do_recover = 1; - break; - case 'O': - if (fileop != '\0') + char c; + while ((c = getopt(argc, argv, "vqhcrO:A:B:C:D:E:F:G:H:I:J:X:")) != -1) { + switch (c) { + case 'v': + verbose++; + break; + case 'q': + verbose--; + if (verbose < 0) + verbose = 0; + break; + case 'h': + case '?': usage(); - fileop = optarg[0]; - switch (fileop) { - case 'c': - case 'r': - case 'd': - break; - default: + break; + case 'c': + do_crash = 1; + break; + case 'r': + do_recover = 1; + break; + case 'O': + if (fileop != '\0') usage(); - break; - } - break; - case 'A': - case 'B': - case 'C': - case 'D': - case 'E': - case 'F': - case 'G': - case 'H': - case 'I': - if (fileop == '\0') - usage(); - int num; - num = atoi(optarg); - if (num < 0 || num > 9) - usage(); - choices[c - 'A'] = num; - break; - case 'X': - if (strcmp(optarg, "novalgrind") == 0) { - // provide a way for the shell script runner to pass an - // arg that suppresses valgrind on this child process + fileop = optarg[0]; + switch (fileop) { + case 'c': + case 'r': + case 'd': + break; + default: + usage(); + break; + } + break; + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + if (fileop == '\0') + usage(); + int num; + num = atoi(optarg); + if (num < 0 || num > 9) + usage(); + choices[c - 'A'] = num; break; - } + case 'X': + if (strcmp(optarg, "novalgrind") == 0) { + // provide a way for the shell script runner to pass an + // arg that suppresses valgrind on this child process + break; + } // otherwise, fall through to an error default: usage(); @@ -222,7 +226,7 @@ do_args(int argc, char * const argv[]) { if (argc!=optind) { usage(); exit(1); } for (i = 0; i < num_choices; i++) { - if (i >= 'G' - 'A' && fileop == 'c') + if (i >= 'H' - 'A' && fileop == 'c') break; if (choices[i] == -1) usage(); @@ -261,6 +265,8 @@ static void env_startup(void) { int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | recover_flag; r = db_env_create(&env, 0); CKERR(r); + r = env->set_dir_per_db(env, get_choice_dir_per_db()); + CKERR(r); env->set_errfile(env, stderr); r = env->open(env, TOKU_TEST_FILENAME, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); @@ -625,8 +631,11 @@ recover_and_verify(void) { else if (did_create_commit_early()) expect_old_name = 1; } - verify_file_exists(oldname, expect_old_name); - verify_file_exists(newname, expect_new_name); + // We can't expect files existence until recovery log was not flushed + if ((get_choice_flush_log_before_crash())) { + verify_file_exists(oldname, expect_old_name); + verify_file_exists(newname, expect_new_name); + } env_shutdown(); } diff --git a/storage/tokudb/PerconaFT/src/ydb-internal.h b/storage/tokudb/PerconaFT/src/ydb-internal.h index 2d6c84126e168..d40f7795b0b86 100644 --- a/storage/tokudb/PerconaFT/src/ydb-internal.h +++ b/storage/tokudb/PerconaFT/src/ydb-internal.h @@ -132,7 +132,8 @@ struct __toku_db_env_internal { int datadir_lockfd; int logdir_lockfd; int tmpdir_lockfd; - bool check_thp; // if set check if transparent huge pages are disables + bool check_thp; // if set check if transparent huge pages are disabled + bool dir_per_db; uint64_t (*get_loader_memory_size_callback)(void); uint64_t default_lock_timeout_msec; uint64_t (*get_lock_timeout_callback)(uint64_t default_lock_timeout_msec); diff --git a/storage/tokudb/PerconaFT/src/ydb.cc b/storage/tokudb/PerconaFT/src/ydb.cc index aed271bce4069..3341f6d76c624 100644 --- a/storage/tokudb/PerconaFT/src/ydb.cc +++ b/storage/tokudb/PerconaFT/src/ydb.cc @@ -1298,6 +1298,22 @@ env_get_check_thp(DB_ENV * env) { return env->i->check_thp; } +static bool env_set_dir_per_db(DB_ENV *env, bool new_val) { + HANDLE_PANICKED_ENV(env); + bool r = env->i->dir_per_db; + env->i->dir_per_db = new_val; + return r; +} + +static bool env_get_dir_per_db(DB_ENV *env) { + HANDLE_PANICKED_ENV(env); + return env->i->dir_per_db; +} + +static const char *env_get_data_dir(DB_ENV *env) { + return env->i->real_data_dir; +} + static int env_dbremove(DB_ENV * env, DB_TXN *txn, const char *fname, const char *dbname, uint32_t flags); static int @@ -2700,6 +2716,9 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) { USENV(do_backtrace); USENV(set_check_thp); USENV(get_check_thp); + USENV(set_dir_per_db); + USENV(get_dir_per_db); + USENV(get_data_dir); #undef USENV // unlocked methods @@ -3045,7 +3064,7 @@ env_dbrename(DB_ENV *env, DB_TXN *txn, const char *fname, const char *dbname, co if (env_is_db_with_dname_open(env, newname)) { return toku_ydb_do_error(env, EINVAL, "Cannot rename dictionary; Dictionary with target name has an open handle.\n"); } - + DBT old_dname_dbt; DBT new_dname_dbt; DBT iname_dbt; @@ -3065,10 +3084,35 @@ env_dbrename(DB_ENV *env, DB_TXN *txn, const char *fname, const char *dbname, co r = EEXIST; } else if (r == DB_NOTFOUND) { + DBT new_iname_dbt; + // Do not rename ft file if 'dir_per_db' option is not set + auto new_iname = + env->get_dir_per_db(env) + ? generate_iname_for_rename_or_open( + env, txn, newname, false) + : std::unique_ptr( + toku_strdup(iname), &toku_free); + toku_fill_dbt( + &new_iname_dbt, new_iname.get(), strlen(new_iname.get()) + 1); + // remove old (dname,iname) and insert (newname,iname) in directory r = toku_db_del(env->i->directory, txn, &old_dname_dbt, DB_DELETE_ANY, true); if (r != 0) { goto exit; } - r = toku_db_put(env->i->directory, txn, &new_dname_dbt, &iname_dbt, 0, true); + + // Do not rename ft file if 'dir_per_db' option is not set + if (env->get_dir_per_db(env)) + r = toku_ft_rename_iname(txn, + env->get_data_dir(env), + iname, + new_iname.get(), + env->i->cachetable); + + r = toku_db_put(env->i->directory, + txn, + &new_dname_dbt, + &new_iname_dbt, + 0, + true); if (r != 0) { goto exit; } //Now that we have writelocks on both dnames, verify that there are still no handles open. (to prevent race conditions) @@ -3091,7 +3135,7 @@ env_dbrename(DB_ENV *env, DB_TXN *txn, const char *fname, const char *dbname, co // otherwise, we're okay in marking this ft as remove on // commit. no new handles can open for this dictionary // because the txn has directory write locks on the dname - if (txn && !can_acquire_table_lock(env, txn, iname)) { + if (txn && !can_acquire_table_lock(env, txn, new_iname.get())) { r = DB_LOCK_NOTGRANTED; } // We don't do anything at the ft or cachetable layer for rename. diff --git a/storage/tokudb/PerconaFT/src/ydb_db.cc b/storage/tokudb/PerconaFT/src/ydb_db.cc index e5bd4e7d089d5..100d1bfa20b14 100644 --- a/storage/tokudb/PerconaFT/src/ydb_db.cc +++ b/storage/tokudb/PerconaFT/src/ydb_db.cc @@ -83,8 +83,7 @@ ydb_db_layer_get_status(YDB_DB_LAYER_STATUS statp) { *statp = ydb_db_layer_status; } -static void -create_iname_hint(const char *dname, char *hint) { +void create_iname_hint(const char *dname, char *hint) { //Requires: size of hint array must be > strlen(dname) //Copy alphanumeric characters only. //Replace strings of non-alphanumeric characters with a single underscore. @@ -105,11 +104,43 @@ create_iname_hint(const char *dname, char *hint) { *hint = '\0'; } +void create_iname_hint_for_dbdir(const char *dname, char *hint) { + assert(dname); + if (*dname == '.') + ++dname; + if (*dname == '/') + ++dname; + bool underscored = false; + bool dbdir_is_parsed = false; + // Do not change the first '/' because this is + // delimiter which splits name into database dir + // and table dir. + while (*dname) { + if (isalnum(*dname) || (*dname == '/' && !dbdir_is_parsed)) { + char c = *dname++; + *hint++ = c; + if (c == '/') + dbdir_is_parsed = true; + underscored = false; + } else { + if (!underscored) + *hint++ = '_'; + dname++; + underscored = true; + } + } + *hint = '\0'; +} + // n < 0 means to ignore mark and ignore n // n >= 0 means to include mark ("_B_" or "_P_") with hex value of n in iname // (intended for use by loader, which will create many inames using one txnid). -static char * -create_iname(DB_ENV *env, uint64_t id1, uint64_t id2, char *hint, const char *mark, int n) { +char *create_iname(DB_ENV *env, + uint64_t id1, + uint64_t id2, + char *hint, + const char *mark, + int n) { int bytes; char inamebase[strlen(hint) + 8 + // hex file format version @@ -138,6 +169,34 @@ create_iname(DB_ENV *env, uint64_t id1, uint64_t id2, char *hint, const char *ma return rval; } +static uint64_t nontransactional_open_id = 0; + +std::unique_ptr generate_iname_for_rename_or_open( + DB_ENV *env, + DB_TXN *txn, + const char *dname, + bool is_open) { + std::unique_ptr result(nullptr, &toku_free); + char hint[strlen(dname) + 1]; + uint64_t id1 = 0; + uint64_t id2 = 0; + + if (txn) { + id1 = toku_txn_get_txnid(db_txn_struct_i(txn)->tokutxn).parent_id64; + id2 = toku_txn_get_txnid(db_txn_struct_i(txn)->tokutxn).child_id64; + } else if (is_open) + id1 = toku_sync_fetch_and_add(&nontransactional_open_id, 1); + + if (env->get_dir_per_db(env) && !toku_os_is_absolute_name(dname)) + create_iname_hint_for_dbdir(dname, hint); + else + create_iname_hint(dname, hint); + + result.reset(create_iname(env, id1, id2, hint, NULL, -1)); + + return result; +} + static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *dbname, DBTYPE dbtype, uint32_t flags, int mode); // Effect: Do the work required of DB->close(). @@ -227,8 +286,6 @@ db_open_subdb(DB * db, DB_TXN * txn, const char *fname, const char *dbname, DBTY return r; } -static uint64_t nontransactional_open_id = 0; - // inames are created here. // algorithm: // begin txn @@ -286,27 +343,15 @@ toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *dbname, DBTYP toku_fill_dbt(&dname_dbt, dname, strlen(dname)+1); toku_init_dbt_flags(&iname_dbt, DB_DBT_REALLOC); r = toku_db_get(db->dbenv->i->directory, txn, &dname_dbt, &iname_dbt, DB_SERIALIZABLE); // allocates memory for iname - char *iname = (char *) iname_dbt.data; + std::unique_ptr iname( + static_cast(iname_dbt.data), &toku_free); if (r == DB_NOTFOUND && !is_db_create) { r = ENOENT; } else if (r==0 && is_db_excl) { r = EEXIST; } else if (r == DB_NOTFOUND) { - char hint[strlen(dname) + 1]; - - // create iname and make entry in directory - uint64_t id1 = 0; - uint64_t id2 = 0; - - if (txn) { - id1 = toku_txn_get_txnid(db_txn_struct_i(txn)->tokutxn).parent_id64; - id2 = toku_txn_get_txnid(db_txn_struct_i(txn)->tokutxn).child_id64; - } else { - id1 = toku_sync_fetch_and_add(&nontransactional_open_id, 1); - } - create_iname_hint(dname, hint); - iname = create_iname(db->dbenv, id1, id2, hint, NULL, -1); // allocated memory for iname - toku_fill_dbt(&iname_dbt, iname, strlen(iname) + 1); + iname = generate_iname_for_rename_or_open(db->dbenv, txn, dname, true); + toku_fill_dbt(&iname_dbt, iname.get(), strlen(iname.get()) + 1); // // put_flags will be 0 for performance only, avoid unnecessary query // if we are creating a hot index, per #3166, we do not want the write lock in directory grabbed. @@ -318,16 +363,13 @@ toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *dbname, DBTYP // we now have an iname if (r == 0) { - r = toku_db_open_iname(db, txn, iname, flags, mode); + r = toku_db_open_iname(db, txn, iname.get(), flags, mode); if (r == 0) { db->i->dname = toku_xstrdup(dname); env_note_db_opened(db->dbenv, db); // tell env that a new db handle is open (using dname) } } - if (iname) { - toku_free(iname); - } return r; } @@ -1181,7 +1223,10 @@ load_inames(DB_ENV * env, DB_TXN * txn, int N, DB * dbs[/*N*/], const char * new toku_fill_dbt(&dname_dbt, dname, strlen(dname)+1); // now create new iname char hint[strlen(dname) + 1]; - create_iname_hint(dname, hint); + if (env->get_dir_per_db(env) && !toku_os_is_absolute_name(dname)) + create_iname_hint_for_dbdir(dname, hint); + else + create_iname_hint(dname, hint); const char *new_iname = create_iname(env, xid.parent_id64, xid.child_id64, hint, mark, i); // allocates memory for iname_in_env new_inames_in_env[i] = new_iname; toku_fill_dbt(&iname_dbt, new_iname, strlen(new_iname) + 1); // iname_in_env goes in directory diff --git a/storage/tokudb/PerconaFT/src/ydb_db.h b/storage/tokudb/PerconaFT/src/ydb_db.h index 8b92dd1c3cb83..8be28857c142b 100644 --- a/storage/tokudb/PerconaFT/src/ydb_db.h +++ b/storage/tokudb/PerconaFT/src/ydb_db.h @@ -43,6 +43,8 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "ydb-internal.h" #include "ydb_txn.h" +#include + typedef enum { YDB_LAYER_DIRECTORY_WRITE_LOCKS = 0, /* total directory write locks taken */ YDB_LAYER_DIRECTORY_WRITE_LOCKS_FAIL, /* total directory write locks unable to be taken */ @@ -119,3 +121,17 @@ toku_db_destruct_autotxn(DB_TXN *txn, int r, bool changed) { } return r; } + +void create_iname_hint_for_dbdir(const char *dname, char *hint); +void create_iname_hint(const char *dname, char *hint); +char *create_iname(DB_ENV *env, + uint64_t id1, + uint64_t id2, + char *hint, + const char *mark, + int n); +std::unique_ptr generate_iname_for_rename_or_open( + DB_ENV *env, + DB_TXN *txn, + const char *dname, + bool is_open); diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 2b121189e8351..5e49e8d95d07a 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -543,6 +543,7 @@ static int tokudb_init_func(void *p) { db_env->change_fsync_log_period(db_env, tokudb::sysvars::fsync_log_period); db_env->set_lock_timeout_callback(db_env, tokudb_lock_timeout_callback); + db_env->set_dir_per_db(db_env, tokudb::sysvars::dir_per_db); db_env->set_loader_memory_size( db_env, diff --git a/storage/tokudb/mysql-test/tokudb/r/dir-per-db-with-custom-data-dir.result b/storage/tokudb/mysql-test/tokudb/r/dir-per-db-with-custom-data-dir.result new file mode 100644 index 0000000000000..a36dbcb28c042 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/dir-per-db-with-custom-data-dir.result @@ -0,0 +1,10 @@ +SELECT @@tokudb_dir_per_db; +@@tokudb_dir_per_db +1 +TOKUDB_DATA_DIR_CHANGED +1 +CREATE DATABASE tokudb_test; +USE tokudb_test; +CREATE TABLE t (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY) ENGINE=tokudb; +DROP TABLE t; +DROP DATABASE tokudb_test; diff --git a/storage/tokudb/mysql-test/tokudb/r/dir_per_db.result b/storage/tokudb/mysql-test/tokudb/r/dir_per_db.result new file mode 100644 index 0000000000000..371f97406c8eb --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/dir_per_db.result @@ -0,0 +1,180 @@ +######## +# tokudb_dir_per_db = 1 +######## +SET GLOBAL tokudb_dir_per_db= 1; +######## +# CREATE +######## +CREATE TABLE t1 (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, b INT(10) UNSIGNED NOT NULL) ENGINE=tokudb; +INSERT INTO t1 SET b = 10; +INSERT INTO t1 SET b = 20; +SELECT b FROM t1 ORDER BY a; +b +10 +20 +CREATE INDEX b ON t1 (b); +CREATE INDEX ab ON t1 (a,b); +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +t1_key_ab_id.tokudb +t1_key_b_id.tokudb +t1_main_id.tokudb +t1_status_id.tokudb +######## +# RENAME +######## +RENAME TABLE t1 TO t2; +SELECT b FROM t2 ORDER BY a; +b +10 +20 +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +t2_key_ab_id.tokudb +t2_key_b_id.tokudb +t2_main_id.tokudb +t2_status_id.tokudb +######## +# DROP +######## +DROP TABLE t2; +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +######## +# tokudb_dir_per_db = 0 +######## +SET GLOBAL tokudb_dir_per_db= 0; +######## +# CREATE +######## +CREATE TABLE t1 (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, b INT(10) UNSIGNED NOT NULL) ENGINE=tokudb; +INSERT INTO t1 SET b = 10; +INSERT INTO t1 SET b = 20; +SELECT b FROM t1 ORDER BY a; +b +10 +20 +CREATE INDEX b ON t1 (b); +CREATE INDEX ab ON t1 (a,b); +## Looking for *.tokudb files in data_dir +_test_t1_key_ab_id.tokudb +_test_t1_key_b_id.tokudb +_test_t1_main_id.tokudb +_test_t1_status_id.tokudb +## Looking for *.tokudb files in data_dir/test +######## +# RENAME +######## +RENAME TABLE t1 TO t2; +SELECT b FROM t2 ORDER BY a; +b +10 +20 +## Looking for *.tokudb files in data_dir +_test_t1_key_ab_id.tokudb +_test_t1_key_b_id.tokudb +_test_t1_main_id.tokudb +_test_t1_status_id.tokudb +## Looking for *.tokudb files in data_dir/test +######## +# DROP +######## +DROP TABLE t2; +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +######## +# CREATE on tokudb_dir_per_db = 0 and RENAME on tokudb_dir_per_db = 1 and vice versa +######## +######## +# tokudb_dir_per_db = (1 - 1); +######## +SET GLOBAL tokudb_dir_per_db= (1 - 1);; +######## +# CREATE +######## +CREATE TABLE t1 (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, b INT(10) UNSIGNED NOT NULL) ENGINE=tokudb; +INSERT INTO t1 SET b = 10; +INSERT INTO t1 SET b = 20; +SELECT b FROM t1 ORDER BY a; +b +10 +20 +CREATE INDEX b ON t1 (b); +CREATE INDEX ab ON t1 (a,b); +## Looking for *.tokudb files in data_dir +_test_t1_key_ab_id.tokudb +_test_t1_key_b_id.tokudb +_test_t1_main_id.tokudb +_test_t1_status_id.tokudb +## Looking for *.tokudb files in data_dir/test +######## +# tokudb_dir_per_db = 1 +######## +SET GLOBAL tokudb_dir_per_db= 1; +######## +# RENAME +######## +RENAME TABLE t1 TO t2; +SELECT b FROM t2 ORDER BY a; +b +10 +20 +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +t2_key_ab_id.tokudb +t2_key_b_id.tokudb +t2_main_id.tokudb +t2_status_id.tokudb +######## +# DROP +######## +DROP TABLE t2; +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +######## +# tokudb_dir_per_db = (1 - 0); +######## +SET GLOBAL tokudb_dir_per_db= (1 - 0);; +######## +# CREATE +######## +CREATE TABLE t1 (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, b INT(10) UNSIGNED NOT NULL) ENGINE=tokudb; +INSERT INTO t1 SET b = 10; +INSERT INTO t1 SET b = 20; +SELECT b FROM t1 ORDER BY a; +b +10 +20 +CREATE INDEX b ON t1 (b); +CREATE INDEX ab ON t1 (a,b); +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +t1_key_ab_id.tokudb +t1_key_b_id.tokudb +t1_main_id.tokudb +t1_status_id.tokudb +######## +# tokudb_dir_per_db = 0 +######## +SET GLOBAL tokudb_dir_per_db= 0; +######## +# RENAME +######## +RENAME TABLE t1 TO t2; +SELECT b FROM t2 ORDER BY a; +b +10 +20 +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +t1_key_ab_id.tokudb +t1_key_b_id.tokudb +t1_main_id.tokudb +t1_status_id.tokudb +######## +# DROP +######## +DROP TABLE t2; +## Looking for *.tokudb files in data_dir +## Looking for *.tokudb files in data_dir/test +SET GLOBAL tokudb_dir_per_db=default; diff --git a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result index 6f9592ddc1fc0..ecd4d07720611 100644 --- a/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result +++ b/storage/tokudb/mysql-test/tokudb/r/i_s_tokudb_lock_waits_released.result @@ -2,6 +2,7 @@ set default_storage_engine='tokudb'; set tokudb_prelock_empty=false; drop table if exists t; create table t (id int primary key); +t should be empty select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; @@ -15,17 +16,21 @@ insert into t values (1); set autocommit=0; set tokudb_lock_timeout=600000; insert into t values (1); +should find the presence of a lock on 1st transaction select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main +should find the presence of a lock_wait on the 2nd transaction select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name REQUEST_TRX_ID BLOCK_TRX_ID ./test/t-main 0001000000 0001000000 LOCK_WAITS_START_TIME test t main +should find the presence of two transactions select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TRX_ID MYSQL_ID TRX_ID MYSQL_ID commit; +verify that the lock on the 1st transaction is released and replaced by the lock for the 2nd transaction select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main @@ -33,6 +38,8 @@ select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name ERROR 23000: Duplicate entry '1' for key 'PRIMARY' commit; +verify that txn_a replace (1) blocks txn_b replace (1) and txn_b eventually gets the lock on (1) and completes +verify that the lock on the 2nd transaction has been released, should be be empty select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; @@ -46,23 +53,28 @@ replace into t values (1); set autocommit=0; set tokudb_lock_timeout=600000; replace into t values (1); +should find the presence of a lock on 1st transaction select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main +should find the presence of a lock_wait on the 2nd transaction select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name REQUEST_TRX_ID BLOCK_TRX_ID ./test/t-main 0001000000 0001000000 LOCK_WAITS_START_TIME test t main +should find the presence of two transactions select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id TRX_ID MYSQL_ID TRX_ID MYSQL_ID commit; +verify that the lock on the 1st transaction is released and replaced by the lock for the 2nd transaction select * from information_schema.tokudb_locks; locks_trx_id locks_mysql_thread_id locks_dname locks_key_left locks_key_right locks_table_schema locks_table_name locks_table_dictionary_name TRX_ID MYSQL_ID ./test/t-main 0001000000 0001000000 test t main select * from information_schema.tokudb_lock_waits; requesting_trx_id blocking_trx_id lock_waits_dname lock_waits_key_left lock_waits_key_right lock_waits_start_time lock_waits_table_schema lock_waits_table_name lock_waits_table_dictionary_name commit; +verify that the lock on the 2nd transaction has been released, should be be empty select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; trx_id trx_mysql_thread_id select * from information_schema.tokudb_locks; diff --git a/storage/tokudb/mysql-test/tokudb/r/row_format.result b/storage/tokudb/mysql-test/tokudb/r/row_format.result new file mode 100644 index 0000000000000..cb66914844562 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/row_format.result @@ -0,0 +1,51 @@ +CREATE TABLE tokudb_row_format_test_1 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_DEFAULT; +CREATE TABLE tokudb_row_format_test_2 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_FAST; +CREATE TABLE tokudb_row_format_test_3 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_SMALL; +CREATE TABLE tokudb_row_format_test_4 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_UNCOMPRESSED; +CREATE TABLE tokudb_row_format_test_5 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_ZLIB; +CREATE TABLE tokudb_row_format_test_6 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_LZMA; +CREATE TABLE tokudb_row_format_test_7 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_QUICKLZ; +CREATE TABLE tokudb_row_format_test_8 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_SNAPPY; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name like 'tokudb_row_format_test%' ORDER BY table_name; +table_name row_format engine +tokudb_row_format_test_1 tokudb_zlib TokuDB +tokudb_row_format_test_2 tokudb_quicklz TokuDB +tokudb_row_format_test_3 tokudb_lzma TokuDB +tokudb_row_format_test_4 tokudb_uncompressed TokuDB +tokudb_row_format_test_5 tokudb_zlib TokuDB +tokudb_row_format_test_6 tokudb_lzma TokuDB +tokudb_row_format_test_7 tokudb_quicklz TokuDB +tokudb_row_format_test_8 tokudb_snappy TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_FAST; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_quicklz TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_SMALL; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_lzma TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_UNCOMPRESSED; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_uncompressed TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_ZLIB; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_zlib TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_SNAPPY; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_snappy TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_QUICKLZ; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_quicklz TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_LZMA; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_lzma TokuDB +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_DEFAULT; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; +table_name row_format engine +tokudb_row_format_test_1 tokudb_zlib TokuDB +DROP TABLE tokudb_row_format_test_1, tokudb_row_format_test_2, tokudb_row_format_test_3, tokudb_row_format_test_4, tokudb_row_format_test_5, tokudb_row_format_test_6, tokudb_row_format_test_7, tokudb_row_format_test_8; diff --git a/storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir-master.opt b/storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir-master.opt new file mode 100644 index 0000000000000..a9090f4d1157e --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir-master.opt @@ -0,0 +1 @@ +--loose-tokudb_data_dir="$MYSQL_TMP_DIR" --loose-tokudb-dir-per-db=1 diff --git a/storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir.test b/storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir.test new file mode 100644 index 0000000000000..7f415a7251595 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/dir-per-db-with-custom-data-dir.test @@ -0,0 +1,16 @@ +--source include/have_tokudb.inc + +SELECT @@tokudb_dir_per_db; + +--disable_query_log +--eval SELECT STRCMP(@@tokudb_data_dir, '$MYSQL_TMP_DIR') = 0 AS TOKUDB_DATA_DIR_CHANGED +--enable_query_log + +CREATE DATABASE tokudb_test; +USE tokudb_test; +CREATE TABLE t (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY) ENGINE=tokudb; + +--file_exists $MYSQL_TMP_DIR/tokudb_test + +DROP TABLE t; +DROP DATABASE tokudb_test; diff --git a/storage/tokudb/mysql-test/tokudb/t/dir_per_db.test b/storage/tokudb/mysql-test/tokudb/t/dir_per_db.test new file mode 100644 index 0000000000000..b638b706d8719 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/dir_per_db.test @@ -0,0 +1,76 @@ +source include/have_tokudb.inc; + +--let $DB= test +--let $DATADIR= `select @@datadir` +--let $i= 2 + +while ($i) { + --dec $i + --echo ######## + --echo # tokudb_dir_per_db = $i + --echo ######## + --eval SET GLOBAL tokudb_dir_per_db= $i + --echo ######## + --echo # CREATE + --echo ######## + CREATE TABLE t1 (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, b INT(10) UNSIGNED NOT NULL) ENGINE=tokudb; + INSERT INTO t1 SET b = 10; + INSERT INTO t1 SET b = 20; + SELECT b FROM t1 ORDER BY a; + CREATE INDEX b ON t1 (b); + CREATE INDEX ab ON t1 (a,b); + --source dir_per_db_show_table_files.inc + --echo ######## + --echo # RENAME + --echo ######## + RENAME TABLE t1 TO t2; + SELECT b FROM t2 ORDER BY a; + --source dir_per_db_show_table_files.inc + --echo ######## + --echo # DROP + --echo ######## + DROP TABLE t2; + --source dir_per_db_show_table_files.inc +} + +--echo ######## +--echo # CREATE on tokudb_dir_per_db = 0 and RENAME on tokudb_dir_per_db = 1 and vice versa +--echo ######## + +--let $i= 2 + +while ($i) { + --dec $i + --let $inv_i= (1 - $i); + --echo ######## + --echo # tokudb_dir_per_db = $inv_i + --echo ######## + --eval SET GLOBAL tokudb_dir_per_db= $inv_i + --echo ######## + --echo # CREATE + --echo ######## + CREATE TABLE t1 (a INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, b INT(10) UNSIGNED NOT NULL) ENGINE=tokudb; + INSERT INTO t1 SET b = 10; + INSERT INTO t1 SET b = 20; + SELECT b FROM t1 ORDER BY a; + CREATE INDEX b ON t1 (b); + CREATE INDEX ab ON t1 (a,b); + --source dir_per_db_show_table_files.inc + --echo ######## + --echo # tokudb_dir_per_db = $i + --echo ######## + --eval SET GLOBAL tokudb_dir_per_db= $i + --echo ######## + --echo # RENAME + --echo ######## + RENAME TABLE t1 TO t2; + SELECT b FROM t2 ORDER BY a; + --source dir_per_db_show_table_files.inc + --echo ######## + --echo # DROP + --echo ######## + DROP TABLE t2; + --source dir_per_db_show_table_files.inc +} + +SET GLOBAL tokudb_dir_per_db=default; diff --git a/storage/tokudb/mysql-test/tokudb/t/dir_per_db_show_table_files.inc b/storage/tokudb/mysql-test/tokudb/t/dir_per_db_show_table_files.inc new file mode 100644 index 0000000000000..bdf7d5b235ff9 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/dir_per_db_show_table_files.inc @@ -0,0 +1,9 @@ +--sorted_result + +--echo ## Looking for *.tokudb files in data_dir +--source include/table_files_replace_pattern.inc +--list_files $DATADIR *.tokudb + +--echo ## Looking for *.tokudb files in data_dir/$DB +--source include/table_files_replace_pattern.inc +--list_files $DATADIR/$DB/ *.tokudb diff --git a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test index 6488f27cfbb63..924b11e29d665 100644 --- a/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test +++ b/storage/tokudb/mysql-test/tokudb/t/i_s_tokudb_lock_waits_released.test @@ -12,7 +12,7 @@ create table t (id int primary key); # verify that txn_a insert (1) blocks txn_b insert (1) and txn_b gets a duplicate key error -# should be empty +--echo t should be empty select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; @@ -28,7 +28,7 @@ set autocommit=0; set tokudb_lock_timeout=600000; # set lock wait timeout to 10 minutes send insert into t values (1); -# should find the presence of a lock on 1st transaction +--echo should find the presence of a lock on 1st transaction connection default; let $wait_condition= select count(*)=1 from information_schema.processlist where info='insert into t values (1)' and state='update'; source include/wait_condition.inc; @@ -37,17 +37,17 @@ real_sleep 1; # delay a little to shorten the update -> write row -> lock wait r replace_column 1 TRX_ID 2 MYSQL_ID; select * from information_schema.tokudb_locks; -# should find the presence of a lock_wait on the 2nd transaction +--echo should find the presence of a lock_wait on the 2nd transaction replace_column 1 REQUEST_TRX_ID 2 BLOCK_TRX_ID 6 LOCK_WAITS_START_TIME; select * from information_schema.tokudb_lock_waits; -# should find the presence of two transactions +--echo should find the presence of two transactions replace_column 1 TRX_ID 2 MYSQL_ID; select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connection conn_a; commit; -# verify that the lock on the 1st transaction is released and replaced by the lock for the 2nd transaction +--echo verify that the lock on the 1st transaction is released and replaced by the lock for the 2nd transaction let $wait_condition= select count(*)=1 from information_schema.tokudb_locks where locks_dname='./test/t-main'; source include/wait_condition.inc; @@ -64,10 +64,8 @@ connection default; disconnect conn_a; disconnect conn_b; -# verify that txn_a replace (1) blocks txn_b replace (1) and txn_b eventually gets the lock on (1) and completes - -# verify that the lock on the 2nd transaction has been released -# should be be empty +--echo verify that txn_a replace (1) blocks txn_b replace (1) and txn_b eventually gets the lock on (1) and completes +--echo verify that the lock on the 2nd transaction has been released, should be be empty select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; @@ -83,7 +81,7 @@ set autocommit=0; set tokudb_lock_timeout=600000; # set lock wait timeout to 10 minutes send replace into t values (1); -# should find the presence of a lock on 1st transaction +--echo should find the presence of a lock on 1st transaction connection default; let $wait_condition= select count(*)=1 from information_schema.processlist where info='replace into t values (1)' and state='update'; source include/wait_condition.inc; @@ -92,17 +90,19 @@ real_sleep 1; # delay a little to shorten the update -> write row -> lock wait r replace_column 1 TRX_ID 2 MYSQL_ID; select * from information_schema.tokudb_locks; -# should find the presence of a lock_wait on the 2nd transaction +--echo should find the presence of a lock_wait on the 2nd transaction replace_column 1 REQUEST_TRX_ID 2 BLOCK_TRX_ID 6 LOCK_WAITS_START_TIME; select * from information_schema.tokudb_lock_waits; -# should find the presence of two transactions +--echo should find the presence of two transactions replace_column 1 TRX_ID 2 MYSQL_ID; select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; connection conn_a; commit; -# verify that the lock on the 1st transaction is released and replaced by the lock for the 2nd transaction +--echo verify that the lock on the 1st transaction is released and replaced by the lock for the 2nd transaction +let $wait_condition= select count(*)=1 from information_schema.tokudb_locks where locks_dname='./test/t-main'; +source include/wait_condition.inc; replace_column 1 TRX_ID 2 MYSQL_ID; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; @@ -115,8 +115,7 @@ connection default; disconnect conn_a; disconnect conn_b; -# verify that the lock on the 2nd transaction has been released -# should be be empty +--echo verify that the lock on the 2nd transaction has been released, should be be empty select trx_id,trx_mysql_thread_id from information_schema.tokudb_trx; select * from information_schema.tokudb_locks; select * from information_schema.tokudb_lock_waits; diff --git a/storage/tokudb/mysql-test/tokudb/t/row_format.test b/storage/tokudb/mysql-test/tokudb/t/row_format.test new file mode 100644 index 0000000000000..6533f8c06be95 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/row_format.test @@ -0,0 +1,41 @@ +# +# Test TokuDB compression option additions to row_format +# +--source include/have_tokudb.inc + +CREATE TABLE tokudb_row_format_test_1 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_DEFAULT; +CREATE TABLE tokudb_row_format_test_2 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_FAST; +CREATE TABLE tokudb_row_format_test_3 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_SMALL; +CREATE TABLE tokudb_row_format_test_4 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_UNCOMPRESSED; +CREATE TABLE tokudb_row_format_test_5 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_ZLIB; +CREATE TABLE tokudb_row_format_test_6 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_LZMA; +CREATE TABLE tokudb_row_format_test_7 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_QUICKLZ; +CREATE TABLE tokudb_row_format_test_8 (a INT) ENGINE=TokuDB ROW_FORMAT=TOKUDB_SNAPPY; + +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name like 'tokudb_row_format_test%' ORDER BY table_name; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_FAST; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_SMALL; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_UNCOMPRESSED; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_ZLIB; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_SNAPPY; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_QUICKLZ; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_LZMA; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +ALTER TABLE tokudb_row_format_test_1 ENGINE=TokuDB ROW_FORMAT=TOKUDB_DEFAULT; +SELECT table_name, row_format, engine FROM information_schema.tables WHERE table_name = 'tokudb_row_format_test_1'; + +DROP TABLE tokudb_row_format_test_1, tokudb_row_format_test_2, tokudb_row_format_test_3, tokudb_row_format_test_4, tokudb_row_format_test_5, tokudb_row_format_test_6, tokudb_row_format_test_7, tokudb_row_format_test_8; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/r/db938.result b/storage/tokudb/mysql-test/tokudb_bugs/r/db938.result index 6ec3a2c807943..30e0bdbebd78b 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/r/db938.result +++ b/storage/tokudb/mysql-test/tokudb_bugs/r/db938.result @@ -23,6 +23,7 @@ set DEBUG_SYNC = 'tokudb_after_truncate_all_dictionarys SIGNAL closed WAIT_FOR d TRUNCATE TABLE t1; set global tokudb_debug_pause_background_job_manager = FALSE; set DEBUG_SYNC = 'now SIGNAL done'; +set DEBUG_SYNC = 'RESET'; drop table t1; set session tokudb_auto_analyze = @orig_auto_analyze; set session tokudb_analyze_in_background = @orig_in_background; diff --git a/storage/tokudb/mysql-test/tokudb_bugs/t/db938.test b/storage/tokudb/mysql-test/tokudb_bugs/t/db938.test index f1912faad0260..50434a79a0017 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/t/db938.test +++ b/storage/tokudb/mysql-test/tokudb_bugs/t/db938.test @@ -40,6 +40,7 @@ insert into t1(b,c) values(0,0), (1,1), (2,2), (3,3); select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; # lets flip to another connection +--source include/count_sessions.inc connect(conn1, localhost, root); # set up the DEBUG_SYNC point @@ -64,6 +65,7 @@ connection conn1; reap; connection default; disconnect conn1; +set DEBUG_SYNC = 'RESET'; drop table t1; set session tokudb_auto_analyze = @orig_auto_analyze; @@ -74,3 +76,4 @@ set session tokudb_analyze_time = @orig_time; set global tokudb_cardinality_scale_percent = @orig_scale_percent; set session default_storage_engine = @orig_default_storage_engine; set global tokudb_debug_pause_background_job_manager = @orig_pause_background_job_manager; +--source include/wait_until_count_sessions.inc diff --git a/storage/tokudb/mysql-test/tokudb_parts/t/partition_debug_sync_tokudb.test b/storage/tokudb/mysql-test/tokudb_parts/t/partition_debug_sync_tokudb.test index be14d8814f04e..f97235a0a2d3a 100644 --- a/storage/tokudb/mysql-test/tokudb_parts/t/partition_debug_sync_tokudb.test +++ b/storage/tokudb/mysql-test/tokudb_parts/t/partition_debug_sync_tokudb.test @@ -56,7 +56,7 @@ partition by range (a) insert into t1 values (1), (11), (21), (33); SELECT * FROM t1; SHOW CREATE TABLE t1; ---replace_result #p# #P# #sp# #SP# +--source include/table_files_replace_pattern.inc --list_files $MYSQLD_DATADIR/test SET DEBUG_SYNC='before_open_in_get_all_tables SIGNAL parked WAIT_FOR open'; @@ -82,7 +82,7 @@ ALTER TABLE t1 REORGANIZE PARTITION p0 INTO disconnect con1; connection default; --reap ---replace_result #p# #P# #sp# #SP# +--source include/table_files_replace_pattern.inc --list_files $MYSQLD_DATADIR/test SHOW CREATE TABLE t1; SELECT * FROM t1; diff --git a/storage/tokudb/tokudb_sysvars.cc b/storage/tokudb/tokudb_sysvars.cc index 84f1c873a26c2..e518561527936 100644 --- a/storage/tokudb/tokudb_sysvars.cc +++ b/storage/tokudb/tokudb_sysvars.cc @@ -66,6 +66,7 @@ uint read_status_frequency = 0; my_bool strip_frm_data = FALSE; char* tmp_dir = NULL; uint write_status_frequency = 0; +my_bool dir_per_db = FALSE; char* version = (char*) TOKUDB_VERSION_STR; // file system reserve as a percentage of total disk space @@ -394,6 +395,18 @@ static MYSQL_SYSVAR_UINT( ~0U, 0); +static void tokudb_dir_per_db_update(THD* thd, + struct st_mysql_sys_var* sys_var, + void* var, const void* save) { + my_bool *value = (my_bool *) var; + *value = *(const my_bool *) save; + db_env->set_dir_per_db(db_env, *value); +} + +static MYSQL_SYSVAR_BOOL(dir_per_db, dir_per_db, + 0, "TokuDB store ft files in db directories", + NULL, tokudb_dir_per_db_update, FALSE); + #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL static MYSQL_SYSVAR_STR( gdb_path, @@ -935,6 +948,7 @@ st_mysql_sys_var* system_variables[] = { MYSQL_SYSVAR(tmp_dir), MYSQL_SYSVAR(version), MYSQL_SYSVAR(write_status_frequency), + MYSQL_SYSVAR(dir_per_db), #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL MYSQL_SYSVAR(gdb_path), diff --git a/storage/tokudb/tokudb_sysvars.h b/storage/tokudb/tokudb_sysvars.h index 70784fdcae3b1..c446e21257075 100644 --- a/storage/tokudb/tokudb_sysvars.h +++ b/storage/tokudb/tokudb_sysvars.h @@ -81,6 +81,7 @@ extern uint read_status_frequency; extern my_bool strip_frm_data; extern char* tmp_dir; extern uint write_status_frequency; +extern my_bool dir_per_db; extern char* version; #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL From 82ab92bd66eaaf951d49082a5c142759da59b137 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 25 Oct 2016 22:35:35 +0000 Subject: [PATCH 137/295] MDEV-10951 Field_newdate::cmp access violation The crash is caused by macro uint3korr() accessing memory (1 byte) past the end of allocated page. The macro is written such it reads 4 bytes instead of 3 and discards the value of the last byte. However, it is not always guaranteed that all uint3korr accesses will be valid (i.e that the caller allocates an extra byte after the value). In particular, the tree in Item_func_group_concat does not account for any extra bytes that it would need for comparison of keys in some cases (Field_newdate::cmp, Field_medium::cmp) The fix change uint3korr so it does not access extra bytes. --- include/byte_order_generic_x86.h | 10 ---------- include/byte_order_generic_x86_64.h | 8 -------- 2 files changed, 18 deletions(-) diff --git a/include/byte_order_generic_x86.h b/include/byte_order_generic_x86.h index 0a71a17829b70..a97dd0f43a37f 100644 --- a/include/byte_order_generic_x86.h +++ b/include/byte_order_generic_x86.h @@ -27,19 +27,9 @@ ((uint32) (uchar) (A)[0]))) #define sint4korr(A) (*((const long *) (A))) #define uint2korr(A) (*((const uint16 *) (A))) - -/* - Attention: Please, note, uint3korr reads 4 bytes (not 3)! - It means, that you have to provide enough allocated space. -*/ -#if defined(HAVE_valgrind) && !defined(_WIN32) #define uint3korr(A) (uint32) (((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ (((uint32) ((uchar) (A)[2])) << 16)) -#else -#define uint3korr(A) (long) (*((const unsigned int *) (A)) & 0xFFFFFF) -#endif - #define uint4korr(A) (*((const uint32 *) (A))) #define uint5korr(A) ((ulonglong)(((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ diff --git a/include/byte_order_generic_x86_64.h b/include/byte_order_generic_x86_64.h index b6b0c5d8ea582..8c7493965a996 100644 --- a/include/byte_order_generic_x86_64.h +++ b/include/byte_order_generic_x86_64.h @@ -27,17 +27,9 @@ ((uint32) (uchar) (A)[0]))) #define sint4korr(A) (int32) (*((int32 *) (A))) #define uint2korr(A) (uint16) (*((uint16 *) (A))) -/* - Attention: Please, note, uint3korr reads 4 bytes (not 3)! - It means, that you have to provide enough allocated space. -*/ -#if defined(HAVE_valgrind) && !defined(_WIN32) #define uint3korr(A) (uint32) (((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ (((uint32) ((uchar) (A)[2])) << 16)) -#else -#define uint3korr(A) (uint32) (*((unsigned int *) (A)) & 0xFFFFFF) -#endif #define uint4korr(A) (uint32) (*((uint32 *) (A))) #define uint5korr(A) ((ulonglong)(((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ From 74961760a4837d2deb33336329c28cf9ad9b4e9e Mon Sep 17 00:00:00 2001 From: sensssz Date: Tue, 25 Oct 2016 18:57:03 -0400 Subject: [PATCH 138/295] A few fixes for VATS in 10.1 --- libmariadb | 1 + .../suite/sys_vars/r/sysvars_innodb.result | 14 + storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/include/lock0lock.h | 6 +- storage/innobase/include/trx0trx.h | 2 + storage/innobase/lock/lock0lock.cc | 511 ++++++++++++------ storage/innobase/trx/trx0trx.cc | 2 + storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/include/lock0lock.h | 6 +- storage/xtradb/include/trx0trx.h | 2 + storage/xtradb/lock/lock0lock.cc | 423 +++++++++++---- storage/xtradb/trx/trx0trx.cc | 2 + 12 files changed, 707 insertions(+), 266 deletions(-) create mode 160000 libmariadb diff --git a/libmariadb b/libmariadb new file mode 160000 index 0000000000000..c8dd0899d484a --- /dev/null +++ b/libmariadb @@ -0,0 +1 @@ +Subproject commit c8dd0899d484ad698ec2da5bc8e3d19ff8b623b9 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index ee596b7f40e0c..f8eacfb3c6cb9 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1335,6 +1335,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY YES COMMAND_LINE_ARGUMENT NONE +VARIABLE_NAME INNODB_LOCK_SCHEDULE_ALGORITHM +SESSION_VALUE NULL +GLOBAL_VALUE fcfs +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE fcfs +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE ENUM +VARIABLE_COMMENT The algorithm Innodb uses for deciding which locks to grant next when a lock is released. Possible values are FCFS grant the locks in First-Come-First-Served order; VATS use the Variance-Aware-Transaction-Scheduling algorithm, which uses an Eldest-Transaction-First heuristic. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST fcfs,vats +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOCK_WAIT_TIMEOUT SESSION_VALUE 50 GLOBAL_VALUE 50 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index a436d079d03d7..42c80b82d4000 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1553,7 +1553,7 @@ thd_is_replication_slave_thread( /*============================*/ THD* thd) /*!< in: thread handle */ { - return((ibool) thd_slave_thread(thd)); + return thd && ((ibool) thd_slave_thread(thd)); } /******************************************************************//** diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 2b1158eddf87c..c00b545003696 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -44,10 +44,10 @@ extern ibool lock_print_waits; #endif /* UNIV_DEBUG */ /** Alternatives for innodb_lock_schedule_algorithm, which can be changed by - setting innodb_lock_schedule_algorithm. */ + setting innodb_lock_schedule_algorithm. */ enum innodb_lock_schedule_algorithm_t { - INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, /*!< First Come First Served */ - INNODB_LOCK_SCHEDULE_ALGORITHM_VATS /*!< Variance-Aware-Transaction-Scheduling */ + INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, /*!< First Come First Served */ + INNODB_LOCK_SCHEDULE_ALGORITHM_VATS /*!< Variance-Aware-Transaction-Scheduling */ }; extern ulong innodb_lock_schedule_algorithm; diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index add5e311957d4..3cc61de8e7650 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -844,6 +844,8 @@ struct trx_t{ time_t start_time; /*!< time the trx state last time became TRX_STATE_ACTIVE */ + clock_t start_time_micro; /*!< start time of transaction in + microseconds */ trx_id_t id; /*!< transaction id */ XID xid; /*!< X/Open XA transaction identification to identify a diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 572dc0e1dc429..198e4ebf6f27e 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -383,6 +383,25 @@ struct lock_stack_t { ulint heap_no; /*!< heap number if rec lock */ }; +/*********************************************************************//** +Checks if a waiting record lock request still has to wait in a queue. +@return lock that is causing the wait */ +static +const lock_t* +lock_rec_has_to_wait_in_queue( +/*==========================*/ + const lock_t* wait_lock); /*!< in: waiting record lock */ + +/*************************************************************//** +Grants a lock to a waiting lock request and releases the waiting transaction. +The caller must hold lock_sys->mutex. */ +static +void +lock_grant( +/*=======*/ + lock_t* lock, /*!< in/out: waiting lock request */ + bool owns_trx_mutex); /*!< in: whether lock->trx->mutex is owned */ + extern "C" void thd_report_wait_for(MYSQL_THD thd, MYSQL_THD other_thd); extern "C" int thd_need_wait_for(const MYSQL_THD thd); extern "C" @@ -1988,82 +2007,26 @@ wsrep_print_wait_locks( } #endif /* WITH_WSREP */ -/*********************************************************************//** -Check if lock1 has higher priority than lock2. -NULL has lowest priority. -Respect the preference of the upper server layer to reduce conflict -during in-order parallel replication. -If neither of them is wait lock, the first one has higher priority. -If only one of them is a wait lock, it has lower priority. -Otherwise, the one with an older transaction has higher priority. -@returns true if lock1 has higher priority, false otherwise. */ -bool -has_higher_priority( - lock_t *lock1, - lock_t *lock2) -{ - if (lock1 == NULL) { - return false; - } else if (lock2 == NULL) { - return true; - } - // Ask the upper server layer if any of the two trx should be prefered. - int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); - if (preference == -1) { - // lock1 is preferred as a victim, so lock2 has higher priority - return false; - } else if (preference == 1) { - // lock2 is preferred as a victim, so lock1 has higher priority - return true; - } - // No preference. Compre them by wait mode and trx age. - if (!lock_get_wait(lock1)) { - return true; - } else if (!lock_get_wait(lock2)) { - return false; - } - return lock1->trx->start_time < lock2->trx->start_time; -} - -/*********************************************************************//** -Insert a lock to the hash list according to the mode (whether it is a wait lock) -and the age of the transaction the it is associated with. -If the lock is not a wait lock, insert it to the head of the hash list. -Otherwise, insert it to the middle of the wait locks according to the age of the -transaciton. -*/ static void -lock_rec_insert_by_trx_age( - lock_t *in_lock, /*!< in: lock to be insert */ - bool wait) /*!< in: whether it's a wait lock */ +lock_rec_insert_to_head( + lock_t *in_lock, /*!< in: lock to be insert */ + ulint rec_fold) /*!< in: rec_fold of the page */ { - ulint space; - ulint page_no; - ulint rec_fold; - hash_cell_t* cell; + hash_cell_t* cell; lock_t* node; - lock_t* next; - space = in_lock->un_member.rec_lock.space; - page_no = in_lock->un_member.rec_lock.page_no; - rec_fold = lock_rec_fold(space, page_no); - cell = hash_get_nth_cell(lock_sys->rec_hash, - hash_calc_hash(rec_fold, lock_sys->rec_hash)); + if (in_lock == NULL) { + return; + } + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); node = (lock_t *) cell->node; - // If in_lock is not a wait lock, we insert it to the head of the list. - if (node == NULL || !wait || has_higher_priority(in_lock, node)) { + if (node != in_lock) { cell->node = in_lock; in_lock->hash = node; - return; - } - while (node != NULL && has_higher_priority((lock_t *) node->hash, in_lock)) { - node = (lock_t *) node->hash; } - next = (lock_t *) node->hash; - node->hash = in_lock; - in_lock->hash = next; } /*********************************************************************//** @@ -2093,8 +2056,10 @@ lock_rec_create( lock_t* lock; ulint page_no; ulint space; + ulint rec_fold; ulint n_bits; ulint n_bytes; + bool wait_lock; const page_t* page; ut_ad(lock_mutex_own()); @@ -2121,6 +2086,8 @@ lock_rec_create( type_mode = type_mode & ~(LOCK_GAP | LOCK_REC_NOT_GAP); } + wait_lock = type_mode & LOCK_WAIT; + /* Make lock bitmap bigger by a safety margin */ n_bits = page_dir_get_n_heap(page) + LOCK_PAGE_BITMAP_MARGIN; n_bytes = 1 + n_bits / 8; @@ -2136,6 +2103,7 @@ lock_rec_create( lock->un_member.rec_lock.space = space; lock->un_member.rec_lock.page_no = page_no; lock->un_member.rec_lock.n_bits = n_bytes * 8; + rec_fold = lock_rec_fold(space, page_no); /* Reset to zero the bitmap which resides immediately after the lock struct */ @@ -2228,13 +2196,27 @@ lock_rec_create( return(lock); } trx_mutex_exit(c_lock->trx); + } else if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && !thd_is_replication_slave_thread(lock->trx->mysql_thd)) { + if (wait_lock) { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); + } else { + lock_rec_insert_to_head(lock, rec_fold); + } } else { - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); } #else - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && !thd_is_replication_slave_thread(lock->trx->mysql_thd)) { + if (wait_lock) { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); + } else { + lock_rec_insert_to_head(lock, rec_fold); + } + } else { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); + } #endif /* WITH_WSREP */ if (!caller_owns_trx_mutex) { @@ -2257,6 +2239,135 @@ lock_rec_create( return(lock); } +/*********************************************************************//** +Check if lock1 has higher priority than lock2. +NULL has lowest priority. +Respect the preference of the upper server layer to reduce conflict +during in-order parallel replication. +If neither of them is wait lock, the first one has higher priority. +If only one of them is a wait lock, it has lower priority. +Otherwise, the one with an older transaction has higher priority. +@returns true if lock1 has higher priority, false otherwise. */ +bool +has_higher_priority( + lock_t *lock1, + lock_t *lock2) +{ + if (lock1 == NULL) { + return false; + } else if (lock2 == NULL) { + return true; + } + // Ask the upper server layer if any of the two trx should be prefered. + int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); + if (preference == -1) { + // lock1 is preferred as a victim, so lock2 has higher priority + return false; + } else if (preference == 1) { + // lock2 is preferred as a victim, so lock1 has higher priority + return true; + } + // No preference. Compre them by wait mode and trx age. + if (!lock_get_wait(lock1)) { + return true; + } else if (!lock_get_wait(lock2)) { + return false; + } + return lock1->trx->start_time_micro <= lock2->trx->start_time_micro; +} + +/*********************************************************************//** +Insert a lock to the hash list according to the mode (whether it is a wait +lock) and the age of the transaction the it is associated with. +If the lock is not a wait lock, insert it to the head of the hash list. +Otherwise, insert it to the middle of the wait locks according to the age of +the transaciton. */ +static +dberr_t +lock_rec_insert_by_trx_age( + lock_t *in_lock) /*!< in: lock to be insert */{ + ulint space; + ulint page_no; + ulint rec_fold; + lock_t* node; + lock_t* next; + hash_cell_t* cell; + + space = in_lock->un_member.rec_lock.space; + page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + + node = (lock_t *) cell->node; + // If in_lock is not a wait lock, we insert it to the head of the list. + if (node == NULL || !lock_get_wait(in_lock) || has_higher_priority(in_lock, node)) { + cell->node = in_lock; + in_lock->hash = node; + if (lock_get_wait(in_lock)) { + lock_grant(in_lock, true); + return DB_SUCCESS_LOCKED_REC; + } + return DB_SUCCESS; + } + while (node != NULL && has_higher_priority((lock_t *) node->hash, + in_lock)) { + node = (lock_t *) node->hash; + } + next = (lock_t *) node->hash; + node->hash = in_lock; + in_lock->hash = next; + + if (lock_get_wait(in_lock) && !lock_rec_has_to_wait_in_queue(in_lock)) { + lock_grant(in_lock, true); + if (cell->node != in_lock) { + // Move it to the front of the queue + node->hash = in_lock->hash; + next = (lock_t *) cell->node; + cell->node = in_lock; + in_lock->hash = next; + } + return DB_SUCCESS_LOCKED_REC; + } + + return DB_SUCCESS; +} + +static +bool +lock_queue_validate( + const lock_t *in_lock) /*!< in: lock whose hash list is to be validated */ +{ + ulint space; + ulint page_no; + ulint rec_fold; + hash_table_t* hash; + hash_cell_t* cell; + lock_t* next; + bool wait_lock = false; + + if (in_lock == NULL) { + return true; + } + + space = in_lock->un_member.rec_lock.space; + page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + next = (lock_t *) cell->node; + while (next != NULL) { + // If this is a granted lock, check that there's no wait lock before it. + if (!lock_get_wait(next)) { + ut_ad(!wait_lock); + } else { + wait_lock = true; + } + next = (lock_t *) next->hash; + } + return true; +} + /*********************************************************************//** Enqueues a waiting request for a lock which cannot be granted immediately. Checks for deadlocks. @@ -2289,6 +2400,9 @@ lock_rec_enqueue_waiting( trx_t* trx; lock_t* lock; trx_id_t victim_trx_id; + ulint space; + ulint page_no; + dberr_t err; ut_ad(lock_mutex_own()); ut_ad(!srv_read_only_mode); @@ -2362,36 +2476,46 @@ lock_rec_enqueue_waiting( transaction as a victim, it is possible that we already have the lock now granted! */ - return(DB_SUCCESS_LOCKED_REC); - } - - // Move it only when it does not cause a deadlock. - if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { - HASH_DELETE(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(buf_block_get_space(block), buf_block_get_page_no(block)), lock); - lock_rec_insert_by_trx_age(lock, true); - } - - trx->lock.que_state = TRX_QUE_LOCK_WAIT; + err = DB_SUCCESS_LOCKED_REC; + } else { + trx->lock.que_state = TRX_QUE_LOCK_WAIT; - trx->lock.was_chosen_as_deadlock_victim = FALSE; - trx->lock.wait_started = ut_time(); + trx->lock.was_chosen_as_deadlock_victim = FALSE; + trx->lock.wait_started = ut_time(); - ut_a(que_thr_stop(thr)); + ut_a(que_thr_stop(thr)); #ifdef UNIV_DEBUG - if (lock_print_waits) { - fprintf(stderr, "Lock wait for trx " TRX_ID_FMT " in index ", - trx->id); - ut_print_name(stderr, trx, FALSE, index->name); - } + if (lock_print_waits) { + fprintf(stderr, "Lock wait for trx " TRX_ID_FMT " in index ", + trx->id); + ut_print_name(stderr, trx, FALSE, index->name); + } #endif /* UNIV_DEBUG */ - MONITOR_INC(MONITOR_LOCKREC_WAIT); + MONITOR_INC(MONITOR_LOCKREC_WAIT); - trx->n_rec_lock_waits++; + trx->n_rec_lock_waits++; - return(DB_LOCK_WAIT); + err = DB_LOCK_WAIT; + } + + // Move it only when it does not cause a deadlock. + if (err != DB_DEADLOCK + && innodb_lock_schedule_algorithm + == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && !thd_is_replication_slave_thread(lock->trx->mysql_thd)) { + space = buf_block_get_space(block); + page_no = buf_block_get_page_no(block); + HASH_DELETE(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(space, page_no), lock); + dberr_t res = lock_rec_insert_by_trx_age(lock); + if (res != DB_SUCCESS) { + return res; + } + } + + return err; } /*********************************************************************//** @@ -2820,13 +2944,16 @@ static void lock_grant( /*=======*/ - lock_t* lock) /*!< in/out: waiting lock request */ + lock_t* lock, /*!< in/out: waiting lock request */ + bool owns_trx_mutex) /*!< in: whether lock->trx->mutex is owned */ { ut_ad(lock_mutex_own()); lock_reset_lock_and_trx_wait(lock); - trx_mutex_enter(lock->trx); + if (!owns_trx_mutex) { + trx_mutex_enter(lock->trx); + } if (lock_get_mode(lock) == LOCK_AUTO_INC) { dict_table_t* table = lock->un_member.tab_lock.table; @@ -2875,7 +3002,9 @@ lock_grant( lock->wait_time = (ulint)difftime(ut_time(), lock->requested_time); - trx_mutex_exit(lock->trx); + if (!owns_trx_mutex) { + trx_mutex_exit(lock->trx); + } } /*************************************************************//** @@ -2913,23 +3042,62 @@ lock_rec_cancel( trx_mutex_exit(lock->trx); } -/*************************************************************//** -Move the lock to the head of the hash list. */ static void -lock_rec_move_to_front( - lock_t *lock_to_move, /*!< in: lock to be moved */ - ulint rec_fold) /*!< in: rec fold of the lock */ +lock_grant_and_move_on_page( + ulint space, + ulint page_no) { - if (lock_to_move != NULL) - { - // Move the target lock to the head of the list - hash_cell_t* cell = hash_get_nth_cell(lock_sys->rec_hash, - hash_calc_hash(rec_fold, lock_sys->rec_hash)); - if (lock_to_move != cell->node) { - lock_t *next = (lock_t *) cell->node; - cell->node = lock_to_move; - lock_to_move->hash = next; + lock_t* lock; + lock_t* next; + lock_t* previous; + ulint rec_fold = lock_rec_fold(space, page_no); + + previous = (lock_t *) hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash))->node; + if (previous == NULL) { + return; + } + if (previous->un_member.rec_lock.space == space && + previous->un_member.rec_lock.page_no == page_no) { + lock = previous; + } + else { + next = (lock_t *) previous->hash; + while (next && + (next->un_member.rec_lock.space != space || + next->un_member.rec_lock.page_no != page_no)) { + previous = next; + next = (lock_t *) previous->hash; + } + lock = (lock_t *) previous->hash; + } + + ut_ad(previous->hash == lock || previous == lock); + /* Grant locks if there are no conflicting locks ahead. + Move granted locks to the head of the list. */ + for (;lock != NULL;) { + /* If the lock is a wait lock on this page, and it does not need to wait. */ + if ((lock->un_member.rec_lock.space == space) + && (lock->un_member.rec_lock.page_no == page_no) + && lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + lock_grant(lock, false); + + if (previous != NULL) { + /* Move the lock to the head of the list. */ + HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); + lock_rec_insert_to_head(lock, rec_fold); + } else { + /* Already at the head of the list. */ + previous = lock; + } + /* Move on to the next lock. */ + lock = static_cast(HASH_GET_NEXT(hash, previous)); + } else { + previous = lock; + lock = static_cast(HASH_GET_NEXT(hash, lock)); } } } @@ -2951,9 +3119,7 @@ lock_rec_dequeue_from_page( { ulint space; ulint page_no; - ulint rec_fold; lock_t* lock; - lock_t* previous = NULL; trx_lock_t* trx_lock; ut_ad(lock_mutex_own()); @@ -2964,7 +3130,6 @@ lock_rec_dequeue_from_page( space = in_lock->un_member.rec_lock.space; page_no = in_lock->un_member.rec_lock.page_no; - rec_fold = lock_rec_fold(space, page_no); in_lock->index->table->n_rec_locks--; @@ -2976,7 +3141,9 @@ lock_rec_dequeue_from_page( MONITOR_INC(MONITOR_RECLOCK_REMOVED); MONITOR_DEC(MONITOR_NUM_RECLOCK); - if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { + if (innodb_lock_schedule_algorithm + == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS || + thd_is_replication_slave_thread(in_lock->trx->mysql_thd)) { /* Check if waiting locks in the queue can now be granted: grant locks if there are no conflicting locks ahead. Stop at the first X lock that is waiting or has been granted. */ @@ -2990,38 +3157,11 @@ lock_rec_dequeue_from_page( /* Grant the lock */ ut_ad(lock->trx != in_lock->trx); - lock_grant(lock); + lock_grant(lock, false); } } } else { - /* Grant locks if there are no conflicting locks ahead. - Move granted locks to the head of the list. */ - for (lock = lock_rec_get_first_on_page_addr(space, page_no); - lock != NULL;) { - - /* If the lock is a wait lock on this page, and it does not need to wait. */ - if ((lock->un_member.rec_lock.space == space) - && (lock->un_member.rec_lock.page_no == page_no) - && lock_get_wait(lock) - && !lock_rec_has_to_wait_in_queue(lock)) { - - lock_grant(lock); - - if (previous != NULL) { - /* Move the lock to the head of the list. */ - HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); - lock_rec_move_to_front(lock, rec_fold); - } else { - /* Already at the head of the list. */ - previous = lock; - } - /* Move on to the next lock. */ - lock = static_cast(HASH_GET_NEXT(hash, previous)); - } else { - previous = lock; - lock = static_cast(HASH_GET_NEXT(hash, lock)); - } - } + lock_grant_and_move_on_page(space, page_no); } } @@ -5319,12 +5459,71 @@ lock_table_dequeue( /* Grant the lock */ ut_ad(in_lock->trx != lock->trx); - lock_grant(lock); + lock_grant(lock, false); } } } /*=========================== LOCK RELEASE ==============================*/ +static +void +lock_grant_and_move_on_rec( + lock_t* first_lock, + ulint heap_no) +{ + lock_t* lock; + lock_t* previous; + ulint space; + ulint page_no; + ulint rec_fold; + + space = first_lock->un_member.rec_lock.space; + page_no = first_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + + previous = (lock_t *) hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash))->node; + if (previous == NULL) { + return; + } + if (previous == first_lock) { + lock = previous; + } else { + while (previous->hash && + previous->hash != first_lock) { + previous = (lock_t *) previous->hash; + } + lock = (lock_t *) previous->hash; + } + /* Grant locks if there are no conflicting locks ahead. + Move granted locks to the head of the list. */ + for (;lock != NULL;) { + + /* If the lock is a wait lock on this page, and it does not need to wait. */ + if (lock->un_member.rec_lock.space == space + && lock->un_member.rec_lock.page_no == page_no + && lock_rec_get_nth_bit(lock, heap_no) + && lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + lock_grant(lock, false); + + if (previous != NULL) { + /* Move the lock to the head of the list. */ + HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); + lock_rec_insert_to_head(lock, rec_fold); + } else { + /* Already at the head of the list. */ + previous = lock; + } + /* Move on to the next lock. */ + lock = static_cast(HASH_GET_NEXT(hash, previous)); + } else { + previous = lock; + lock = static_cast(HASH_GET_NEXT(hash, lock)); + } + } +} /*************************************************************//** Removes a granted record lock of a transaction from the queue and grants @@ -5388,17 +5587,24 @@ lock_rec_unlock( ut_a(!lock_get_wait(lock)); lock_rec_reset_nth_bit(lock, heap_no); - /* Check if we can now grant waiting lock requests */ + if (innodb_lock_schedule_algorithm + == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS || + thd_is_replication_slave_thread(lock->trx->mysql_thd)) { - for (lock = first_lock; lock != NULL; - lock = lock_rec_get_next(heap_no, lock)) { - if (lock_get_wait(lock) - && !lock_rec_has_to_wait_in_queue(lock)) { + /* Check if we can now grant waiting lock requests */ - /* Grant the lock */ - ut_ad(trx != lock->trx); - lock_grant(lock); + for (lock = first_lock; lock != NULL; + lock = lock_rec_get_next(heap_no, lock)) { + if (lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + /* Grant the lock */ + ut_ad(trx != lock->trx); + lock_grant(lock, false); + } } + } else { + lock_grant_and_move_on_rec(first_lock, heap_no); } lock_mutex_exit(); @@ -6442,6 +6648,9 @@ lock_rec_queue_validate( } } + ut_ad(innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS || + lock_queue_validate(lock)); + func_exit: if (!locked_lock_trx_sys) { lock_mutex_exit(); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 1a99e159d4182..cde3244876937 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -919,6 +919,8 @@ trx_start_low( trx->start_time = ut_time(); + trx->start_time_micro = clock(); + MONITOR_INC(MONITOR_TRX_ACTIVE); } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 7ce65e5bc6fc6..4ddf8f3c50d8d 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1826,7 +1826,7 @@ thd_is_replication_slave_thread( /*============================*/ THD* thd) /*!< in: thread handle */ { - return((ibool) thd_slave_thread(thd)); + return thd && ((ibool) thd_slave_thread(thd)); } /******************************************************************//** diff --git a/storage/xtradb/include/lock0lock.h b/storage/xtradb/include/lock0lock.h index 742da74f1b634..a12ca1d85e65e 100644 --- a/storage/xtradb/include/lock0lock.h +++ b/storage/xtradb/include/lock0lock.h @@ -46,10 +46,10 @@ extern ibool lock_print_waits; #endif /* UNIV_DEBUG */ /** Alternatives for innodb_lock_schedule_algorithm, which can be changed by - setting innodb_lock_schedule_algorithm. */ + setting innodb_lock_schedule_algorithm. */ enum innodb_lock_schedule_algorithm_t { - INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, /*!< First Come First Served */ - INNODB_LOCK_SCHEDULE_ALGORITHM_VATS /*!< Variance-Aware-Transaction-Scheduling */ + INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS, /*!< First Come First Served */ + INNODB_LOCK_SCHEDULE_ALGORITHM_VATS /*!< Variance-Aware-Transaction-Scheduling */ }; extern ulong innodb_lock_schedule_algorithm; diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h index ec810405b5a1e..59d6d961544fb 100644 --- a/storage/xtradb/include/trx0trx.h +++ b/storage/xtradb/include/trx0trx.h @@ -877,6 +877,8 @@ struct trx_t{ time_t start_time; /*!< time the trx state last time became TRX_STATE_ACTIVE */ + clock_t start_time_micro; /*!< start time of transaction in + microseconds */ trx_id_t id; /*!< transaction id */ XID xid; /*!< X/Open XA transaction identification to identify a diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 2091f926153f8..e626821ed7eee 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -383,6 +383,25 @@ struct lock_stack_t { ulint heap_no; /*!< heap number if rec lock */ }; +/*********************************************************************//** +Checks if a waiting record lock request still has to wait in a queue. +@return lock that is causing the wait */ +static +const lock_t* +lock_rec_has_to_wait_in_queue( +/*==========================*/ + const lock_t* wait_lock); /*!< in: waiting record lock */ + +/*************************************************************//** +Grants a lock to a waiting lock request and releases the waiting transaction. +The caller must hold lock_sys->mutex. */ +static +void +lock_grant( +/*=======*/ + lock_t* lock, /*!< in/out: waiting lock request */ + bool owns_trx_mutex); /*!< in: whether lock->trx->mutex is owned */ + extern "C" void thd_report_wait_for(MYSQL_THD thd, MYSQL_THD other_thd); extern "C" int thd_need_wait_for(const MYSQL_THD thd); extern "C" @@ -2045,48 +2064,121 @@ has_higher_priority( } else if (!lock_get_wait(lock2)) { return false; } - return lock1->trx->start_time < lock2->trx->start_time; + return lock1->trx->start_time_micro <= lock2->trx->start_time_micro; } /*********************************************************************//** -Insert a lock to the hash list according to the mode (whether it is a wait lock) -and the age of the transaction the it is associated with. +Insert a lock to the hash list according to the mode (whether it is a wait +lock) and the age of the transaction the it is associated with. If the lock is not a wait lock, insert it to the head of the hash list. -Otherwise, insert it to the middle of the wait locks according to the age of the -transaciton. -*/ +Otherwise, insert it to the middle of the wait locks according to the age of +the transaciton. */ static -void +dberr_t lock_rec_insert_by_trx_age( - lock_t *in_lock, /*!< in: lock to be insert */ - bool wait) /*!< in: whether it's a wait lock */ -{ - ulint space; - ulint page_no; - ulint rec_fold; - hash_cell_t* cell; + lock_t *in_lock) /*!< in: lock to be insert */{ + ulint space; + ulint page_no; + ulint rec_fold; lock_t* node; lock_t* next; + hash_cell_t* cell; space = in_lock->un_member.rec_lock.space; page_no = in_lock->un_member.rec_lock.page_no; rec_fold = lock_rec_fold(space, page_no); cell = hash_get_nth_cell(lock_sys->rec_hash, - hash_calc_hash(rec_fold, lock_sys->rec_hash)); + hash_calc_hash(rec_fold, lock_sys->rec_hash)); node = (lock_t *) cell->node; // If in_lock is not a wait lock, we insert it to the head of the list. - if (node == NULL || !wait || has_higher_priority(in_lock, node)) { + if (node == NULL || !lock_get_wait(in_lock) || has_higher_priority(in_lock, node)) { cell->node = in_lock; in_lock->hash = node; - return; + if (lock_get_wait(in_lock)) { + lock_grant(in_lock, true); + return DB_SUCCESS_LOCKED_REC; + } + return DB_SUCCESS; } - while (node != NULL && has_higher_priority((lock_t *) node->hash, in_lock)) { + while (node != NULL && has_higher_priority((lock_t *) node->hash, + in_lock)) { node = (lock_t *) node->hash; } next = (lock_t *) node->hash; node->hash = in_lock; in_lock->hash = next; + + if (lock_get_wait(in_lock) && !lock_rec_has_to_wait_in_queue(in_lock)) { + lock_grant(in_lock, true); + if (cell->node != in_lock) { + // Move it to the front of the queue + node->hash = in_lock->hash; + next = (lock_t *) cell->node; + cell->node = in_lock; + in_lock->hash = next; + } + return DB_SUCCESS_LOCKED_REC; + } + + return DB_SUCCESS; +} + +static +bool +lock_queue_validate( + const lock_t *in_lock) /*!< in: lock whose hash list is to be validated */ +{ + ulint space; + ulint page_no; + ulint rec_fold; + hash_table_t* hash; + hash_cell_t* cell; + lock_t* next; + bool wait_lock = false; + + if (in_lock == NULL) { + return true; + } + + space = in_lock->un_member.rec_lock.space; + page_no = in_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + next = (lock_t *) cell->node; + while (next != NULL) { + // If this is a granted lock, check that there's no wait lock before it. + if (!lock_get_wait(next)) { + ut_ad(!wait_lock); + } else { + wait_lock = true; + } + next = (lock_t *) next->hash; + } + return true; +} + +static +void +lock_rec_insert_to_head( + lock_t *in_lock, /*!< in: lock to be insert */ + ulint rec_fold) /*!< in: rec_fold of the page */ +{ + hash_cell_t* cell; + lock_t* node; + + if (in_lock == NULL) { + return; + } + + cell = hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash)); + node = (lock_t *) cell->node; + if (node != in_lock) { + cell->node = in_lock; + in_lock->hash = node; + } } /*********************************************************************//** @@ -2116,8 +2208,10 @@ lock_rec_create( lock_t* lock; ulint page_no; ulint space; + ulint rec_fold; ulint n_bits; ulint n_bytes; + bool wait_lock; const page_t* page; ut_ad(lock_mutex_own()); @@ -2144,6 +2238,8 @@ lock_rec_create( type_mode = type_mode & ~(LOCK_GAP | LOCK_REC_NOT_GAP); } + wait_lock = type_mode & LOCK_WAIT; + /* Make lock bitmap bigger by a safety margin */ n_bits = page_dir_get_n_heap(page) + LOCK_PAGE_BITMAP_MARGIN; n_bytes = 1 + n_bits / 8; @@ -2159,6 +2255,7 @@ lock_rec_create( lock->un_member.rec_lock.space = space; lock->un_member.rec_lock.page_no = page_no; lock->un_member.rec_lock.n_bits = n_bytes * 8; + rec_fold = lock_rec_fold(space, page_no); /* Reset to zero the bitmap which resides immediately after the lock struct */ @@ -2251,13 +2348,27 @@ lock_rec_create( return(lock); } trx_mutex_exit(c_lock->trx); + } else if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && !thd_is_replication_slave_thread(lock->trx->mysql_thd)) { + if (wait_lock) { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); + } else { + lock_rec_insert_to_head(lock, rec_fold); + } } else { - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); } #else - HASH_INSERT(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(space, page_no), lock); + if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && !thd_is_replication_slave_thread(lock->trx->mysql_thd)) { + if (wait_lock) { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); + } else { + lock_rec_insert_to_head(lock, rec_fold); + } + } else { + HASH_INSERT(lock_t, hash, lock_sys->rec_hash, rec_fold, lock); + } #endif /* WITH_WSREP */ lock_sys->rec_num++; @@ -2316,6 +2427,9 @@ lock_rec_enqueue_waiting( trx_id_t victim_trx_id; ulint sec; ulint ms; + ulint space; + ulint page_no; + dberr_t err; ut_ad(lock_mutex_own()); @@ -2390,41 +2504,51 @@ lock_rec_enqueue_waiting( transaction as a victim, it is possible that we already have the lock now granted! */ - return(DB_SUCCESS_LOCKED_REC); - } - - // Move it only when it does not cause a deadlock. - if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS) { - HASH_DELETE(lock_t, hash, lock_sys->rec_hash, - lock_rec_fold(buf_block_get_space(block), buf_block_get_page_no(block)), lock); - lock_rec_insert_by_trx_age(lock, true); - } - - trx->lock.que_state = TRX_QUE_LOCK_WAIT; + err = DB_SUCCESS_LOCKED_REC; + } else { + trx->lock.que_state = TRX_QUE_LOCK_WAIT; - trx->lock.was_chosen_as_deadlock_victim = FALSE; - trx->lock.wait_started = ut_time(); + trx->lock.was_chosen_as_deadlock_victim = FALSE; + trx->lock.wait_started = ut_time(); - if (UNIV_UNLIKELY(trx->take_stats)) { - ut_usectime(&sec, &ms); - trx->lock_que_wait_ustarted = (ib_uint64_t)sec * 1000000 + ms; - } + if (UNIV_UNLIKELY(trx->take_stats)) { + ut_usectime(&sec, &ms); + trx->lock_que_wait_ustarted = (ib_uint64_t)sec * 1000000 + ms; + } - ut_a(que_thr_stop(thr)); + ut_a(que_thr_stop(thr)); #ifdef UNIV_DEBUG - if (lock_print_waits) { - fprintf(stderr, "Lock wait for trx " TRX_ID_FMT " in index ", - trx->id); - ut_print_name(stderr, trx, FALSE, index->name); - } + if (lock_print_waits) { + fprintf(stderr, "Lock wait for trx " TRX_ID_FMT " in index ", + trx->id); + ut_print_name(stderr, trx, FALSE, index->name); + } #endif /* UNIV_DEBUG */ - MONITOR_INC(MONITOR_LOCKREC_WAIT); + MONITOR_INC(MONITOR_LOCKREC_WAIT); - trx->n_rec_lock_waits++; + trx->n_rec_lock_waits++; - return(DB_LOCK_WAIT); + err = DB_LOCK_WAIT; + } + + // Move it only when it does not cause a deadlock. + if (err != DB_DEADLOCK + && innodb_lock_schedule_algorithm + == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS + && !thd_is_replication_slave_thread(lock->trx->mysql_thd)) { + space = buf_block_get_space(block); + page_no = buf_block_get_page_no(block); + HASH_DELETE(lock_t, hash, lock_sys->rec_hash, + lock_rec_fold(space, page_no), lock); + dberr_t res = lock_rec_insert_by_trx_age(lock); + if (res != DB_SUCCESS) { + return res; + } + } + + return err; } /*********************************************************************//** @@ -2856,13 +2980,16 @@ static void lock_grant( /*=======*/ - lock_t* lock) /*!< in/out: waiting lock request */ + lock_t* lock, /*!< in/out: waiting lock request */ + bool owns_trx_mutex) /*!< in: whether lock->trx->mutex is owned */ { ut_ad(lock_mutex_own()); lock_reset_lock_and_trx_wait(lock); - trx_mutex_enter(lock->trx); + if (!owns_trx_mutex) { + trx_mutex_enter(lock->trx); + } if (lock_get_mode(lock) == LOCK_AUTO_INC) { dict_table_t* table = lock->un_member.tab_lock.table; @@ -2911,7 +3038,9 @@ lock_grant( lock->wait_time = (ulint)difftime(ut_time(), lock->requested_time); - trx_mutex_exit(lock->trx); + if (!owns_trx_mutex) { + trx_mutex_exit(lock->trx); + } } /*************************************************************//** @@ -2949,23 +3078,62 @@ lock_rec_cancel( trx_mutex_exit(lock->trx); } -/*************************************************************//** -Move the lock to the head of the hash list. */ static void -lock_rec_move_to_front( - lock_t *lock_to_move, /*!< in: lock to be moved */ - ulint rec_fold) /*!< in: rec fold of the lock */ +lock_grant_and_move_on_page( + ulint space, + ulint page_no) { - if (lock_to_move != NULL) - { - // Move the target lock to the head of the list - hash_cell_t* cell = hash_get_nth_cell(lock_sys->rec_hash, - hash_calc_hash(rec_fold, lock_sys->rec_hash)); - if (lock_to_move != cell->node) { - lock_t *next = (lock_t *) cell->node; - cell->node = lock_to_move; - lock_to_move->hash = next; + lock_t* lock; + lock_t* next; + lock_t* previous; + ulint rec_fold = lock_rec_fold(space, page_no); + + previous = (lock_t *) hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash))->node; + if (previous == NULL) { + return; + } + if (previous->un_member.rec_lock.space == space && + previous->un_member.rec_lock.page_no == page_no) { + lock = previous; + } + else { + next = (lock_t *) previous->hash; + while (next && + (next->un_member.rec_lock.space != space || + next->un_member.rec_lock.page_no != page_no)) { + previous = next; + next = (lock_t *) previous->hash; + } + lock = (lock_t *) previous->hash; + } + + ut_ad(previous->hash == lock || previous == lock); + /* Grant locks if there are no conflicting locks ahead. + Move granted locks to the head of the list. */ + for (;lock != NULL;) { + /* If the lock is a wait lock on this page, and it does not need to wait. */ + if ((lock->un_member.rec_lock.space == space) + && (lock->un_member.rec_lock.page_no == page_no) + && lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + lock_grant(lock, false); + + if (previous != NULL) { + /* Move the lock to the head of the list. */ + HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); + lock_rec_insert_to_head(lock, rec_fold); + } else { + /* Already at the head of the list. */ + previous = lock; + } + /* Move on to the next lock. */ + lock = static_cast(HASH_GET_NEXT(hash, previous)); + } else { + previous = lock; + lock = static_cast(HASH_GET_NEXT(hash, lock)); } } } @@ -2987,9 +3155,7 @@ lock_rec_dequeue_from_page( { ulint space; ulint page_no; - ulint rec_fold; lock_t* lock; - lock_t* previous = NULL; trx_lock_t* trx_lock; ut_ad(lock_mutex_own()); @@ -3000,7 +3166,6 @@ lock_rec_dequeue_from_page( space = in_lock->un_member.rec_lock.space; page_no = in_lock->un_member.rec_lock.page_no; - rec_fold = lock_rec_fold(space, page_no); in_lock->index->table->n_rec_locks--; @@ -3013,7 +3178,9 @@ lock_rec_dequeue_from_page( MONITOR_INC(MONITOR_RECLOCK_REMOVED); MONITOR_DEC(MONITOR_NUM_RECLOCK); - if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { + if (innodb_lock_schedule_algorithm + == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS || + thd_is_replication_slave_thread(in_lock->trx->mysql_thd)) { /* Check if waiting locks in the queue can now be granted: grant locks if there are no conflicting locks ahead. Stop at the first X lock that is waiting or has been granted. */ @@ -3027,38 +3194,11 @@ lock_rec_dequeue_from_page( /* Grant the lock */ ut_ad(lock->trx != in_lock->trx); - lock_grant(lock); + lock_grant(lock, false); } } } else { - /* Grant locks if there are no conflicting locks ahead. - Move granted locks to the head of the list. */ - for (lock = lock_rec_get_first_on_page_addr(space, page_no); - lock != NULL;) { - - /* If the lock is a wait lock on this page, and it does not need to wait. */ - if ((lock->un_member.rec_lock.space == space) - && (lock->un_member.rec_lock.page_no == page_no) - && lock_get_wait(lock) - && !lock_rec_has_to_wait_in_queue(lock)) { - - lock_grant(lock); - - if (previous != NULL) { - /* Move the lock to the head of the list. */ - HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); - lock_rec_move_to_front(lock, rec_fold); - } else { - /* Already at the head of the list. */ - previous = lock; - } - /* Move on to the next lock. */ - lock = static_cast(HASH_GET_NEXT(hash, previous)); - } else { - previous = lock; - lock = static_cast(HASH_GET_NEXT(hash, lock)); - } - } + lock_grant_and_move_on_page(space, page_no); } } @@ -5370,12 +5510,71 @@ lock_table_dequeue( /* Grant the lock */ ut_ad(in_lock->trx != lock->trx); - lock_grant(lock); + lock_grant(lock, false); } } } /*=========================== LOCK RELEASE ==============================*/ +static +void +lock_grant_and_move_on_rec( + lock_t* first_lock, + ulint heap_no) +{ + lock_t* lock; + lock_t* previous; + ulint space; + ulint page_no; + ulint rec_fold; + + space = first_lock->un_member.rec_lock.space; + page_no = first_lock->un_member.rec_lock.page_no; + rec_fold = lock_rec_fold(space, page_no); + + previous = (lock_t *) hash_get_nth_cell(lock_sys->rec_hash, + hash_calc_hash(rec_fold, lock_sys->rec_hash))->node; + if (previous == NULL) { + return; + } + if (previous == first_lock) { + lock = previous; + } else { + while (previous->hash && + previous->hash != first_lock) { + previous = (lock_t *) previous->hash; + } + lock = (lock_t *) previous->hash; + } + /* Grant locks if there are no conflicting locks ahead. + Move granted locks to the head of the list. */ + for (;lock != NULL;) { + + /* If the lock is a wait lock on this page, and it does not need to wait. */ + if (lock->un_member.rec_lock.space == space + && lock->un_member.rec_lock.page_no == page_no + && lock_rec_get_nth_bit(lock, heap_no) + && lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + lock_grant(lock, false); + + if (previous != NULL) { + /* Move the lock to the head of the list. */ + HASH_GET_NEXT(hash, previous) = HASH_GET_NEXT(hash, lock); + lock_rec_insert_to_head(lock, rec_fold); + } else { + /* Already at the head of the list. */ + previous = lock; + } + /* Move on to the next lock. */ + lock = static_cast(HASH_GET_NEXT(hash, previous)); + } else { + previous = lock; + lock = static_cast(HASH_GET_NEXT(hash, lock)); + } + } +} /*************************************************************//** Removes a granted record lock of a transaction from the queue and grants @@ -5439,17 +5638,24 @@ lock_rec_unlock( ut_a(!lock_get_wait(lock)); lock_rec_reset_nth_bit(lock, heap_no); - /* Check if we can now grant waiting lock requests */ + if (innodb_lock_schedule_algorithm + == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS || + thd_is_replication_slave_thread(lock->trx->mysql_thd)) { - for (lock = first_lock; lock != NULL; - lock = lock_rec_get_next(heap_no, lock)) { - if (lock_get_wait(lock) - && !lock_rec_has_to_wait_in_queue(lock)) { + /* Check if we can now grant waiting lock requests */ - /* Grant the lock */ - ut_ad(trx != lock->trx); - lock_grant(lock); + for (lock = first_lock; lock != NULL; + lock = lock_rec_get_next(heap_no, lock)) { + if (lock_get_wait(lock) + && !lock_rec_has_to_wait_in_queue(lock)) { + + /* Grant the lock */ + ut_ad(trx != lock->trx); + lock_grant(lock, false); + } } + } else { + lock_grant_and_move_on_rec(first_lock, heap_no); } lock_mutex_exit(); @@ -6505,6 +6711,9 @@ lock_rec_queue_validate( } } + ut_ad(innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS || + lock_queue_validate(lock)); + func_exit: if (!locked_lock_trx_sys) { lock_mutex_exit(); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 41f8c16619042..ec57a8e5c5424 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -1117,6 +1117,8 @@ trx_start_low( trx->start_time = ut_time(); + trx->start_time_micro = clock(); + MONITOR_INC(MONITOR_TRX_ACTIVE); } From ad5b88a892d3e78c7192f5eb77094b46c600ab94 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 26 Oct 2016 09:26:34 +0000 Subject: [PATCH 139/295] Fix build error in XtraDB on Windows. coming from Percona's workaround for glibc bug http://bugs.mysql.com/bug.php?id=82886 --- storage/xtradb/os/os0thread.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/storage/xtradb/os/os0thread.cc b/storage/xtradb/os/os0thread.cc index 9729268348c5a..af826027efc36 100644 --- a/storage/xtradb/os/os0thread.cc +++ b/storage/xtradb/os/os0thread.cc @@ -220,10 +220,19 @@ void os_thread_join( os_thread_t thread) { + /*This function is currently only used to workaround glibc bug + described in http://bugs.mysql.com/bug.php?id=82886 + + On Windows, no workarounds are necessary, all threads + are "detached" upon thread exit (handle is closed), so we do + nothing. + */ +#ifndef _WIN32 int ret MY_ATTRIBUTE((unused)) = pthread_join(thread, NULL); /* Waiting on already-quit threads is allowed */ ut_ad(ret == 0 || ret == ESRCH); +#endif } /*****************************************************************//** From 9155cc7090998a5b28a1f502466640b08242c6e8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 31 Aug 2016 15:57:02 +1000 Subject: [PATCH 140/295] MDEV-10292: Tokudb - PerconaFT - compile error in recent gcc The following directives to ignore warnings where in the PerconaFT build in tokudb. These generate errors when g++ ... -o xxx.so is used to compile are shared object. As these don't actually hit any warnings they have been removed. * -Wno-ignored-attributes * -Wno-pointer-bool-conversion Signed-off-by: Daniel Black --- storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake index a7292a89d872f..769bdffa5d99c 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake @@ -98,9 +98,7 @@ set_cflags_if_supported( -Wno-error=address-of-array-temporary -Wno-error=tautological-constant-out-of-range-compare -Wno-error=maybe-uninitialized - -Wno-ignored-attributes -Wno-error=extern-c-compat - -Wno-pointer-bool-conversion -fno-rtti -fno-exceptions -Wno-error=nonnull-compare From a3c980b381ead0ea13df8314258c7a8d11fe5cd1 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Mon, 24 Oct 2016 15:26:11 +0400 Subject: [PATCH 141/295] MDEV-10824 - Crash in CREATE OR REPLACE TABLE t1 AS SELECT spfunc() Code flow hit incorrect branch while closing table instances before removal. This branch expects thread to hold open table instance, whereas CREATE OR REPLACE doesn't actually hold open table instance. Before CREATE OR REPLACE TABLE it was impossible to hit this condition in LTM_PRELOCKED mode, thus the problem didn't expose itself during DROP TABLE or DROP DATABASE. Fixed by adjusting condition to take into account LTM_PRELOCKED mode, which can be set during CREATE OR REPLACE TABLE. --- mysql-test/r/create_or_replace.result | 11 +++++++++++ mysql-test/t/create_or_replace.test | 12 ++++++++++++ sql/sql_table.cc | 3 ++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/create_or_replace.result b/mysql-test/r/create_or_replace.result index 3a894e9fcb140..a43dc2eaca447 100644 --- a/mysql-test/r/create_or_replace.result +++ b/mysql-test/r/create_or_replace.result @@ -442,3 +442,14 @@ KILL QUERY con_id; ERROR 70100: Query execution was interrupted drop table t1; DROP TABLE t2; +# +# MDEV-10824 - Crash in CREATE OR REPLACE TABLE t1 AS SELECT spfunc() +# +CREATE TABLE t1(a INT); +CREATE FUNCTION f1() RETURNS VARCHAR(16383) RETURN 'test'; +CREATE OR REPLACE TABLE t1 AS SELECT f1(); +LOCK TABLE t1 WRITE; +CREATE OR REPLACE TABLE t1 AS SELECT f1(); +UNLOCK TABLES; +DROP FUNCTION f1; +DROP TABLE t1; diff --git a/mysql-test/t/create_or_replace.test b/mysql-test/t/create_or_replace.test index 7bba2b341c043..b37417f39d0c8 100644 --- a/mysql-test/t/create_or_replace.test +++ b/mysql-test/t/create_or_replace.test @@ -386,3 +386,15 @@ drop table t1; # Cleanup # DROP TABLE t2; + +--echo # +--echo # MDEV-10824 - Crash in CREATE OR REPLACE TABLE t1 AS SELECT spfunc() +--echo # +CREATE TABLE t1(a INT); +CREATE FUNCTION f1() RETURNS VARCHAR(16383) RETURN 'test'; +CREATE OR REPLACE TABLE t1 AS SELECT f1(); +LOCK TABLE t1 WRITE; +CREATE OR REPLACE TABLE t1 AS SELECT f1(); +UNLOCK TABLES; +DROP FUNCTION f1; +DROP TABLE t1; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 7cf31ee4fe832..050a3383612a6 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2464,7 +2464,8 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, if (table_type && table_type != view_pseudo_hton) ha_lock_engine(thd, table_type); - if (thd->locked_tables_mode) + if (thd->locked_tables_mode == LTM_LOCK_TABLES || + thd->locked_tables_mode == LTM_PRELOCKED_UNDER_LOCK_TABLES) { if (wait_while_table_is_used(thd, table->table, HA_EXTRA_NOT_USED)) { From 59a7bc35fc6526568e49f1087c022c5d01da088a Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 26 Oct 2016 14:09:11 +0400 Subject: [PATCH 142/295] Removed duplicate open_strategy assignments It is set in sql_yacc.yy. --- sql/sql_parse.cc | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index cbf723c1b493a..70511fcd849e3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2857,12 +2857,6 @@ case SQLCOM_PREPARE: create_info.table_charset= 0; } - /* - For CREATE TABLE we should not open the table even if it exists. - If the table exists, we should either not create it or replace it - */ - lex->query_tables->open_strategy= TABLE_LIST::OPEN_STUB; - /* If we are a slave, we should add OR REPLACE if we don't have IF EXISTS. This will help a slave to recover from @@ -8225,12 +8219,6 @@ bool create_table_precheck(THD *thd, TABLE_LIST *tables, if (check_fk_parent_table_access(thd, &lex->create_info, &lex->alter_info, create_table->db)) goto err; - /* - For CREATE TABLE we should not open the table even if it exists. - If the table exists, we should either not create it or replace it - */ - lex->query_tables->open_strategy= TABLE_LIST::OPEN_STUB; - error= FALSE; err: From 5569ac00590ba139bbc575c20de4c682919721e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 25 Oct 2016 15:08:15 +0300 Subject: [PATCH 143/295] MDEV-11126: Crash while altering persistent virtual column Problem was that if old virtual column is computed and stored there was no check if new column is really virtual column. --- mysql-test/r/alter_table.result | 55 +++++++++++++++++++++++++++++++++ mysql-test/t/alter_table.test | 25 +++++++++++++++ sql/sql_table.cc | 1 + 3 files changed, 81 insertions(+) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index e572fdb197cc8..2e371ac6ae6e3 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2021,3 +2021,58 @@ ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS event_id (event_id,market_id); Warnings: Note 1061 Multiple primary key defined DROP TABLE t1; +# +# MDEV-11126 Crash while altering persistent virtual column +# +CREATE TABLE `tab1` ( +`id` bigint(20) NOT NULL AUTO_INCREMENT, +`field2` set('option1','option2','option3','option4') NOT NULL, +`field3` set('option1','option2','option3','option4','option5') NOT NULL, +`field4` set('option1','option2','option3','option4') NOT NULL, +`field5` varchar(32) NOT NULL, +`field6` varchar(32) NOT NULL, +`field7` varchar(32) NOT NULL, +`field8` varchar(32) NOT NULL, +`field9` int(11) NOT NULL DEFAULT '1', +`field10` varchar(16) NOT NULL, +`field11` enum('option1','option2','option3') NOT NULL DEFAULT 'option1', +`v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT, +PRIMARY KEY (`id`) +) DEFAULT CHARSET=latin1; +ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128); +SHOW CREATE TABLE `tab1`; +Table Create Table +tab1 CREATE TABLE `tab1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `field2` set('option1','option2','option3','option4') NOT NULL, + `field3` set('option1','option2','option3','option4','option5') NOT NULL, + `field4` set('option1','option2','option3','option4') NOT NULL, + `field5` varchar(32) NOT NULL, + `field6` varchar(32) NOT NULL, + `field7` varchar(32) NOT NULL, + `field8` varchar(32) NOT NULL, + `field9` int(11) NOT NULL DEFAULT '1', + `field10` varchar(16) NOT NULL, + `field11` enum('option1','option2','option3') NOT NULL DEFAULT 'option1', + `v_col` varchar(128) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT; +SHOW CREATE TABLE `tab1`; +Table Create Table +tab1 CREATE TABLE `tab1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `field2` set('option1','option2','option3','option4') NOT NULL, + `field3` set('option1','option2','option3','option4','option5') NOT NULL, + `field4` set('option1','option2','option3','option4') NOT NULL, + `field5` varchar(32) NOT NULL, + `field6` varchar(32) NOT NULL, + `field7` varchar(32) NOT NULL, + `field8` varchar(32) NOT NULL, + `field9` int(11) NOT NULL DEFAULT '1', + `field10` varchar(16) NOT NULL, + `field11` enum('option1','option2','option3') NOT NULL DEFAULT 'option1', + `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE `tab1`; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index 05d915ec478b5..d2b8a6082a623 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1712,3 +1712,28 @@ CREATE TABLE t1 ( ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS event_id (event_id,market_id); DROP TABLE t1; +--echo # +--echo # MDEV-11126 Crash while altering persistent virtual column +--echo # + +CREATE TABLE `tab1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `field2` set('option1','option2','option3','option4') NOT NULL, + `field3` set('option1','option2','option3','option4','option5') NOT NULL, + `field4` set('option1','option2','option3','option4') NOT NULL, + `field5` varchar(32) NOT NULL, + `field6` varchar(32) NOT NULL, + `field7` varchar(32) NOT NULL, + `field8` varchar(32) NOT NULL, + `field9` int(11) NOT NULL DEFAULT '1', + `field10` varchar(16) NOT NULL, + `field11` enum('option1','option2','option3') NOT NULL DEFAULT 'option1', + `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT, + PRIMARY KEY (`id`) +) DEFAULT CHARSET=latin1; + +ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128); +SHOW CREATE TABLE `tab1`; +ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT; +SHOW CREATE TABLE `tab1`; +DROP TABLE `tab1`; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 050a3383612a6..5d4c551d7306f 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -6274,6 +6274,7 @@ static bool fill_alter_inplace_info(THD *thd, (field->stored_in_db || field->vcol_info->is_in_partitioning_expr())) { if (is_equal == IS_EQUAL_NO || + !new_field->vcol_info || !field->vcol_info->is_equal(new_field->vcol_info)) ha_alter_info->handler_flags|= Alter_inplace_info::ALTER_COLUMN_VCOL; else From 25932708b138aa89e5e9cea080e49d914f7bb724 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Oct 2016 12:30:18 +0200 Subject: [PATCH 144/295] backport include/search_pattern_in_file.inc from 10.1 --- mysql-test/include/search_pattern_in_file.inc | 15 +++++++++------ mysql-test/r/lowercase_fs_on.result | 1 + mysql-test/r/named_pipe.result | 1 + mysql-test/r/view.result | 1 + mysql-test/r/wait_timeout_not_windows.result | 1 + .../innodb/r/innodb-change-buffer-recovery.result | 1 + mysql-test/suite/rpl/r/rpl_checksum.result | 1 + mysql-test/suite/rpl/r/rpl_gtid_errorlog.result | 2 ++ 8 files changed, 17 insertions(+), 6 deletions(-) diff --git a/mysql-test/include/search_pattern_in_file.inc b/mysql-test/include/search_pattern_in_file.inc index 0d09cdcd36efb..84237026ed06e 100644 --- a/mysql-test/include/search_pattern_in_file.inc +++ b/mysql-test/include/search_pattern_in_file.inc @@ -60,12 +60,12 @@ perl; use strict; - my $search_file= $ENV{'SEARCH_FILE'} or die "SEARCH_FILE not set"; - my $search_pattern= $ENV{'SEARCH_PATTERN'} or die "SEARCH_PATTERN not set"; - my $search_range= $ENV{'SEARCH_RANGE'}; + my $search_file= $ENV{'SEARCH_FILE'} or die "SEARCH_FILE not set"; + my $search_pattern= $ENV{'SEARCH_PATTERN'} or die "SEARCH_PATTERN not set"; + my $search_range= $ENV{'SEARCH_RANGE'}; my $file_content; $search_range= 50000 unless $search_range =~ /-?[0-9]+/; - open(FILE, "$search_file") or die("Unable to open '$search_file': $!\n"); + open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n"); if ($search_range >= 0) { read(FILE, $file_content, $search_range, 0); } else { @@ -75,7 +75,10 @@ perl; read(FILE, $file_content, -$search_range, 0); } close(FILE); - if ( not $file_content =~ m{$search_pattern} ) { - die("# ERROR: The file '$search_file' does not contain the expected pattern $search_pattern\n->$file_content<-\n"); + $search_file =~ s{^.*?([^/\\]+)$}{$1}; + if ($file_content =~ m{$search_pattern}) { + print "FOUND /$search_pattern/ in $search_file\n" + } else { + print "NOT FOUND /$search_pattern/ in $search_file\n" } EOF diff --git a/mysql-test/r/lowercase_fs_on.result b/mysql-test/r/lowercase_fs_on.result index a090f46cfbf8b..b844b3f77dde4 100644 --- a/mysql-test/r/lowercase_fs_on.result +++ b/mysql-test/r/lowercase_fs_on.result @@ -1,3 +1,4 @@ # # Bug#20198490 : LOWER_CASE_TABLE_NAMES=0 ON WINDOWS LEADS TO PROBLEMS # +FOUND /\[ERROR\] The server option \'lower_case_table_names\' is configured to use case sensitive table names/ in my_restart.err diff --git a/mysql-test/r/named_pipe.result b/mysql-test/r/named_pipe.result index ddd48f0ba9166..43fb44beece1d 100644 --- a/mysql-test/r/named_pipe.result +++ b/mysql-test/r/named_pipe.result @@ -2154,3 +2154,4 @@ Privat (Private Nutzung) Mobilfunk Warnings: Warning 1052 Column 'kundentyp' in group statement is ambiguous drop table t1; +FOUND /\[ERROR\] Create named pipe failed/ in second-mysqld.err diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 52c379d03aff6..924b3a11fef1c 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -5432,6 +5432,7 @@ DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1, t2; create view v1 as select 1; +FOUND /mariadb-version/ in v1.frm drop view v1; # # MDEV-7260: Crash in get_best_combination when executing multi-table diff --git a/mysql-test/r/wait_timeout_not_windows.result b/mysql-test/r/wait_timeout_not_windows.result index df70aa9922160..867787a8ed340 100644 --- a/mysql-test/r/wait_timeout_not_windows.result +++ b/mysql-test/r/wait_timeout_not_windows.result @@ -1,3 +1,4 @@ set global log_warnings=2; set @@wait_timeout=1; +FOUND /Aborted.*Got timeout reading communication packets/ in mysqld.1.err set global log_warnings=@@log_warnings; diff --git a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result index cc2a0373444f8..07e13008e2718 100644 --- a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result +++ b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result @@ -33,6 +33,7 @@ INSERT INTO t1 VALUES(1,'X',1); SET DEBUG_DBUG='+d,crash_after_log_ibuf_upd_inplace'; SELECT b FROM t1 LIMIT 3; ERROR HY000: Lost connection to MySQL server during query +FOUND /Wrote log record for ibuf update in place operation/ in my_restart.err CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/rpl/r/rpl_checksum.result b/mysql-test/suite/rpl/r/rpl_checksum.result index 94d215e596a92..9e37fbf40b18e 100644 --- a/mysql-test/suite/rpl/r/rpl_checksum.result +++ b/mysql-test/suite/rpl/r/rpl_checksum.result @@ -143,6 +143,7 @@ SET debug_dbug= @old_dbug; INSERT INTO t4 VALUES (2); include/wait_for_slave_sql_error.inc [errno=1590] Last_SQL_Error = 'The incident LOST_EVENTS occurred on the master. Message: error writing to the binary log' +FOUND /Slave SQL: The incident LOST_EVENTS occurred on the master\. Message: error writing to the binary log, Internal MariaDB error code: 1590/ in mysqld.2.err SELECT * FROM t4 ORDER BY a; a 1 diff --git a/mysql-test/suite/rpl/r/rpl_gtid_errorlog.result b/mysql-test/suite/rpl/r/rpl_gtid_errorlog.result index 204615201d95d..e247ea9c2a714 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_errorlog.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_errorlog.result @@ -38,5 +38,7 @@ a 3 4 5 +FOUND /Slave SQL: Error 'Duplicate entry .* on query\. .*Query: '.*', Gtid 0-1-100, Internal MariaDB error code:|Slave SQL: Could not execute Write_rows.*table test.t1; Duplicate entry.*, Gtid 0-1-100, Internal MariaDB error/ in mysqld.2.err +FOUND /Slave SQL: The incident LOST_EVENTS occurred on the master\. Message: , Internal MariaDB error code: 1590/ in mysqld.2.err DROP TABLE t1; include/rpl_end.inc From 22490a0d709d0c53da94799accb038bf270ed411 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Oct 2016 13:26:43 +0200 Subject: [PATCH 145/295] MDEV-8345 STOP SLAVE should not cause an ERROR to be logged to the error log cherry-pick from 5.7: commit 6b24763 Author: Manish Kumar Date: Tue Mar 27 13:10:42 2012 +0530 BUG#12977988 - ON STOP SLAVE: ERROR READING PACKET FROM SERVER: LOST CONNECTION TO MYSQL SERVER BUG#11761457 - ERROR 2013 + "ERROR READING RELAY LOG EVENT" ON STOP SLAVEBUG#12977988 - ON STOP SLAVE: ERROR READING PACKET FROM SERVER: LOST CONNECTION TO MYSQL SERVER --- .../suite/rpl/r/rpl_stop_slave_error.result | 6 ++++++ .../suite/rpl/t/rpl_stop_slave_error-slave.opt | 1 + .../suite/rpl/t/rpl_stop_slave_error.test | 17 +++++++++++++++++ sql/slave.cc | 9 +++++++-- 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_stop_slave_error.result create mode 100644 mysql-test/suite/rpl/t/rpl_stop_slave_error-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_stop_slave_error.test diff --git a/mysql-test/suite/rpl/r/rpl_stop_slave_error.result b/mysql-test/suite/rpl/r/rpl_stop_slave_error.result new file mode 100644 index 0000000000000..2bd372a9a915b --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_stop_slave_error.result @@ -0,0 +1,6 @@ +include/master-slave.inc +[connection master] +include/stop_slave.inc +NOT FOUND /Error reading packet from server: Lost connection/ in slave_log.err +include/start_slave.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_stop_slave_error-slave.opt b/mysql-test/suite/rpl/t/rpl_stop_slave_error-slave.opt new file mode 100644 index 0000000000000..32c4527a91575 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_stop_slave_error-slave.opt @@ -0,0 +1 @@ +--log-error=$MYSQLTEST_VARDIR/tmp/slave_log.err diff --git a/mysql-test/suite/rpl/t/rpl_stop_slave_error.test b/mysql-test/suite/rpl/t/rpl_stop_slave_error.test new file mode 100644 index 0000000000000..a88981c15c4fd --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_stop_slave_error.test @@ -0,0 +1,17 @@ +# +# MDEV-8345 STOP SLAVE should not cause an ERROR to be logged to the error log +# +source include/have_binlog_format_mixed.inc; # don't repeat the test three times +source include/master-slave.inc; + +connection master; +sync_slave_with_master; +source include/stop_slave.inc; +let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/slave_log.err; +let SEARCH_PATTERN=Error reading packet from server: Lost connection; +let SEARCH_RANGE= -50000; +source include/search_pattern_in_file.inc; + +source include/start_slave.inc; +source include/rpl_end.inc; + diff --git a/sql/slave.cc b/sql/slave.cc index 6dc1a66a2ac90..a124ca6be7e34 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3120,8 +3120,13 @@ static ulong read_event(MYSQL* mysql, Master_info *mi, bool* suppress_warnings) *suppress_warnings= TRUE; } else - sql_print_error("Error reading packet from server: %s ( server_errno=%d)", - mysql_error(mysql), mysql_errno(mysql)); + { + if (!mi->rli.abort_slave) + { + sql_print_error("Error reading packet from server: %s (server_errno=%d)", + mysql_error(mysql), mysql_errno(mysql)); + } + } DBUG_RETURN(packet_error); } From 26b87c332ff78a7aca04930ad86fbf7acc793222 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Thu, 27 Oct 2016 00:04:26 +0400 Subject: [PATCH 146/295] MDEV-10846 Running mysqldump backup twice returns error: Table 'mysql.proc' doesn't exist. The mysql_rm_db() doesn't seem to expect the 'mysql' database to be deleted. Checks for that added. Also fixed the bug MDEV-11105 Table named 'db' has weird side effect. The db.opt file now removed separately. --- mysql-test/r/drop.result | 6 ++++++ mysql-test/t/drop.test | 9 +++++++++ sql/sql_db.cc | 26 +++++++++++++++++++++----- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/drop.result b/mysql-test/r/drop.result index c23ffbe327b63..c25ae9e305517 100644 --- a/mysql-test/r/drop.result +++ b/mysql-test/r/drop.result @@ -209,3 +209,9 @@ INSERT INTO table1 VALUES (1); ERROR 42S02: Unknown table 't.notable' DROP TABLE table1,table2; # End BUG#34750 +# +# MDEV-11105 Table named 'db' has weird side effect. +# +CREATE DATABASE mysqltest; +CREATE TABLE mysqltest.db(id INT); +DROP DATABASE mysqltest; diff --git a/mysql-test/t/drop.test b/mysql-test/t/drop.test index d9784bc819a86..a3e96953bac02 100644 --- a/mysql-test/t/drop.test +++ b/mysql-test/t/drop.test @@ -313,3 +313,12 @@ INSERT INTO table1 VALUES (1); DROP TABLE table1,table2; --echo # End BUG#34750 + +--echo # +--echo # MDEV-11105 Table named 'db' has weird side effect. +--echo # + +CREATE DATABASE mysqltest; +CREATE TABLE mysqltest.db(id INT); +DROP DATABASE mysqltest; + diff --git a/sql/sql_db.cc b/sql/sql_db.cc index e89c3d9e74566..0a3ff64113f04 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -784,7 +784,7 @@ bool mysql_alter_db(THD *thd, const char *db, HA_CREATE_INFO *create_info) bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) { ulong deleted_tables= 0; - bool error= true; + bool error= true, rm_mysql_schema; char path[FN_REFLEN + 16]; MY_DIR *dirp; uint length; @@ -809,6 +809,18 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) length= build_table_filename(path, sizeof(path) - 1, db, "", "", 0); strmov(path+length, MY_DB_OPT_FILE); // Append db option file name del_dbopt(path); // Remove dboption hash entry + /* + Now remove the db.opt file. + The 'find_db_tables_and_rm_known_files' doesn't remove this file + if there exists a table with the name 'db', so let's just do it + separately. We know this file exists and needs to be deleted anyway. + */ + if (my_delete_with_symlink(path, MYF(0)) && my_errno != ENOENT) + { + my_error(EE_DELETE, MYF(0), path, my_errno); + DBUG_RETURN(true); + } + path[length]= '\0'; // Remove file name /* See if the directory exists */ @@ -835,7 +847,8 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) Disable drop of enabled log tables, must be done before name locking. This check is only needed if we are dropping the "mysql" database. */ - if ((my_strcasecmp(system_charset_info, MYSQL_SCHEMA_NAME.str, db) == 0)) + if ((rm_mysql_schema= + (my_strcasecmp(system_charset_info, MYSQL_SCHEMA_NAME.str, db) == 0))) { for (table= tables; table; table= table->next_local) if (check_if_log_table(table, TRUE, "DROP")) @@ -848,7 +861,7 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) lock_db_routines(thd, dbnorm)) goto exit; - if (!in_bootstrap) + if (!in_bootstrap && !rm_mysql_schema) { for (table= tables; table; table= table->next_local) { @@ -893,10 +906,13 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) ha_drop_database(path); tmp_disable_binlog(thd); query_cache_invalidate1(thd, dbnorm); - (void) sp_drop_db_routines(thd, dbnorm); /* @todo Do not ignore errors */ + if (!rm_mysql_schema) + { + (void) sp_drop_db_routines(thd, dbnorm); /* @todo Do not ignore errors */ #ifdef HAVE_EVENT_SCHEDULER - Events::drop_schema_events(thd, dbnorm); + Events::drop_schema_events(thd, dbnorm); #endif + } reenable_binlog(thd); /* From 9d4a0dde0ae3e0d46b4c5c0967c25862d467e94e Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 24 Oct 2016 10:15:11 -0700 Subject: [PATCH 147/295] Fixed bug mdev-11096. 1. When min/max value is provided the null flag for it must be set to 0 in the bitmap Culumn_statistics::column_stat_nulls. 2. When the calculation of the selectivity of the range condition over a column requires min and max values for the column then we have to check that these values are provided. --- mysql-test/r/selectivity.result | 24 ++++++++++++++++++++++ mysql-test/r/selectivity_innodb.result | 28 ++++++++++++++++++++++++-- mysql-test/t/selectivity.test | 19 +++++++++++++++++ sql/sql_statistics.cc | 15 ++++---------- sql/sql_statistics.h | 5 +++++ 5 files changed, 78 insertions(+), 13 deletions(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 620bdc6bd50c2..c2364e11ceb36 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1446,3 +1446,27 @@ a b i set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; DROP TABLE t1,t2; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-11096: range condition over column without statistical data +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=3; +create table t1(col1 char(32)); +insert into t1 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t1 persistent for columns () indexes (); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +explain extended +select * from t1 where col1 > 'b' and col1 < 'e'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`col1` AS `col1` from `test`.`t1` where ((`test`.`t1`.`col1` > 'b') and (`test`.`t1`.`col1` < 'e')) +select * from t1 where col1 > 'b' and col1 < 'e'; +col1 +c +d +drop table t1; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 0acbb465ba852..882f51515b2e8 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -802,9 +802,9 @@ insert into t2 values (2),(3); explain extended select * from t1 where a in ( select b from t2 ) AND ( a > 3 ); id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY t1 ALL NULL NULL NULL NULL 1 0.00 Using where +1 PRIMARY t1 ALL NULL NULL NULL NULL 1 100.00 Using where 1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 100.00 -2 MATERIALIZED t2 ALL NULL NULL NULL NULL 2 0.00 +2 MATERIALIZED t2 ALL NULL NULL NULL NULL 2 100.00 Warnings: Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` semi join (`test`.`t2`) where ((`test`.`t1`.`a` > 3)) select * from t1 where a in ( select b from t2 ) AND ( a > 3 ); @@ -1450,6 +1450,30 @@ a b i set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; DROP TABLE t1,t2; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-11096: range condition over column without statistical data +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=3; +create table t1(col1 char(32)); +insert into t1 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t1 persistent for columns () indexes (); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +explain extended +select * from t1 where col1 > 'b' and col1 < 'e'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`col1` AS `col1` from `test`.`t1` where ((`test`.`t1`.`col1` > 'b') and (`test`.`t1`.`col1` < 'e')) +select * from t1 where col1 > 'b' and col1 < 'e'; +col1 +c +d +drop table t1; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; set @tmp_oucs= @@optimizer_use_condition_selectivity; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index c46ff69295fa7..1321046009e2d 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -970,6 +970,25 @@ set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivit DROP TABLE t1,t2; +set use_stat_tables=@save_use_stat_tables; + +--echo # +--echo # Bug mdev-11096: range condition over column without statistical data +--echo # + +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=3; +create table t1(col1 char(32)); +insert into t1 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t1 persistent for columns () indexes (); + +explain extended +select * from t1 where col1 > 'b' and col1 < 'e'; +select * from t1 where col1 > 'b' and col1 < 'e'; + +drop table t1; + +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 47a5a40ebeb44..70080a6b4f162 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -1003,11 +1003,13 @@ class Column_stat: public Stat_table switch (i) { case COLUMN_STAT_MIN_VALUE: + table_field->read_stats->min_value->set_notnull(); stat_field->val_str(&val); table_field->read_stats->min_value->store(val.ptr(), val.length(), &my_charset_bin); break; case COLUMN_STAT_MAX_VALUE: + table_field->read_stats->max_value->set_notnull(); stat_field->val_str(&val); table_field->read_stats->max_value->store(val.ptr(), val.length(), &my_charset_bin); @@ -3659,17 +3661,8 @@ double get_column_range_cardinality(Field *field, { double avg_frequency= col_stats->get_avg_frequency(); res= avg_frequency; - /* - psergey-todo: what does check for min_value, max_value mean? - min/max_value are set to NULL in alloc_statistics_for_table() and - alloc_statistics_for_table_share(). Both functions will immediately - call create_min_max_statistical_fields_for_table and - create_min_max_statistical_fields_for_table_share() respectively, - which will set min/max_value to be valid pointers, unless OOM - occurs. - */ if (avg_frequency > 1.0 + 0.000001 && - col_stats->min_value && col_stats->max_value) + col_stats->min_max_values_are_provided()) { Histogram *hist= &col_stats->histogram; if (hist->is_available()) @@ -3692,7 +3685,7 @@ double get_column_range_cardinality(Field *field, } else { - if (col_stats->min_value && col_stats->max_value) + if (col_stats->min_max_values_are_provided()) { double sel, min_mp_pos, max_mp_pos; diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 46e5cef22d1ac..8e5f8107849a7 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -388,6 +388,11 @@ class Column_statistics avg_frequency= (ulong) (val * Scale_factor_avg_frequency); } + bool min_max_values_are_provided() + { + return !is_null(COLUMN_STAT_MIN_VALUE) && + !is_null(COLUMN_STAT_MIN_VALUE); + } }; From d451d772fdaa554eeb96ae12f96c3a32a6fd4d66 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 26 Oct 2016 10:59:38 -0700 Subject: [PATCH 148/295] Fixed bug mdev-9628. In the function create_key_parts_for_pseudo_indexes() the key part structures of pseudo-indexes created for BLOB fields were set incorrectly. Also the key parts for long fields must be 'truncated' up to the maximum length acceptable for key parts. --- mysql-test/r/selectivity.result | 47 ++++++++++++++ mysql-test/r/selectivity_innodb.result | 85 ++++++++++++++++++++++++++ mysql-test/t/selectivity.test | 33 ++++++++++ mysql-test/t/selectivity_innodb.test | 25 ++++++++ sql/opt_range.cc | 9 ++- 5 files changed, 198 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index c2364e11ceb36..8fb5cd17c518a 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1470,3 +1470,50 @@ d drop table t1; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-9628: unindexed blob column without min-max statistics +# with optimizer_use_condition_selectivity=3 +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=3; +create table t1(col1 char(32)); +insert into t1 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +create table t2(col1 text); +insert into t2 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t2; +Table Op Msg_type Msg_text +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status OK +select * from t1 where col1 > 'b' and col1 < 'd'; +col1 +c +explain extended +select * from t1 where col1 > 'b' and col1 < 'd'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 28.57 Using where +Warnings: +Note 1003 select `test`.`t1`.`col1` AS `col1` from `test`.`t1` where ((`test`.`t1`.`col1` > 'b') and (`test`.`t1`.`col1` < 'd')) +select * from t2 where col1 > 'b' and col1 < 'd'; +col1 +c +explain extended +select * from t2 where col1 > 'b' and col1 < 'd'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 8 100.00 Using where +Warnings: +Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where ((`test`.`t2`.`col1` > 'b') and (`test`.`t2`.`col1` < 'd')) +select * from t2 where col1 < 'b' and col1 > 'd'; +col1 +explain extended +select * from t2 where col1 < 'b' and col1 > 'd'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +Warnings: +Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where 0 +drop table t1,t2; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 882f51515b2e8..3d15131dbb55e 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1474,6 +1474,53 @@ d drop table t1; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-9628: unindexed blob column without min-max statistics +# with optimizer_use_condition_selectivity=3 +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=3; +create table t1(col1 char(32)); +insert into t1 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +create table t2(col1 text); +insert into t2 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t2; +Table Op Msg_type Msg_text +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status OK +select * from t1 where col1 > 'b' and col1 < 'd'; +col1 +c +explain extended +select * from t1 where col1 > 'b' and col1 < 'd'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 8 28.57 Using where +Warnings: +Note 1003 select `test`.`t1`.`col1` AS `col1` from `test`.`t1` where ((`test`.`t1`.`col1` > 'b') and (`test`.`t1`.`col1` < 'd')) +select * from t2 where col1 > 'b' and col1 < 'd'; +col1 +c +explain extended +select * from t2 where col1 > 'b' and col1 < 'd'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 8 100.00 Using where +Warnings: +Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where ((`test`.`t2`.`col1` > 'b') and (`test`.`t2`.`col1` < 'd')) +select * from t2 where col1 < 'b' and col1 > 'd'; +col1 +explain extended +select * from t2 where col1 < 'b' and col1 > 'd'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +Warnings: +Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where 0 +drop table t1,t2; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; set @tmp_oucs= @@optimizer_use_condition_selectivity; @@ -1560,6 +1607,44 @@ where t1.child_user_id=t3.id and t1.child_group_id is null and t2.lower_group_na parent_id child_group_id child_user_id id lower_group_name directory_id id drop table t1,t2,t3; # +# MDEV-9187: duplicate of bug mdev-9628 +# +set use_stat_tables = preferably; +set optimizer_use_condition_selectivity=3; +CREATE TABLE t1 (f1 char(32)) ENGINE=InnoDB; +INSERT INTO t1 VALUES ('foo'),('bar'),('qux'); +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM t1 WHERE f1 < 'm'; +f1 +foo +bar +EXPLAIN EXTENDED +SELECT * FROM t1 WHERE f1 < 'm'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 72.09 Using where +Warnings: +Note 1003 select `test`.`t1`.`f1` AS `f1` from `test`.`t1` where (`test`.`t1`.`f1` < 'm') +CREATE TABLE t2 (f1 TEXT) ENGINE=InnoDB; +INSERT INTO t2 VALUES ('foo'),('bar'),('qux'); +ANALYZE TABLE t2; +Table Op Msg_type Msg_text +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status OK +SELECT * FROM t2 WHERE f1 <> 'qux'; +f1 +foo +bar +EXPLAIN EXTENDED +SELECT * FROM t2 WHERE f1 <> 'qux'; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 3 100.00 Using where +Warnings: +Note 1003 select `test`.`t2`.`f1` AS `f1` from `test`.`t2` where (`test`.`t2`.`f1` <> 'qux') +DROP TABLE t1,t2; +# # End of 10.0 tests # set use_stat_tables= @tmp_ust; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 1321046009e2d..8efc5216ba09c 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -992,3 +992,36 @@ drop table t1; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +--echo # +--echo # Bug mdev-9628: unindexed blob column without min-max statistics +--echo # with optimizer_use_condition_selectivity=3 +--echo # + +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=3; + +create table t1(col1 char(32)); +insert into t1 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t1; + +create table t2(col1 text); +insert into t2 values ('a'),('b'),('c'),('d'), ('e'),('f'),('g'),('h'); +analyze table t2; + +select * from t1 where col1 > 'b' and col1 < 'd'; +explain extended +select * from t1 where col1 > 'b' and col1 < 'd'; + +select * from t2 where col1 > 'b' and col1 < 'd'; +explain extended +select * from t2 where col1 > 'b' and col1 < 'd'; + +select * from t2 where col1 < 'b' and col1 > 'd'; +explain extended +select * from t2 where col1 < 'b' and col1 > 'd'; + +drop table t1,t2; + +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; + diff --git a/mysql-test/t/selectivity_innodb.test b/mysql-test/t/selectivity_innodb.test index d6a77eac60004..25aa0abbc3b45 100644 --- a/mysql-test/t/selectivity_innodb.test +++ b/mysql-test/t/selectivity_innodb.test @@ -109,6 +109,31 @@ where t1.child_user_id=t3.id and t1.child_group_id is null and t2.lower_group_na drop table t1,t2,t3; +--echo # +--echo # MDEV-9187: duplicate of bug mdev-9628 +--echo # + +set use_stat_tables = preferably; +set optimizer_use_condition_selectivity=3; + +CREATE TABLE t1 (f1 char(32)) ENGINE=InnoDB; +INSERT INTO t1 VALUES ('foo'),('bar'),('qux'); +ANALYZE TABLE t1; + +SELECT * FROM t1 WHERE f1 < 'm'; +EXPLAIN EXTENDED +SELECT * FROM t1 WHERE f1 < 'm'; + +CREATE TABLE t2 (f1 TEXT) ENGINE=InnoDB; +INSERT INTO t2 VALUES ('foo'),('bar'),('qux'); +ANALYZE TABLE t2; + +SELECT * FROM t2 WHERE f1 <> 'qux'; +EXPLAIN EXTENDED +SELECT * FROM t2 WHERE f1 <> 'qux'; + +DROP TABLE t1,t2; + --echo # --echo # End of 10.0 tests --echo # diff --git a/sql/opt_range.cc b/sql/opt_range.cc index e0ca43e6d726f..5d6891a1edf53 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -3345,9 +3345,16 @@ bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param, { Field *field= *field_ptr; uint16 store_length; + uint16 max_key_part_length= (uint16) table->file->max_key_part_length(); key_part->key= keys; key_part->part= 0; - key_part->length= (uint16) field->key_length(); + if (field->flags & BLOB_FLAG) + key_part->length= max_key_part_length; + else + { + key_part->length= (uint16) field->key_length(); + set_if_smaller(key_part->length, max_key_part_length); + } store_length= key_part->length; if (field->real_maybe_null()) store_length+= HA_KEY_NULL_LENGTH; From a0795655ab8d1cbcd88a155ba72ebf93864f82dc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 27 Oct 2016 12:23:31 +0200 Subject: [PATCH 149/295] MDEV-10846 Running mysqldump backup twice returns error: Table 'mysql.proc' doesn't exist. Update test results after 26b87c3 --- mysql-test/r/mysqldump.result | 3 --- 1 file changed, 3 deletions(-) diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index b6de51c8b03f2..cb3c28f42cdf2 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -5236,9 +5236,6 @@ SET @@global.log_output="TABLE"; SET @@global.general_log='OFF'; SET @@global.slow_query_log='OFF'; DROP DATABASE mysql; -Warnings: -Error 1146 Table 'mysql.proc' doesn't exist -Error 1146 Table 'mysql.event' doesn't exist SHOW CREATE TABLE mysql.general_log; Table Create Table general_log CREATE TABLE `general_log` ( From eca8c324e9a02f530853580991b11b587f54b24a Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 27 Oct 2016 19:07:55 +0200 Subject: [PATCH 150/295] Typo fixed. --- sql/item_subselect.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 5cdfa427997bc..e70922bb5d3ca 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -2620,8 +2620,8 @@ static bool check_equality_for_exist2in(Item_func *func, args[0]->all_used_tables() == OUTER_REF_TABLE_BIT) { /* It is Item_field or Item_direct_view_ref) */ - DBUG_ASSERT(args[0]->type() == Item::FIELD_ITEM || - args[0]->type() == Item::REF_ITEM); + DBUG_ASSERT(args[1]->type() == Item::FIELD_ITEM || + args[1]->type() == Item::REF_ITEM); *local_field= (Item_ident *)args[1]; *outer_exp= args[0]; return TRUE; From aec43216c834c2156e9aca7b7dc182fbdd1916de Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 26 Oct 2016 21:38:58 +0000 Subject: [PATCH 151/295] MDEV-9409 Windows - workaround VS2015 CRT bug that makes mysqldump/mysql_install_db.exe fail The bug is described in https://connect.microsoft.com/VisualStudio/Feedback/Details/1902345 When reading from a pipe in text mode, using CRT function such as fread(), some newlines may be lost. Workaround is to use binary mode on reading side and if necessary, replace \r\n with \n. --- client/mysqltest.cc | 28 ++++++++++++++++++++++------ sql/mysqld.cc | 6 ++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index acb9e8b1e0c06..5daa0e72270a1 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -1705,11 +1705,11 @@ int cat_file(DYNAMIC_STRING* ds, const char* filename) while((len= my_read(fd, (uchar*)&buff, sizeof(buff)-1, MYF(0))) > 0) { - char *p= buff, *start= buff; - while (p < buff+len) + char *p= buff, *start= buff,*end=buff+len; + while (p < end) { /* Convert cr/lf to lf */ - if (*p == '\r' && *(p+1) && *(p+1)== '\n') + if (*p == '\r' && p+1 < end && *(p+1)== '\n') { /* Add fake newline instead of cr and output the line */ *p= '\n'; @@ -3367,16 +3367,32 @@ void do_exec(struct st_command *command) ds_result= &ds_sorted; } +#ifdef _WIN32 + /* Workaround for CRT bug, MDEV-9409 */ + _setmode(fileno(res_file), O_BINARY); +#endif + while (fgets(buf, sizeof(buf), res_file)) { + int len = (int)strlen(buf); +#ifdef _WIN32 + /* Strip '\r' off newlines. */ + if (len > 1 && buf[len-2] == '\r' && buf[len-1] == '\n') + { + buf[len-2] = '\n'; + buf[len-1] = 0; + len--; + } +#endif if (disable_result_log) { - buf[strlen(buf)-1]=0; + if (len) + buf[len-1] = 0; DBUG_PRINT("exec_result",("%s", buf)); } else { - replace_dynstr_append(ds_result, buf); + replace_dynstr_append_mem(ds_result, buf, len); } } error= pclose(res_file); @@ -5200,7 +5216,7 @@ typedef struct static st_error global_error_names[] = { - { "", -1U, "" }, + { "", ~0U, "" }, #include { 0, 0, 0 } }; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index be9e21d6746d8..8eb92cafc03e3 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4949,6 +4949,12 @@ int mysqld_main(int argc, char **argv) setbuf(stderr, NULL); FreeConsole(); // Remove window } + + if (fileno(stdin) >= 0) + { + /* Disable CRLF translation (MDEV-9409). */ + _setmode(fileno(stdin), O_BINARY); + } #endif /* From d8cb6822bc7deecba26021368c2f08fa6bf59e58 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 26 Oct 2016 21:54:41 +0000 Subject: [PATCH 152/295] VS2015 build fixes - new location of signtool - silence a nonsensical warning from stl header (complain about noexcept() function attribute, if /EHsc is not set) --- cmake/install_macros.cmake | 1 + cmake/os/Windows.cmake | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake index 15e2dc4930bea..22a525d734423 100644 --- a/cmake/install_macros.cmake +++ b/cmake/install_macros.cmake @@ -208,6 +208,7 @@ IF(WIN32) FIND_PROGRAM(SIGNTOOL_EXECUTABLE signtool PATHS "$ENV{ProgramFiles}/Microsoft SDKs/Windows/v7.0A/bin" "$ENV{ProgramFiles}/Windows Kits/8.0/bin/x86" + "$ENV{ProgramFiles}/Windows Kits/8.1/bin/x86" ) IF(NOT SIGNTOOL_EXECUTABLE) MESSAGE(FATAL_ERROR diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index e1055ab58944d..283f79b3b4116 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -120,7 +120,7 @@ IF(MSVC) #TODO: update the code and remove the disabled warnings SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /we4099") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099") IF(CMAKE_SIZEOF_VOID_P MATCHES 8) # _WIN64 is defined by the compiler itself. From 2a2e79b702b28149c8f9984d24367fe3ad7afa13 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 27 Oct 2016 13:03:49 +0000 Subject: [PATCH 153/295] MDEV-11157 Windows - Upgrade installer to use HeidiSQL 9.4 --- win/packaging/heidisql.cmake | 2 +- win/packaging/heidisql.wxi.in | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/win/packaging/heidisql.cmake b/win/packaging/heidisql.cmake index e2636eb4af8c6..772834e7c7d81 100644 --- a/win/packaging/heidisql.cmake +++ b/win/packaging/heidisql.cmake @@ -1,4 +1,4 @@ -SET(HEIDISQL_BASE_NAME "HeidiSQL_9.3_Portable") +SET(HEIDISQL_BASE_NAME "HeidiSQL_9.4_Portable") SET(HEIDISQL_ZIP "${HEIDISQL_BASE_NAME}.zip") SET(HEIDISQL_URL "http://www.heidisql.com/downloads/releases/${HEIDISQL_ZIP}") SET(HEIDISQL_DOWNLOAD_DIR ${THIRD_PARTY_DOWNLOAD_LOCATION}/${HEIDISQL_BASE_NAME}) diff --git a/win/packaging/heidisql.wxi.in b/win/packaging/heidisql.wxi.in index 4f07a07627ccc..1d5c4430cd24f 100644 --- a/win/packaging/heidisql.wxi.in +++ b/win/packaging/heidisql.wxi.in @@ -45,14 +45,20 @@ - - + + + + + + + + @@ -70,8 +76,10 @@ - + + + From cb7b03b00d9b19dbb3ca00947c6e7637917c4e5a Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Fri, 28 Oct 2016 13:34:13 +0400 Subject: [PATCH 154/295] MDEV-11164 - hardening-wrapper has been removed from Debian Sid The hardening-wrapper package has been removed from Debian Sid. Remove dependency and relevant settings. --- debian/control | 1 - debian/dist/Debian/rules | 1 - debian/dist/Ubuntu/rules | 1 - 3 files changed, 3 deletions(-) diff --git a/debian/control b/debian/control index 93f715f2345ad..6bad46542d677 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,6 @@ Build-Depends: bison, debhelper, dpatch, gawk, - hardening-wrapper, libjemalloc-dev (>= 3.0.0), libncurses5-dev (>= 5.0-6), libpam0g-dev, diff --git a/debian/dist/Debian/rules b/debian/dist/Debian/rules index 2122a3c815455..8366d997979cd 100755 --- a/debian/dist/Debian/rules +++ b/debian/dist/Debian/rules @@ -1,7 +1,6 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -export DEB_BUILD_HARDENING=1 PACKAGE=mariadb-10.0 diff --git a/debian/dist/Ubuntu/rules b/debian/dist/Ubuntu/rules index 8d993124b56c8..6f34d2eac5cfd 100755 --- a/debian/dist/Ubuntu/rules +++ b/debian/dist/Ubuntu/rules @@ -1,7 +1,6 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -export DEB_BUILD_HARDENING=1 PACKAGE=mariadb-10.0 From 425d341b980e412d5083056da7b017fbc8bb02e2 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Fri, 28 Oct 2016 11:46:15 -0400 Subject: [PATCH 155/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a82a4e4d77d54..36834bdb92df1 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=28 +MYSQL_VERSION_PATCH=29 From 7ed5563bbee301bf8217080dc78ea6a3e78e23a8 Mon Sep 17 00:00:00 2001 From: SachinSetiya Date: Wed, 26 Oct 2016 14:52:24 +0530 Subject: [PATCH 156/295] MDEV-11016 wsrep_node_is_ready() check is too strict Problem:- The condition that checks for node readiness is too strict as it does not allow SELECTs even if these selects do not access any tables. For example,if we run SELECT 1; OR SELECT @@max_allowed_packet; Solution:- We need not to report this error when all_tables(lex->query_tables) is NULL: --- .../suite/galera/r/galera_var_dirty_reads.result | 13 +++++++++++++ .../suite/galera/t/galera_var_dirty_reads.test | 7 +++++++ sql/sql_parse.cc | 5 ++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6d703c8cf95a7..6a2aa1eb5e71e 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,6 +18,19 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +SET @@session.wsrep_dirty_reads=OFF; +SELECT 2; +2 +2 +SELECT @@max_allowed_packet; +@@max_allowed_packet +4194304 +SELECT 2+2 from DUAL; +2+2 +4 +SELECT sysdate() from DUAL; +sysdate() +2016-10-28 23:13:06 SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index dfd8d5ecf2905..0c81779ca6555 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -36,6 +36,13 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; +#Select query which does not access table should be allowed MDEV-11016 +SET @@session.wsrep_dirty_reads=OFF; +SELECT 2; +SELECT @@max_allowed_packet; +SELECT 2+2 from DUAL; +SELECT sysdate() from DUAL; + --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index a15f3da8e1278..41c57dca2efee 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2650,12 +2650,15 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. We however, allow SET, - SHOW and SELECT queries (only if wsrep_dirty_reads is set). + SHOW and SELECT queries (only if wsrep_dirty_reads is set or when it + does not access any table). */ if (lex->sql_command != SQLCOM_SET_OPTION && !wsrep_is_show_query(lex->sql_command) && !(thd->variables.wsrep_dirty_reads && lex->sql_command == SQLCOM_SELECT) && + !(lex->sql_command == SQLCOM_SELECT && + !all_tables) && !wsrep_node_is_ready(thd)) goto error; } From c1bbedbd4a20a8bd9354296244b6ec6263b29c6b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 28 Oct 2016 20:37:18 +0200 Subject: [PATCH 157/295] AWS key Management plugin - add plugin variable for the region AWS C++ SDK always defaults region to us-east-1 for clientConfiguration (ignoring config file or env.variable) This patch introduces a plugin variable 'region' to make it usable for master keys created in regions other than 'us-east-1'. --- plugin/aws_key_management/aws_key_management_plugin.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugin/aws_key_management/aws_key_management_plugin.cc b/plugin/aws_key_management/aws_key_management_plugin.cc index 20a795eb437ad..e94c551bebe65 100644 --- a/plugin/aws_key_management/aws_key_management_plugin.cc +++ b/plugin/aws_key_management/aws_key_management_plugin.cc @@ -77,6 +77,7 @@ static const char *key_spec_names[]={ "AES_128", "AES_256", 0 }; /* Plugin variables */ static char* master_key_id; +static char* region; static unsigned long key_spec; static unsigned long log_level; static int rotate_key; @@ -163,6 +164,10 @@ static int plugin_init(void *p) InitializeAWSLogging(Aws::MakeShared("aws_key_management_plugin", (Aws::Utils::Logging::LogLevel) log_level)); Aws::Client::ClientConfiguration clientConfiguration; + if (region && region[0]) + { + clientConfiguration.region = region; + } if (request_timeout) { clientConfiguration.requestTimeoutMs= request_timeout; @@ -585,6 +590,10 @@ static MYSQL_SYSVAR_INT(request_timeout, request_timeout, "Timeout in milliseconds for create HTTPS connection or execute AWS request. Specify 0 to use SDK default.", NULL, NULL, 0, 0, INT_MAX, 1); +static MYSQL_SYSVAR_STR(region, region, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "AWS region. For example us-east-1, or eu-central-1. If no value provided, SDK default is used.", + NULL, NULL, ""); static struct st_mysql_sys_var* settings[]= { MYSQL_SYSVAR(master_key_id), @@ -592,6 +601,7 @@ static struct st_mysql_sys_var* settings[]= { MYSQL_SYSVAR(rotate_key), MYSQL_SYSVAR(log_level), MYSQL_SYSVAR(request_timeout), + MYSQL_SYSVAR(region), NULL }; From bc323727de312b32e80ae9590e2346414746a594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 27 Oct 2016 08:18:14 +0300 Subject: [PATCH 158/295] MDEV-10977: [ERROR] InnoDB: Block in space_id 0 in file ibdata1 encrypted. MDEV-10394: Innodb system table space corrupted Analysis: After we have read the page in buf_page_io_complete try to find if the page is encrypted or corrupted. Encryption was determined by reading FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION field from FIL-header as a key_version. However, this field is not always zero even when encryption is not used. Thus, incorrect key_version could lead situation where decryption is tried to page that is not encrypted. Fix: We still read key_version information from FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION field but also check if tablespace has encryption information before trying encrypt the page. --- storage/innobase/buf/buf0buf.cc | 78 ++++++++- storage/xtradb/buf/buf0buf.cc | 269 ++++++++++++++++++++------------ 2 files changed, 240 insertions(+), 107 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index b3eaf4bf5a926..9491309806396 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -526,6 +526,14 @@ buf_page_is_checksum_valid_crc32( { ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); +#ifdef UNIV_DEBUG_LEVEL2 + if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", + checksum_field1, checksum_field2, crc32); + } +#endif + return(checksum_field1 == crc32 && checksum_field2 == crc32); } @@ -553,6 +561,13 @@ buf_page_is_checksum_valid_innodb( if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { +#ifdef UNIV_DEBUG_LEVEL2 + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + mach_read_from_4(read_buf + FIL_PAGE_LSN) + ); +#endif return(false); } @@ -563,6 +578,13 @@ buf_page_is_checksum_valid_innodb( if (checksum_field1 != 0 && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { +#ifdef UNIV_DEBUG_LEVEL2 + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + mach_read_from_4(read_buf + FIL_PAGE_LSN) + ); +#endif return(false); } @@ -581,6 +603,16 @@ buf_page_is_checksum_valid_none( ulint checksum_field1, ulint checksum_field2) { +#ifdef UNIV_DEBUG_LEVEL2 + if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + mach_read_from_4(read_buf + FIL_PAGE_LSN) + ); + } +#endif + return(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } @@ -598,9 +630,21 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; + ulint space_id = mach_read_from_4( + read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + bool page_encrypted = false; + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + if (crypt_data && + crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && + fil_page_is_encrypted(read_buf)) { + page_encrypted = true; + } if (!page_encrypted && !zip_size && memcmp(read_buf + FIL_PAGE_LSN + 4, @@ -610,6 +654,11 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ + ib_logf(IB_LOG_LEVEL_INFO, + "Log sequence number at the start %lu and the end %lu do not match.", + mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), + mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); + return(TRUE); } @@ -676,6 +725,9 @@ buf_page_is_corrupted( /* make sure that the page is really empty */ for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) { if (read_buf[i] != 0) { + ib_logf(IB_LOG_LEVEL_INFO, + "Checksum fields zero but page is not empty."); + return(TRUE); } } @@ -686,7 +738,7 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4(read_buf + FIL_PAGE_SPACE_ID); + const srv_checksum_algorithm_t curr_algo = static_cast(srv_checksum_algorithm); @@ -4531,7 +4583,7 @@ buf_page_io_complete( if (io_type == BUF_IO_READ) { ulint read_page_no; ulint read_space_id; - byte* frame; + byte* frame = NULL; if (!buf_page_decrypt_after_read(bpage)) { /* encryption error! */ @@ -4540,7 +4592,8 @@ buf_page_io_complete( } else { frame = ((buf_block_t*) bpage)->frame; } - goto corrupt; + + goto database_corrupted; } if (buf_page_get_zip_size(bpage)) { @@ -4551,7 +4604,7 @@ buf_page_io_complete( FALSE)) { buf_pool->n_pend_unzip--; - goto corrupt; + goto database_corrupted; } buf_pool->n_pend_unzip--; } else { @@ -4614,7 +4667,7 @@ buf_page_io_complete( } goto page_not_corrupt; ;); -corrupt: +database_corrupted: bool corrupted = buf_page_check_corrupt(bpage); if (corrupted) { @@ -6177,6 +6230,19 @@ buf_page_decrypt_after_read( bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; + ulint space_id = mach_read_from_4( + dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + if (!crypt_data || + (crypt_data && + crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && + key_version != 0)) { + key_version = 0; + } /* If page is encrypted read post-encryption checksum */ if (!page_compressed_encrypted && key_version != 0) { diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index a81d21a04b582..e77225a3bd201 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -595,6 +595,14 @@ buf_page_is_checksum_valid_crc32( { ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); +#ifdef UNIV_DEBUG_LEVEL2 + if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", + checksum_field1, checksum_field2, crc32); + } +#endif + return(checksum_field1 == crc32 && checksum_field2 == crc32); } @@ -622,6 +630,13 @@ buf_page_is_checksum_valid_innodb( if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { +#ifdef UNIV_DEBUG_LEVEL2 + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + mach_read_from_4(read_buf + FIL_PAGE_LSN) + ); +#endif return(false); } @@ -632,6 +647,13 @@ buf_page_is_checksum_valid_innodb( if (checksum_field1 != 0 && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { +#ifdef UNIV_DEBUG_LEVEL2 + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + mach_read_from_4(read_buf + FIL_PAGE_LSN) + ); +#endif return(false); } @@ -650,6 +672,16 @@ buf_page_is_checksum_valid_none( ulint checksum_field1, ulint checksum_field2) { +#ifdef UNIV_DEBUG_LEVEL2 + if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { + ib_logf(IB_LOG_LEVEL_INFO, + "Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + mach_read_from_4(read_buf + FIL_PAGE_LSN) + ); + } +#endif + return(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } @@ -667,9 +699,21 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; + ulint space_id = mach_read_from_4( + read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + bool page_encrypted = false; + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + if (crypt_data && + crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && + fil_page_is_encrypted(read_buf)) { + page_encrypted = true; + } if (!page_encrypted && !zip_size && memcmp(read_buf + FIL_PAGE_LSN + 4, @@ -679,6 +723,11 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ + ib_logf(IB_LOG_LEVEL_INFO, + "Log sequence number at the start %lu and the end %lu do not match.", + mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), + mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); + return(TRUE); } @@ -745,6 +794,9 @@ buf_page_is_corrupted( /* make sure that the page is really empty */ for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) { if (read_buf[i] != 0) { + ib_logf(IB_LOG_LEVEL_INFO, + "Checksum fields zero but page is not empty."); + return(TRUE); } } @@ -755,7 +807,7 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4(read_buf + FIL_PAGE_SPACE_ID); + const srv_checksum_algorithm_t curr_algo = static_cast(srv_checksum_algorithm); @@ -4645,7 +4697,7 @@ buf_page_io_complete( if (io_type == BUF_IO_READ) { ulint read_page_no; ulint read_space_id; - byte* frame; + byte* frame = NULL; if (!buf_page_decrypt_after_read(bpage)) { /* encryption error! */ @@ -4654,7 +4706,8 @@ buf_page_io_complete( } else { frame = ((buf_block_t*) bpage)->frame; } - goto corrupt; + + goto database_corrupted; } if (buf_page_get_zip_size(bpage)) { @@ -4666,7 +4719,8 @@ buf_page_io_complete( os_atomic_decrement_ulint( &buf_pool->n_pend_unzip, 1); - goto corrupt; + + goto database_corrupted; } os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1); } else { @@ -4714,119 +4768,119 @@ buf_page_io_complete( if (UNIV_LIKELY(!bpage->is_corrupt || !srv_pass_corrupt_table)) { - /* From version 3.23.38 up we store the page checksum - to the 4 first bytes of the page end lsn field */ - - if (buf_page_is_corrupted(true, frame, - buf_page_get_zip_size(bpage))) { - - /* Not a real corruption if it was triggered by - error injection */ - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - ib_logf(IB_LOG_LEVEL_INFO, - "Simulated page corruption"); - return(true); - } - goto page_not_corrupt; - ;); -corrupt: + /* From version 3.23.38 up we store the page checksum + to the 4 first bytes of the page end lsn field */ + + if (buf_page_is_corrupted(true, frame, + buf_page_get_zip_size(bpage))) { + + /* Not a real corruption if it was triggered by + error injection */ + DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", + if (bpage->space > TRX_SYS_SPACE + && buf_mark_space_corrupt(bpage)) { + ib_logf(IB_LOG_LEVEL_INFO, + "Simulated page corruption"); + return(true); + } + goto page_not_corrupt; + ;); +database_corrupted: - bool corrupted = buf_page_check_corrupt(bpage); + bool corrupted = buf_page_check_corrupt(bpage); - if (corrupted) { - fil_system_enter(); - space = fil_space_get_by_id(bpage->space); - fil_system_exit(); - ib_logf(IB_LOG_LEVEL_ERROR, - "Database page corruption on disk" - " or a failed"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Space %u file %s read of page %u.", - bpage->space, - space ? space->name : "NULL", - bpage->offset); - ib_logf(IB_LOG_LEVEL_ERROR, - "You may have to recover" - " from a backup."); + if (corrupted) { + fil_system_enter(); + space = fil_space_get_by_id(bpage->space); + fil_system_exit(); + ib_logf(IB_LOG_LEVEL_ERROR, + "Database page corruption on disk" + " or a failed"); + ib_logf(IB_LOG_LEVEL_ERROR, + "Space %u file %s read of page %u.", + bpage->space, + space ? space->name : "NULL", + bpage->offset); + ib_logf(IB_LOG_LEVEL_ERROR, + "You may have to recover" + " from a backup."); - buf_page_print(frame, buf_page_get_zip_size(bpage), - BUF_PAGE_PRINT_NO_CRASH); + buf_page_print(frame, buf_page_get_zip_size(bpage), + BUF_PAGE_PRINT_NO_CRASH); - ib_logf(IB_LOG_LEVEL_ERROR, - "It is also possible that your operating" - "system has corrupted its own file cache."); - ib_logf(IB_LOG_LEVEL_ERROR, - "and rebooting your computer removes the error."); - ib_logf(IB_LOG_LEVEL_ERROR, - "If the corrupt page is an index page you can also try to"); - ib_logf(IB_LOG_LEVEL_ERROR, - "fix the corruption by dumping, dropping, and reimporting"); - ib_logf(IB_LOG_LEVEL_ERROR, - "the corrupt table. You can use CHECK"); - ib_logf(IB_LOG_LEVEL_ERROR, - "TABLE to scan your table for corruption."); - ib_logf(IB_LOG_LEVEL_ERROR, - "See also " - REFMAN "forcing-innodb-recovery.html" - " about forcing recovery."); - } - - if (srv_pass_corrupt_table && bpage->space != 0 - && bpage->space < SRV_LOG_SPACE_FIRST_ID) { - trx_t* trx; - - fprintf(stderr, - "InnoDB: space %u will be treated as corrupt.\n", - bpage->space); - fil_space_set_corrupt(bpage->space); - - trx = innobase_get_trx(); - if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) { - dict_table_set_corrupt_by_space(bpage->space, FALSE); - } else { - dict_table_set_corrupt_by_space(bpage->space, TRUE); + ib_logf(IB_LOG_LEVEL_ERROR, + "It is also possible that your operating" + "system has corrupted its own file cache."); + ib_logf(IB_LOG_LEVEL_ERROR, + "and rebooting your computer removes the error."); + ib_logf(IB_LOG_LEVEL_ERROR, + "If the corrupt page is an index page you can also try to"); + ib_logf(IB_LOG_LEVEL_ERROR, + "fix the corruption by dumping, dropping, and reimporting"); + ib_logf(IB_LOG_LEVEL_ERROR, + "the corrupt table. You can use CHECK"); + ib_logf(IB_LOG_LEVEL_ERROR, + "TABLE to scan your table for corruption."); + ib_logf(IB_LOG_LEVEL_ERROR, + "See also " + REFMAN "forcing-innodb-recovery.html" + " about forcing recovery."); } - bpage->is_corrupt = TRUE; - } - if (srv_force_recovery < SRV_FORCE_IGNORE_CORRUPT) { - /* If page space id is larger than TRX_SYS_SPACE - (0), we will attempt to mark the corresponding - table as corrupted instead of crashing server */ - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - return(false); - } else { - corrupted = buf_page_check_corrupt(bpage); + if (srv_pass_corrupt_table && bpage->space != 0 + && bpage->space < SRV_LOG_SPACE_FIRST_ID) { + trx_t* trx; - if (corrupted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Ending processing because of a corrupt database page."); + fprintf(stderr, + "InnoDB: space %u will be treated as corrupt.\n", + bpage->space); + fil_space_set_corrupt(bpage->space); - ut_error; + trx = innobase_get_trx(); + if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) { + dict_table_set_corrupt_by_space(bpage->space, FALSE); + } else { + dict_table_set_corrupt_by_space(bpage->space, TRUE); } + bpage->is_corrupt = TRUE; + } - ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, - "Table in tablespace %lu encrypted." - "However key management plugin or used key_id %u is not found or" - " used encryption algorithm or method does not match." - " Can't continue opening the table.", - (ulint)bpage->space, bpage->key_version); + if (srv_force_recovery < SRV_FORCE_IGNORE_CORRUPT) { + /* If page space id is larger than TRX_SYS_SPACE + (0), we will attempt to mark the corresponding + table as corrupted instead of crashing server */ + if (bpage->space > TRX_SYS_SPACE + && buf_mark_space_corrupt(bpage)) { + return(false); + } else { + corrupted = buf_page_check_corrupt(bpage); - if (bpage->space > TRX_SYS_SPACE) { if (corrupted) { - buf_mark_space_corrupt(bpage); + ib_logf(IB_LOG_LEVEL_ERROR, + "Ending processing because of a corrupt database page."); + + ut_error; } - } else { - ut_error; + + ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, + "Table in tablespace %lu encrypted." + "However key management plugin or used key_id %u is not found or" + " used encryption algorithm or method does not match." + " Can't continue opening the table.", + (ulint)bpage->space, bpage->key_version); + + if (bpage->space > TRX_SYS_SPACE) { + if (corrupted) { + buf_mark_space_corrupt(bpage); + } + } else { + ut_error; + } + return(false); } - return(false); } } - } } /**/ DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", @@ -6374,6 +6428,19 @@ buf_page_decrypt_after_read( bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; + ulint space_id = mach_read_from_4( + dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + if (!crypt_data || + (crypt_data && + crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && + key_version != 0)) { + key_version = 0; + } /* If page is encrypted read post-encryption checksum */ if (!page_compressed_encrypted && key_version != 0) { From 885577fb10cba63a4a140bd91257a3cd2b402159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 27 Oct 2016 14:51:10 +0300 Subject: [PATCH 159/295] MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing Two problems: (1) When pushing warning to sql-layer we need to check that thd != NULL to avoid NULL-pointer reference. (2) At tablespace key rotation if used key_id is not found from encryption plugin tablespace should not be rotated. --- mysql-test/std_data/keys2.txt | 1 + .../encryption/r/innodb-missing-key.result | 54 +++++++++++++++ .../suite/encryption/t/innodb-missing-key.opt | 5 ++ .../encryption/t/innodb-missing-key.test | 68 +++++++++++++++++++ storage/innobase/buf/buf0buf.cc | 18 +++-- storage/innobase/fil/fil0crypt.cc | 6 ++ storage/innobase/handler/ha_innodb.cc | 50 +++++++++----- storage/xtradb/buf/buf0buf.cc | 11 +-- storage/xtradb/fil/fil0crypt.cc | 6 ++ storage/xtradb/handler/ha_innodb.cc | 36 +++++----- 10 files changed, 211 insertions(+), 44 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-missing-key.result create mode 100644 mysql-test/suite/encryption/t/innodb-missing-key.opt create mode 100644 mysql-test/suite/encryption/t/innodb-missing-key.test diff --git a/mysql-test/std_data/keys2.txt b/mysql-test/std_data/keys2.txt index aa1600b894da9..5b98fbeebd282 100644 --- a/mysql-test/std_data/keys2.txt +++ b/mysql-test/std_data/keys2.txt @@ -4,3 +4,4 @@ 4;205379930183490D3BECA139BDF4DB5B 5;E2D944D5D837A1DCB22FF7FD397892EE 6;BAFE99B0BB87F2CD33A6AF26A11F6BD1 +19;678D6B0063824BACCE33224B385104B35F30FF5749F0EBC030A0955DBC7FAC34 diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result new file mode 100644 index 0000000000000..b59ec158273c9 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -0,0 +1,54 @@ +call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); +call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); + +# Start server with keys2.txt +CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; +CREATE TABLE t2(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1; +CREATE TABLE t3(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=NO; +INSERT INTO t1(b) VALUES ('thisissecredmessage'); +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t2 SELECT * FROM t1; +INSERT INTO t3 SELECT * FROM t1; + +# Restart server with keys3.txt +set global innodb_encryption_rotate_key_age = 1; +use test; +CREATE TABLE t4(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1; +SELECT SLEEP(5); +SLEEP(5) +0 +SELECT COUNT(1) FROM t3; +COUNT(1) +2048 +SELECT COUNT(1) FROM t2; +COUNT(1) +2048 +SELECT COUNT(1) FROM t2,t1 where t2.a = t1.a; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +SELECT COUNT(1) FROM t1 where b = 'ab'; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +SELECT COUNT(1) FROM t1; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB + +# Start server with keys2.txt +SELECT COUNT(1) FROM t1; +COUNT(1) +2048 +SELECT COUNT(1) FROM t2; +COUNT(1) +2048 +SELECT COUNT(1) FROM t3; +COUNT(1) +2048 +DROP TABLE t1, t2, t3; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.opt b/mysql-test/suite/encryption/t/innodb-missing-key.opt new file mode 100644 index 0000000000000..02691695cbd9e --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-missing-key.opt @@ -0,0 +1,5 @@ +--innodb-encrypt-tables +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption + diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test new file mode 100644 index 0000000000000..8fcfb7661171f --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -0,0 +1,68 @@ +--source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# embedded does not support restart +-- source include/not_embedded.inc +-- source include/not_valgrind.inc +# Avoid CrashReporter popup on Mac +-- source include/not_crashrep.inc + +# +# MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing +# +call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); +call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); + +--echo +--echo # Start server with keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; +CREATE TABLE t2(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1; +CREATE TABLE t3(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=NO; +INSERT INTO t1(b) VALUES ('thisissecredmessage'); +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t1(b) SELECT b FROM t1; +INSERT INTO t2 SELECT * FROM t1; +INSERT INTO t3 SELECT * FROM t1; + +--echo +--echo # Restart server with keys3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +-- source include/restart_mysqld.inc + +set global innodb_encryption_rotate_key_age = 1; +use test; +CREATE TABLE t4(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1; +SELECT SLEEP(5); +SELECT COUNT(1) FROM t3; +SELECT COUNT(1) FROM t2; +--error 1296 +SELECT COUNT(1) FROM t2,t1 where t2.a = t1.a; +--error 1296 +SELECT COUNT(1) FROM t1 where b = 'ab'; +--error 1296 +SELECT COUNT(1) FROM t1; + +--echo +--echo # Start server with keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +SELECT COUNT(1) FROM t1; +SELECT COUNT(1) FROM t2; +SELECT COUNT(1) FROM t3; + +DROP TABLE t1, t2, t3; + + diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 9491309806396..9d098c1caf661 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -317,6 +317,9 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) +/* prototypes for new functions added to ha_innodb.cc */ +trx_t* innobase_get_trx(); + /********************************************************************//** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted @@ -4485,7 +4488,6 @@ buf_page_check_corrupt( ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - unsigned key_version = bpage->key_version; bool page_compressed = bpage->page_encrypted; ulint stored_checksum = bpage->stored_checksum; ulint calculated_checksum = bpage->stored_checksum; @@ -4495,6 +4497,7 @@ buf_page_check_corrupt( fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_t* space = fil_space_found_by_id(space_id); bool corrupted = true; + ulint key_version = bpage->key_version; if (key_version != 0 || page_compressed_encrypted) { bpage->encrypted = true; @@ -4524,7 +4527,7 @@ buf_page_check_corrupt( stored_checksum, calculated_checksum); } ib_logf(IB_LOG_LEVEL_ERROR, - "Reason could be that key_version %u in page " + "Reason could be that key_version %lu in page " "or in crypt_data %p could not be found.", key_version, crypt_data); ib_logf(IB_LOG_LEVEL_ERROR, @@ -4538,7 +4541,7 @@ buf_page_check_corrupt( "Block in space_id %lu in file %s encrypted.", space_id, space ? space->name : "NULL"); ib_logf(IB_LOG_LEVEL_ERROR, - "However key management plugin or used key_id %u is not found or" + "However key management plugin or used key_id %lu is not found or" " used encryption algorithm or method does not match.", key_version); ib_logf(IB_LOG_LEVEL_ERROR, @@ -4718,6 +4721,7 @@ buf_page_io_complete( return(false); } else { corrupted = buf_page_check_corrupt(bpage); + ulint key_version = bpage->key_version; if (corrupted) { ib_logf(IB_LOG_LEVEL_ERROR, @@ -4726,12 +4730,12 @@ buf_page_io_complete( ut_error; } - ib_push_warning((void *)NULL, DB_DECRYPTION_FAILED, + ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, "Table in tablespace %lu encrypted." - "However key management plugin or used key_id %u is not found or" + "However key management plugin or used key_id %lu is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.", - (ulint)bpage->space, bpage->key_version); + (ulint)bpage->space, key_version); if (bpage->space > TRX_SYS_SPACE) { if (corrupted) { @@ -4890,7 +4894,7 @@ buf_all_freed_instance( const buf_block_t* block = buf_chunk_not_freed(chunk); if (UNIV_LIKELY_NULL(block)) { - if (block->page.key_version == 0) { + if (block->page.key_version == 0) { fil_space_t* space = fil_space_get(block->page.space); ib_logf(IB_LOG_LEVEL_ERROR, "Page %u %u still fixed or dirty.", diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index bedaf7cfe4967..72e3454cbb54e 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1351,6 +1351,12 @@ fil_crypt_space_needs_rotation( } } + /* If used key_id is not found from encryption plugin we can't + continue to rotate the tablespace */ + if (fil_crypt_get_latest_key_version(crypt_data) == ENCRYPTION_KEY_VERSION_INVALID) { + return false; + } + mutex_enter(&crypt_data->mutex); do { diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index f18ce89fb4c7f..d790bf4c4b69c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2601,6 +2601,20 @@ check_trx_exists( return(trx); } +/************************************************************************* +Gets current trx. */ +trx_t* +innobase_get_trx() +{ + THD *thd=current_thd; + if (likely(thd != 0)) { + trx_t*& trx = thd_to_trx(thd); + return(trx); + } else { + return(NULL); + } +} + /*********************************************************************//** Note that a transaction has been registered with MySQL. @return true if transaction is registered with MySQL 2PC coordinator */ @@ -20664,15 +20678,17 @@ ib_push_warning( char *buf; #define MAX_BUF_SIZE 4*1024 - va_start(args, format); - buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); - vsprintf(buf,format, args); + if (thd) { + va_start(args, format); + buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); + vsprintf(buf,format, args); - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - convert_error_code_to_mysql((dberr_t)error, 0, thd), - buf); - my_free(buf); - va_end(args); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + convert_error_code_to_mysql((dberr_t)error, 0, thd), + buf); + my_free(buf); + va_end(args); + } } /********************************************************************//** @@ -20694,15 +20710,17 @@ ib_push_warning( thd = current_thd; } - va_start(args, format); - buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); - vsprintf(buf,format, args); + if (thd) { + va_start(args, format); + buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); + vsprintf(buf,format, args); - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - convert_error_code_to_mysql((dberr_t)error, 0, thd), - buf); - my_free(buf); - va_end(args); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + convert_error_code_to_mysql((dberr_t)error, 0, thd), + buf); + my_free(buf); + va_end(args); + } } /********************************************************************//** diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index e77225a3bd201..f5ac842f4e873 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -4601,7 +4601,6 @@ buf_page_check_corrupt( ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - unsigned key_version = bpage->key_version; bool page_compressed = bpage->page_encrypted; ulint stored_checksum = bpage->stored_checksum; ulint calculated_checksum = bpage->stored_checksum; @@ -4611,6 +4610,7 @@ buf_page_check_corrupt( fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_t* space = fil_space_found_by_id(space_id); bool corrupted = true; + ulint key_version = bpage->key_version; if (key_version != 0 || page_compressed_encrypted) { bpage->encrypted = true; @@ -4640,7 +4640,7 @@ buf_page_check_corrupt( stored_checksum, calculated_checksum); } ib_logf(IB_LOG_LEVEL_ERROR, - "Reason could be that key_version %u in page " + "Reason could be that key_version %lu in page " "or in crypt_data %p could not be found.", key_version, crypt_data); ib_logf(IB_LOG_LEVEL_ERROR, @@ -4654,7 +4654,7 @@ buf_page_check_corrupt( "Block in space_id %lu in file %s encrypted.", space_id, space ? space->name : "NULL"); ib_logf(IB_LOG_LEVEL_ERROR, - "However key management plugin or used key_id %u is not found or" + "However key management plugin or used key_id %lu is not found or" " used encryption algorithm or method does not match.", key_version); ib_logf(IB_LOG_LEVEL_ERROR, @@ -4855,6 +4855,7 @@ buf_page_io_complete( return(false); } else { corrupted = buf_page_check_corrupt(bpage); + ulint key_version = bpage->key_version; if (corrupted) { ib_logf(IB_LOG_LEVEL_ERROR, @@ -4865,10 +4866,10 @@ buf_page_io_complete( ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, "Table in tablespace %lu encrypted." - "However key management plugin or used key_id %u is not found or" + "However key management plugin or used key_id %lu is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.", - (ulint)bpage->space, bpage->key_version); + (ulint)bpage->space, key_version); if (bpage->space > TRX_SYS_SPACE) { if (corrupted) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index bedaf7cfe4967..72e3454cbb54e 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -1351,6 +1351,12 @@ fil_crypt_space_needs_rotation( } } + /* If used key_id is not found from encryption plugin we can't + continue to rotate the tablespace */ + if (fil_crypt_get_latest_key_version(crypt_data) == ENCRYPTION_KEY_VERSION_INVALID) { + return false; + } + mutex_enter(&crypt_data->mutex); do { diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 85bc897efc2aa..bd76d524e2520 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -22245,15 +22245,17 @@ ib_push_warning( char *buf; #define MAX_BUF_SIZE 4*1024 - va_start(args, format); - buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); - vsprintf(buf,format, args); + if (thd) { + va_start(args, format); + buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); + vsprintf(buf,format, args); - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - convert_error_code_to_mysql((dberr_t)error, 0, thd), - buf); - my_free(buf); - va_end(args); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + convert_error_code_to_mysql((dberr_t)error, 0, thd), + buf); + my_free(buf); + va_end(args); + } } /********************************************************************//** @@ -22275,15 +22277,17 @@ ib_push_warning( thd = current_thd; } - va_start(args, format); - buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); - vsprintf(buf,format, args); + if (thd) { + va_start(args, format); + buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); + vsprintf(buf,format, args); - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - convert_error_code_to_mysql((dberr_t)error, 0, thd), - buf); - my_free(buf); - va_end(args); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + convert_error_code_to_mysql((dberr_t)error, 0, thd), + buf); + my_free(buf); + va_end(args); + } } /********************************************************************//** From 84ce681969c3cdec095a45517164275989f95477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 27 Oct 2016 15:01:15 +0300 Subject: [PATCH 160/295] MDEV-10917: Warning suggesting that innodb_page_size is experimental may be inaccurate Removed experimental from message. --- storage/innobase/handler/ha_innodb.cc | 11 ++++++----- storage/xtradb/handler/ha_innodb.cc | 12 +++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d790bf4c4b69c..79d8ae0dc81a6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3444,15 +3444,16 @@ innobase_init( } if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_DEF) { - fprintf(stderr, - "InnoDB: Warning: innodb_page_size has been " - "changed from default value %d to %ldd. (###EXPERIMENTAL### " - "operation)\n", UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); + ib_logf(IB_LOG_LEVEL_INFO, + "innodb_page_size has been " + "changed from default value %d to %ldd.", + UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); /* There is hang on buffer pool when trying to get a new page if buffer pool size is too small for large page sizes */ if (innobase_buffer_pool_size < (24 * 1024 * 1024)) { - fprintf(stderr, "InnoDB: Error: innobase_page_size %lu requires " + ib_logf(IB_LOG_LEVEL_INFO, + "innobase_page_size %lu requires " "innodb_buffer_pool_size > 24M current %lld", UNIV_PAGE_SIZE, innobase_buffer_pool_size); goto error; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index bd76d524e2520..0223e58edac86 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3859,17 +3859,19 @@ innobase_init( } if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_DEF) { - fprintf(stderr, - "InnoDB: Warning: innodb_page_size has been " - "changed from default value %d to %ldd. (###EXPERIMENTAL### " - "operation)\n", UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); + ib_logf(IB_LOG_LEVEL_INFO, + "innodb_page_size has been " + "changed from default value %d to %ldd.", + UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); /* There is hang on buffer pool when trying to get a new page if buffer pool size is too small for large page sizes */ if (innobase_buffer_pool_size < (24 * 1024 * 1024)) { - fprintf(stderr, "InnoDB: Error: innobase_page_size %lu requires " + ib_logf(IB_LOG_LEVEL_INFO, + "innobase_page_size %lu requires " "innodb_buffer_pool_size > 24M current %lld", UNIV_PAGE_SIZE, innobase_buffer_pool_size); + goto error; } } From de0f77a2a87e0d77df9cd2b079f51e3142db1e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 28 Oct 2016 09:27:03 +0300 Subject: [PATCH 161/295] MDEV-11106: Improve error messages when importing tablespaces Add error message when used index_id in index page is not found from configuration file. --- storage/innobase/row/row0import.cc | 9 +++++++++ storage/xtradb/row/row0import.cc | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 6dd3c4eea9448..3a47e3fc6aaba 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1896,6 +1896,15 @@ PageConverter::update_index_page( row_index_t* index = find_index(id); if (index == 0) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Page for tablespace %lu is " + " index page with id %lu but that" + " index is not found from configuration file." + " Current index name %s and id %lu.", + m_space, + id, + m_index->m_name, + m_index->m_id); m_index = 0; return(DB_CORRUPTION); } diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index 4c7cb7d33b532..d45ce907304eb 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1899,6 +1899,15 @@ PageConverter::update_index_page( row_index_t* index = find_index(id); if (index == 0) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Page for tablespace %lu is " + " index page with id %lu but that" + " index is not found from configuration file." + " Current index name %s and id %lu.", + m_space, + id, + m_index->m_name, + m_index->m_id); m_index = 0; return(DB_CORRUPTION); } From bb4b8c74702b8e8c6eea14da9a913f6d7cc61302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 28 Oct 2016 13:59:35 +0300 Subject: [PATCH 162/295] MDEV-9099: Test encryption.innodb_encryption_discard_import fails on buildbot --- .../suite/encryption/include/innodb-util.pl | 126 ++++++++++++++++++ .../r/innodb_encryption_discard_import.result | 43 ++---- .../t/innodb_encryption_discard_import.test | 93 +++---------- 3 files changed, 152 insertions(+), 110 deletions(-) create mode 100644 mysql-test/suite/encryption/include/innodb-util.pl diff --git a/mysql-test/suite/encryption/include/innodb-util.pl b/mysql-test/suite/encryption/include/innodb-util.pl new file mode 100644 index 0000000000000..241545dac185f --- /dev/null +++ b/mysql-test/suite/encryption/include/innodb-util.pl @@ -0,0 +1,126 @@ +# +# Utility functions to copy files for WL#5522 +# +# All the tables must be in the same database, you can call it like so: +# ib_backup_tablespaces("test", "t1", "blah", ...). + +use File::Copy; +use File::Spec; + +sub ib_normalize_path { + my ($path) = @_; +} + +sub ib_backup_tablespace { + my ($db, $table) = @_; + my $datadir = $ENV{'MYSQLD_DATADIR'}; + my $cfg_file = sprintf("%s.cfg", $table); + my $ibd_file = sprintf("%s.ibd", $table); + my $tmpd = $ENV{'MYSQLTEST_VARDIR'} . "/tmp"; + + my @args = (File::Spec->catfile($datadir, $db, $ibd_file), + File::Spec->catfile($tmpd, $ibd_file)); + + copy(@args) or die "copy @args failed: $!"; + + my @args = (File::Spec->catfile($datadir, $db, $cfg_file), + File::Spec->catfile($tmpd, $cfg_file)); + + copy(@args) or die "copy @args failed: $!"; +} + +sub ib_cleanup { + my ($db, $table) = @_; + my $datadir = $ENV{'MYSQLD_DATADIR'}; + my $cfg_file = sprintf("%s.cfg", $table); + + print "unlink: $cfg_file\n"; + + # These may or may not exist + unlink(File::Spec->catfile($datadir, $db, $cfg_file)); +} + +sub ib_unlink_tablespace { + my ($db, $table) = @_; + my $datadir = $ENV{'MYSQLD_DATADIR'}; + my $ibd_file = sprintf("%s.ibd", $table); + + print "unlink: $ibd_file\n"; + # This may or may not exist + unlink(File::Spec->catfile($datadir, $db, $ibd_file)); + + ib_cleanup($db, $table); +} + +sub ib_backup_tablespaces { + my ($db, @tables) = @_; + + foreach my $table (@tables) { + print "backup: $table\n"; + ib_backup_tablespace($db, $table); + } +} + +sub ib_discard_tablespace { } + +sub ib_discard_tablespaces { } + +sub ib_restore_cfg_file { + my ($tmpd, $datadir, $db, $table) = @_; + my $cfg_file = sprintf("%s.cfg", $table); + + my @args = (File::Spec->catfile($tmpd, $cfg_file), + File::Spec->catfile($datadir, "$db", $cfg_file)); + + copy(@args) or die "copy @args failed: $!"; +} + +sub ib_restore_ibd_file { + my ($tmpd, $datadir, $db, $table) = @_; + my $ibd_file = sprintf("%s.ibd", $table); + + my @args = (File::Spec->catfile($tmpd, $ibd_file), + File::Spec->catfile($datadir, $db, $ibd_file)); + + copy(@args) or die "copy @args failed: $!"; +} + +sub ib_restore_tablespace { + my ($db, $table) = @_; + my $datadir = $ENV{'MYSQLD_DATADIR'}; + my $tmpd = $ENV{'MYSQLTEST_VARDIR'} . "/tmp"; + + ib_restore_cfg_file($tmpd, $datadir, $db, $table); + ib_restore_ibd_file($tmpd, $datadir, $db, $table); +} + +sub ib_restore_tablespaces { + my ($db, @tables) = @_; + + foreach my $table (@tables) { + print "restore: $table .ibd and .cfg files\n"; + ib_restore_tablespace($db, $table); + } +} + +sub ib_restore_cfg_files { + my ($db, @tables) = @_; + my $datadir = $ENV{'MYSQLD_DATADIR'}; + my $tmpd = $ENV{'MYSQLTEST_VARDIR'} . "/tmp"; + + foreach my $table (@tables) { + print "restore: $table .cfg file\n"; + ib_restore_cfg_file($tmpd, $datadir, $db, $table); + } +} + +sub ib_restore_ibd_files { + my ($db, @tables) = @_; + my $datadir = $ENV{'MYSQLD_DATADIR'}; + my $tmpd = $ENV{'MYSQLTEST_VARDIR'} . "/tmp"; + + foreach my $table (@tables) { + print "restore: $table .ibd file\n"; + ib_restore_ibd_file($tmpd, $datadir, $db, $table); + } +} diff --git a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result index f8c59b7bcc304..40284dbf71d72 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result @@ -28,62 +28,41 @@ NOT FOUND /foobar/ in t1.ibd NOT FOUND /temp/ in t2.ibd # t3 ... on expecting NOT FOUND NOT FOUND /barfoo/ in t3.ibd -FLUSH TABLE t1, t2, t3 FOR EXPORT; -# List before copying files -t1.cfg t1.frm t1.ibd -t2.cfg t2.frm t2.ibd -t3.cfg t3.frm t3.ibd -UNLOCK TABLES; -# Restarting server -# Done restarting server -# List before t1 DISCARD +FLUSH TABLES t1, t2, t3 FOR EXPORT; +backup: t1 +backup: t2 +backup: t3 +t1.cfg t1.frm t1.ibd +t2.cfg t2.frm t2.ibd +t3.cfg t3.frm t3.ibd -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; +UNLOCK TABLES; ALTER TABLE t1 DISCARD TABLESPACE; ALTER TABLE t2 DISCARD TABLESPACE; ALTER TABLE t3 DISCARD TABLESPACE; -# List after t1 DISCARD -t1.frm -t2.frm -t3.frm -# Restarting server -# Done restarting server -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; -# Tablespaces should be still encrypted -# t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd -# t2 ... on expecting NOT FOUND -NOT FOUND /temp/ in t2.ibd -# t3 ... on expecting NOT FOUND -NOT FOUND /barfoo/ in t3.ibd +restore: t1 .ibd and .cfg files +restore: t2 .ibd and .cfg files +restore: t3 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't1' SELECT COUNT(1) FROM t1; COUNT(1) 10000 ALTER TABLE t2 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't2' SELECT COUNT(1) FROM t2; COUNT(1) 10000 ALTER TABLE t3 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't3' SELECT COUNT(1) FROM t3; COUNT(1) 10000 diff --git a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test index def3665eeff0e..a3789ea7ca718 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test @@ -2,12 +2,12 @@ -- source include/have_example_key_management_plugin.inc -- source include/not_valgrind.inc -- source include/not_embedded.inc --- source include/not_windows.inc call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded."); ---let $MYSQLD_TMPDIR = `SELECT @@tmpdir` ---let $MYSQLD_DATADIR = `SELECT @@datadir` +let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; +let $MYSQLD_DATADIR = `SELECT @@datadir`; + --let SEARCH_RANGE = 10000000 --let $id = `SELECT RAND()` --let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd @@ -66,88 +66,25 @@ set autocommit=1; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc -FLUSH TABLE t1, t2, t3 FOR EXPORT; - ---echo # List before copying files +let MYSQLD_DATADIR =`SELECT @@datadir`; --list_files $MYSQLD_DATADIR/test ---disable_result_log ---error 0,1,2 ---remove_file $MYSQLD_TMPDIR/t1.cfg ---error 0,1,2 ---remove_file $MYSQLD_TMPDIR/t1.ibd ---error 0,1,2 ---remove_file $MYSQLD_TMPDIR/t2.cfg ---error 0,1,2 ---remove_file $MYSQLD_TMPDIR/t2.ibd ---error 0,1,2 ---remove_file $MYSQLD_TMPDIR/t3.cfg ---error 0,1,2 ---remove_file $MYSQLD_TMPDIR/t3.ibd ---enable_result_log ---copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1$id.cfg ---copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1$id.ibd ---copy_file $MYSQLD_DATADIR/test/t2.cfg $MYSQLD_TMPDIR/t2$id.cfg ---copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_TMPDIR/t2$id.ibd ---copy_file $MYSQLD_DATADIR/test/t3.cfg $MYSQLD_TMPDIR/t3$id.cfg ---copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_TMPDIR/t3$id.ibd -UNLOCK TABLES; - ---echo # Restarting server --- source include/restart_mysqld.inc ---echo # Done restarting server ---echo # List before t1 DISCARD +FLUSH TABLES t1, t2, t3 FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1","t2","t3"); +EOF --list_files $MYSQLD_DATADIR/test - -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; +UNLOCK TABLES; ALTER TABLE t1 DISCARD TABLESPACE; ALTER TABLE t2 DISCARD TABLESPACE; ALTER TABLE t3 DISCARD TABLESPACE; ---echo # List after t1 DISCARD ---list_files $MYSQLD_DATADIR/test ---error 0,1,2 ---remove_file $MYSQLD_DATADIR/test/t1.cfg ---error 0,1,2 ---remove_file $MYSQLD_DATADIR/test/t1.ibd ---error 0,1,2 ---remove_file $MYSQLD_DATADIR/test/t2.cfg ---error 0,1,2 ---remove_file $MYSQLD_DATADIR/test/t2.ibd ---error 0,1,2 ---remove_file $MYSQLD_DATADIR/test/t3.cfg ---error 0,1,2 ---remove_file $MYSQLD_DATADIR/test/t3.ibd ---enable_result_log ---echo # Restarting server --- source include/restart_mysqld.inc ---echo # Done restarting server - -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; - ---copy_file $MYSQLD_TMPDIR/t1$id.cfg $MYSQLD_DATADIR/test/t1.cfg ---copy_file $MYSQLD_TMPDIR/t1$id.ibd $MYSQLD_DATADIR/test/t1.ibd ---copy_file $MYSQLD_TMPDIR/t2$id.cfg $MYSQLD_DATADIR/test/t2.cfg ---copy_file $MYSQLD_TMPDIR/t2$id.ibd $MYSQLD_DATADIR/test/t2.ibd ---copy_file $MYSQLD_TMPDIR/t3$id.cfg $MYSQLD_DATADIR/test/t3.cfg ---copy_file $MYSQLD_TMPDIR/t3$id.ibd $MYSQLD_DATADIR/test/t3.ibd - ---sleep 5 ---echo # Tablespaces should be still encrypted ---let SEARCH_PATTERN=foobar ---echo # t1 yes on expecting NOT FOUND --- let SEARCH_FILE=$t1_IBD --- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp ---echo # t2 ... on expecting NOT FOUND --- let SEARCH_FILE=$t2_IBD --- source include/search_pattern_in_file.inc ---echo # t3 ... on expecting NOT FOUND ---let SEARCH_PATTERN=barfoo --- let SEARCH_FILE=$t3_IBD --- source include/search_pattern_in_file.inc +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1","t2","t3"); +ib_restore_tablespaces("test", "t1","t2","t3"); +EOF ALTER TABLE t1 IMPORT TABLESPACE; SELECT COUNT(1) FROM t1; From 58b5c40b09b5134913d33752c88cce3003550b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 29 Oct 2016 12:57:48 +0300 Subject: [PATCH 163/295] Remove accidentally added directory. --- libmariadb | 1 - 1 file changed, 1 deletion(-) delete mode 160000 libmariadb diff --git a/libmariadb b/libmariadb deleted file mode 160000 index c8dd0899d484a..0000000000000 --- a/libmariadb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c8dd0899d484ad698ec2da5bc8e3d19ff8b623b9 From 67e6cfd086e41d8918ae0a6b3077a84880819bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sun, 30 Oct 2016 09:16:02 +0200 Subject: [PATCH 164/295] Add suppression for new InnoDB error log error as this test intentionally produces this error. --- mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result | 5 +++++ mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test | 1 + 2 files changed, 6 insertions(+) diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index 0e863f5849e1b..53bc22e3d3c49 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("InnoDB: Page for tablespace .* "); SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@innodb_file_per_table @@ -575,6 +576,10 @@ set global innodb_monitor_enable = default; set global innodb_monitor_disable = default; set global innodb_monitor_reset = default; set global innodb_monitor_reset_all = default; +Warnings: +Error 145 Table './mtr/test_suppressions' is marked as crashed and should be repaired +Error 1194 Table 'test_suppressions' is marked as crashed and should be repaired +Error 1034 1 client is using or hasn't closed the table properly SET GLOBAL INNODB_FILE_PER_TABLE=1; SET GLOBAL INNODB_FILE_FORMAT=Antelope; SET SESSION innodb_strict_mode=0; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index 4b03ac008d29d..701255d36ec19 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -17,6 +17,7 @@ # allow test to run only when innodb-page-size=16 --source include/have_innodb_16k.inc +call mtr.add_suppression("InnoDB: Page for tablespace .* "); let MYSQLD_DATADIR =`SELECT @@datadir`; let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; From 9aa73153db765166712dd45e07da7f274c216a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 31 Oct 2016 08:48:00 +0200 Subject: [PATCH 165/295] MDEV-11183: innodb.innodb-wl5522-debug fails in buildbot and outside --- mysql-test/suite/innodb/r/innodb-wl5522-debug.result | 2 ++ mysql-test/suite/innodb/t/innodb-wl5522-debug.test | 3 +++ 2 files changed, 5 insertions(+) diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result index 0c914ebc7a64d..ee8bd22c58ff4 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result @@ -1,3 +1,5 @@ +call mtr.add_suppression("InnoDB: Page for tablespace .* "); +FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@innodb_file_per_table diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test index 9c0c11fadab9d..e8a037ab2f157 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test @@ -15,6 +15,9 @@ -- source include/have_innodb.inc +call mtr.add_suppression("InnoDB: Page for tablespace .* "); +FLUSH TABLES; + let MYSQLD_DATADIR =`SELECT @@datadir`; let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; let $pathfix=/: '.*test_wl5522.*t1.ibd'/: 'test_wl5522\\t1.ibd'/; From cb5685a87239d35ff4173192cbc345605bdb1e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 31 Oct 2016 08:49:36 +0200 Subject: [PATCH 166/295] MDEV-11184: innodb.innodb-wl5522-debug-zip fails in buildbot on Windows --- mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result | 5 +---- mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index 53bc22e3d3c49..6231a87580499 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,4 +1,5 @@ call mtr.add_suppression("InnoDB: Page for tablespace .* "); +FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@innodb_file_per_table @@ -576,10 +577,6 @@ set global innodb_monitor_enable = default; set global innodb_monitor_disable = default; set global innodb_monitor_reset = default; set global innodb_monitor_reset_all = default; -Warnings: -Error 145 Table './mtr/test_suppressions' is marked as crashed and should be repaired -Error 1194 Table 'test_suppressions' is marked as crashed and should be repaired -Error 1034 1 client is using or hasn't closed the table properly SET GLOBAL INNODB_FILE_PER_TABLE=1; SET GLOBAL INNODB_FILE_FORMAT=Antelope; SET SESSION innodb_strict_mode=0; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index 701255d36ec19..80aa8f558fd25 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -18,6 +18,7 @@ --source include/have_innodb_16k.inc call mtr.add_suppression("InnoDB: Page for tablespace .* "); +FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; From 923a7f8675e262b45ee63bffb617477ca5b5011d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 31 Oct 2016 12:16:53 +0200 Subject: [PATCH 167/295] MDEV-11188: rpl.rpl_parallel_partition fails with valgrind warnings in buildbot and outside --- storage/xtradb/handler/ha_innodb.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 35ca8791c9994..e95fc365be8d7 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -11248,6 +11248,12 @@ ha_innobase::create( zip_dict_ids = static_cast( mem_heap_alloc(heap, form->s->fields * sizeof(ulint))); + /* This is currently required for valgrind because MariaDB does + not currently support compressed columns. */ + for (size_t field_idx = 0; field_idx < form->s->fields; ++field_idx) { + zip_dict_ids[field_idx] = ULINT_UNDEFINED; + } + const char* err_zip_dict_name = 0; if (!innobase_check_zip_dicts(form, zip_dict_ids, trx, &err_zip_dict_name)) { From 554c60ab0d383e8e6d473294fb86a9db0b0d2b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 31 Oct 2016 12:44:06 +0200 Subject: [PATCH 168/295] MDEV-11182: InnoDB: Assertion failure in file buf0buf.cc line 4730 (encryption.create_or_replace fails in buildbot and outside) Analysis: Problem is that page is encrypted but encryption information on page 0 has already being changed. Fix: If page header contains key_version != 0 and even if based on current encryption information tablespace is not encrypted we need to check is page corrupted. If it is not, then we know that page is not encrypted. If page is corrupted, we need to try to decrypt it and then compare the stored and calculated checksums to see is page corrupted or not. --- storage/innobase/buf/buf0buf.cc | 46 +++++++++++++++++++++++++++++---- storage/xtradb/buf/buf0buf.cc | 39 ++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 9d098c1caf661..7eca60417a1fd 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -68,6 +68,11 @@ Created 11/5/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif +/* Enable this for checksum error messages. */ +//#ifdef UNIV_DEBUG +//#define UNIV_DEBUG_LEVEL2 1 +//#endif + /* IMPLEMENTATION OF THE BUFFER POOL ================================= @@ -533,7 +538,7 @@ buf_page_is_checksum_valid_crc32( if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { ib_logf(IB_LOG_LEVEL_INFO, "Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", - checksum_field1, checksum_field2, crc32); + checksum_field1, checksum_field2, (ulint)crc32); } #endif @@ -584,7 +589,7 @@ buf_page_is_checksum_valid_innodb( #ifdef UNIV_DEBUG_LEVEL2 ib_logf(IB_LOG_LEVEL_INFO, "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", - checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), mach_read_from_4(read_buf + FIL_PAGE_LSN) ); #endif @@ -610,7 +615,7 @@ buf_page_is_checksum_valid_none( if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { ib_logf(IB_LOG_LEVEL_INFO, "Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", - checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC, mach_read_from_4(read_buf + FIL_PAGE_LSN) ); } @@ -707,6 +712,7 @@ buf_page_is_corrupted( if (zip_size) { return(!page_zip_verify_checksum(read_buf, zip_size)); } + if (page_encrypted) { return (FALSE); } @@ -1970,6 +1976,11 @@ buf_pool_watch_set( buf_pool->watch[]. However, it is not in the critical code path as this function will be called only by the purge thread. */ +/* Enable this for checksum error messages. Currently on by +default on UNIV_DEBUG for encryption bugs. */ +#ifdef UNIV_DEBUG +#define UNIV_DEBUG_LEVEL2 1 +#endif /* To obey latching order first release the hash_lock. */ rw_lock_x_unlock(hash_lock); @@ -4490,7 +4501,7 @@ buf_page_check_corrupt( ((buf_block_t*) bpage)->frame; bool page_compressed = bpage->page_encrypted; ulint stored_checksum = bpage->stored_checksum; - ulint calculated_checksum = bpage->stored_checksum; + ulint calculated_checksum = bpage->calculated_checksum; bool page_compressed_encrypted = bpage->page_compressed; ulint space_id = mach_read_from_4( dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -4596,6 +4607,10 @@ buf_page_io_complete( frame = ((buf_block_t*) bpage)->frame; } + ib_logf(IB_LOG_LEVEL_INFO, + "Page %u in tablespace %u encryption error key_version %u.", + bpage->offset, bpage->space, bpage->key_version); + goto database_corrupted; } @@ -4607,6 +4622,11 @@ buf_page_io_complete( FALSE)) { buf_pool->n_pend_unzip--; + + ib_logf(IB_LOG_LEVEL_INFO, + "Page %u in tablespace %u zip_decompress failure.", + bpage->offset, bpage->space); + goto database_corrupted; } buf_pool->n_pend_unzip--; @@ -6245,7 +6265,23 @@ buf_page_decrypt_after_read( (crypt_data && crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && key_version != 0)) { - key_version = 0; + byte* frame = NULL; + + if (buf_page_get_zip_size(bpage)) { + frame = bpage->zip.data; + } else { + frame = ((buf_block_t*) bpage)->frame; + } + + /* If page is not corrupted at this point, page can't be + encrypted, thus set key_version to 0. If page is corrupted, + we assume at this point that it is encrypted as page + contained key_version != 0. Note that page could still be + really corrupted. This we will find out after decrypt by + checking page checksums. */ + if (!buf_page_is_corrupted(false, frame, buf_page_get_zip_size(bpage))) { + key_version = 0; + } } /* If page is encrypted read post-encryption checksum */ diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index f5ac842f4e873..0d159fa496b28 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -65,6 +65,10 @@ Created 11/5/1995 Heikki Tuuri #include "fil0pagecompress.h" #include "ha_prototypes.h" +/* Enable this for checksum error messages. */ +//#ifdef UNIV_DEBUG +//#define UNIV_DEBUG_LEVEL2 1 +//#endif /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -599,7 +603,7 @@ buf_page_is_checksum_valid_crc32( if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { ib_logf(IB_LOG_LEVEL_INFO, "Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", - checksum_field1, checksum_field2, crc32); + checksum_field1, checksum_field2, (ulint)crc32); } #endif @@ -650,7 +654,7 @@ buf_page_is_checksum_valid_innodb( #ifdef UNIV_DEBUG_LEVEL2 ib_logf(IB_LOG_LEVEL_INFO, "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", - checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), mach_read_from_4(read_buf + FIL_PAGE_LSN) ); #endif @@ -676,7 +680,7 @@ buf_page_is_checksum_valid_none( if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { ib_logf(IB_LOG_LEVEL_INFO, "Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", - checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC, mach_read_from_4(read_buf + FIL_PAGE_LSN) ); } @@ -773,6 +777,7 @@ buf_page_is_corrupted( if (zip_size) { return(!page_zip_verify_checksum(read_buf, zip_size)); } + if (page_encrypted) { return (FALSE); } @@ -4603,7 +4608,7 @@ buf_page_check_corrupt( ((buf_block_t*) bpage)->frame; bool page_compressed = bpage->page_encrypted; ulint stored_checksum = bpage->stored_checksum; - ulint calculated_checksum = bpage->stored_checksum; + ulint calculated_checksum = bpage->calculated_checksum; bool page_compressed_encrypted = bpage->page_compressed; ulint space_id = mach_read_from_4( dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -4707,6 +4712,10 @@ buf_page_io_complete( frame = ((buf_block_t*) bpage)->frame; } + ib_logf(IB_LOG_LEVEL_INFO, + "Page %u in tablespace %u encryption error key_version %u.", + bpage->offset, bpage->space, bpage->key_version); + goto database_corrupted; } @@ -4720,6 +4729,10 @@ buf_page_io_complete( os_atomic_decrement_ulint( &buf_pool->n_pend_unzip, 1); + ib_logf(IB_LOG_LEVEL_INFO, + "Page %u in tablespace %u zip_decompress failure.", + bpage->offset, bpage->space); + goto database_corrupted; } os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1); @@ -6440,7 +6453,23 @@ buf_page_decrypt_after_read( (crypt_data && crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && key_version != 0)) { - key_version = 0; + byte* frame = NULL; + + if (buf_page_get_zip_size(bpage)) { + frame = bpage->zip.data; + } else { + frame = ((buf_block_t*) bpage)->frame; + } + + /* If page is not corrupted at this point, page can't be + encrypted, thus set key_version to 0. If page is corrupted, + we assume at this point that it is encrypted as page + contained key_version != 0. Note that page could still be + really corrupted. This we will find out after decrypt by + checking page checksums. */ + if (!buf_page_is_corrupted(false, frame, buf_page_get_zip_size(bpage))) { + key_version = 0; + } } /* If page is encrypted read post-encryption checksum */ From 9741e0ea7250ef088b29294049902c7f568ba3f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 1 Nov 2016 07:52:28 +0200 Subject: [PATCH 169/295] Initialize zip_dict_ids table and avoid referencing array items as currently MariaDB does not support compressed columns. --- storage/xtradb/handler/handler0alter.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 9cbc738e6a200..362ac9c1b89e8 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -2894,6 +2894,14 @@ prepare_inplace_alter_table_dict( mem_heap_alloc(ctx->heap, altered_table->s->fields * sizeof(ulint))); + /* This is currently required for valgrind because MariaDB does + not currently support compressed columns. */ + for (size_t field_idx = 0; + field_idx < altered_table->s->fields; + ++field_idx) { + zip_dict_ids[field_idx] = ULINT_UNDEFINED; + } + const char* err_zip_dict_name = 0; if (!innobase_check_zip_dicts(altered_table, zip_dict_ids, ctx->trx, &err_zip_dict_name)) { @@ -3271,6 +3279,7 @@ prepare_inplace_alter_table_dict( DBUG_ASSERT(error == DB_SUCCESS); +#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS /* Adding compression dictionary <-> compressed table column links to the SYS_ZIP_DICT_COLS table. @@ -3279,6 +3288,7 @@ prepare_inplace_alter_table_dict( innobase_create_zip_dict_references(altered_table, ctx->trx->table_id, zip_dict_ids, ctx->trx); } +#endif /* Commit the data dictionary transaction in order to release the table locks on the system tables. This means that if From 7196691b44b65e12cb5cca6f17c8d0f091eb443f Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 1 Nov 2016 17:20:12 -0400 Subject: [PATCH 170/295] Fix/disable some failing galera tests. --- mysql-test/suite/galera/disabled.def | 4 +++- .../include/auto_increment_offset_restore.inc | 6 ++++++ .../include/auto_increment_offset_save.inc | 8 ++++++++ .../galera/r/galera_sst_xtrabackup-v2.result | 3 ++- mysql-test/suite/galera/t/galera#414.test | 1 + .../galera/t/galera_sst_xtrabackup-v2.test | 10 ++++++++++ .../suite/galera/t/galera_wan_restart_ist.test | 17 +++++++++++++++-- mysql-test/suite/galera_3nodes/disabled.def | 2 ++ 8 files changed, 47 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 4aa15d27661f6..771053778d3aa 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -28,4 +28,6 @@ galera_flush : mysql-wsrep/issues/229 galera_transaction_read_only : mysql-wsrep/issues/229 galera_gcs_fragment : Incorrect arguments to SET galera_flush_local : Fails sporadically -galera_binlog_stmt_autoinc : TODO: investigate \ No newline at end of file +galera_binlog_stmt_autoinc : TODO: investigate +galera_sst_xtrabackup-v2-options : TODO: Fix test case +mysql-wsrep#33 : TODO: investigate diff --git a/mysql-test/suite/galera/include/auto_increment_offset_restore.inc b/mysql-test/suite/galera/include/auto_increment_offset_restore.inc index 6218dfd6f2c56..1248ed100ca4f 100644 --- a/mysql-test/suite/galera/include/auto_increment_offset_restore.inc +++ b/mysql-test/suite/galera/include/auto_increment_offset_restore.inc @@ -32,4 +32,10 @@ if ($node_3) --connection $node_3 --eval SET @@global.auto_increment_offset = $auto_increment_offset_node_3; } + +if ($node_4) +{ +--connection $node_4 +--eval SET @@global.auto_increment_offset = $auto_increment_offset_node_4; +} --enable_query_log diff --git a/mysql-test/suite/galera/include/auto_increment_offset_save.inc b/mysql-test/suite/galera/include/auto_increment_offset_save.inc index 3c4db3f381cd1..216c689ec8cc7 100644 --- a/mysql-test/suite/galera/include/auto_increment_offset_save.inc +++ b/mysql-test/suite/galera/include/auto_increment_offset_save.inc @@ -13,6 +13,8 @@ # Connection handle for 2nd node # $node_3 (optional) # Connection handle for 3rd node +# $node_4 (optional) +# Connection handle for 4th node if (!$node_1) { @@ -35,3 +37,9 @@ if ($node_3) let $auto_increment_offset_node_3 = `SELECT @@global.auto_increment_offset`; } +if ($node_4) +{ + --connection $node_4 + let $auto_increment_offset_node_4 = `SELECT @@global.auto_increment_offset`; +} + diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2.result b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2.result index 750d73b615f55..df2d9190a4b56 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2.result +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2.result @@ -277,7 +277,7 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; -SET GLOBAL debug = 'd,sync.alter_opened_table'; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ALTER TABLE t1 ADD COLUMN f2 INTEGER; SET wsrep_sync_wait = 0; Killing server ... @@ -356,3 +356,4 @@ COUNT(*) = 0 DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; diff --git a/mysql-test/suite/galera/t/galera#414.test b/mysql-test/suite/galera/t/galera#414.test index b426e6510b645..0ee6dcac700bd 100644 --- a/mysql-test/suite/galera/t/galera#414.test +++ b/mysql-test/suite/galera/t/galera#414.test @@ -3,6 +3,7 @@ # --source include/big_test.inc +--source include/have_innodb.inc --source include/galera_cluster.inc # We perform the shutdown/restart sequence in here. If there was a crash during shutdown, MTR will detect it diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test index c6823795e59d0..aac6822170a86 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test @@ -2,8 +2,18 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + --source suite/galera/include/galera_st_shutdown_slave.inc --source suite/galera/include/galera_st_clean_slave.inc --source suite/galera/include/galera_st_kill_slave.inc --source suite/galera/include/galera_st_kill_slave_ddl.inc + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--source include/galera_end.inc diff --git a/mysql-test/suite/galera/t/galera_wan_restart_ist.test b/mysql-test/suite/galera/t/galera_wan_restart_ist.test index 42f63df3acc63..1cf5d4c7f74ee 100644 --- a/mysql-test/suite/galera/t/galera_wan_restart_ist.test +++ b/mysql-test/suite/galera/t/galera_wan_restart_ist.test @@ -12,6 +12,16 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--let $node_4=node_4 +--source include/auto_increment_offset_save.inc + SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --connection node_1 @@ -21,11 +31,9 @@ INSERT INTO t1 VALUES (1); --connection node_2 INSERT INTO t1 VALUES (2); ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 INSERT INTO t1 VALUES (3); ---connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 --connection node_4 INSERT INTO t1 VALUES (4); @@ -146,3 +154,8 @@ CALL mtr.add_suppression("Action message in non-primary configuration from membe --connection node_4 CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--source include/galera_end.inc diff --git a/mysql-test/suite/galera_3nodes/disabled.def b/mysql-test/suite/galera_3nodes/disabled.def index fb23a81bfb8b8..ca55c41ff72f4 100644 --- a/mysql-test/suite/galera_3nodes/disabled.def +++ b/mysql-test/suite/galera_3nodes/disabled.def @@ -3,3 +3,5 @@ galera_evs_suspect_timeout : TODO: investigate galera_innobackupex_backup : TODO: investigate galera_slave_options_do :MDEV-8798 galera_slave_options_ignore : MDEV-8798 +galera_pc_bootstrap : TODO: Investigate: Timeout in wait_condition.inc +galera_pc_weight : Test times out From c18054deb2b5cfcf1f13aa71574406f2bafb87c6 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 2 Nov 2016 08:20:15 +0400 Subject: [PATCH 171/295] MDEV-10347 mysqld got signal 11 --- mysql-test/r/null.result | 17 +++++++++++++++++ mysql-test/t/null.test | 17 +++++++++++++++++ sql/item_cmpfunc.h | 5 +++++ 3 files changed, 39 insertions(+) diff --git a/mysql-test/r/null.result b/mysql-test/r/null.result index fcaaeb2192f3e..d25e78c1ce499 100644 --- a/mysql-test/r/null.result +++ b/mysql-test/r/null.result @@ -1572,6 +1572,23 @@ SELECT * FROM t1 WHERE NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(N i DROP TABLE t1; # +# MDEV-10347 mysqld got signal 11 +# +CREATE TABLE t1 (f1 VARCHAR(10), f2 VARCHAR(40)); +CREATE TABLE t2 (f3 VARCHAR(20)); +PREPARE stmt FROM " + SELECT ( + SELECT IFNULL(f3,4) FROM t2 + WHERE IFNULL(NULLIF(f1,''),1) + ) AS sq + FROM t1 + GROUP BY f2 +"; +EXECUTE stmt; +sq +DEALLOCATE PREPARE stmt; +DROP TABLE t2,t1; +# # MDEV-10236 Where expression with NOT function gives incorrect result # CREATE TABLE t1 (c1 INT); diff --git a/mysql-test/t/null.test b/mysql-test/t/null.test index 84b3f0696834e..a55f5ce58d3ac 100644 --- a/mysql-test/t/null.test +++ b/mysql-test/t/null.test @@ -1006,6 +1006,23 @@ INSERT INTO t1 VALUES (1),(2); SELECT * FROM t1 WHERE NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(NULLIF(i = ROUND(0), 14), 13), 12), 11), 10), 9), 8), 7), 6), 5), 4), 3), 2), 1); DROP TABLE t1; +--echo # +--echo # MDEV-10347 mysqld got signal 11 +--echo # + +CREATE TABLE t1 (f1 VARCHAR(10), f2 VARCHAR(40)); +CREATE TABLE t2 (f3 VARCHAR(20)); +PREPARE stmt FROM " + SELECT ( + SELECT IFNULL(f3,4) FROM t2 + WHERE IFNULL(NULLIF(f1,''),1) + ) AS sq + FROM t1 + GROUP BY f2 +"; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +DROP TABLE t2,t1; --echo # --echo # MDEV-10236 Where expression with NOT function gives incorrect result diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 2e066b895e946..9a9d0e65ff676 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1022,6 +1022,11 @@ class Item_func_nullif :public Item_func_hybrid_field_type Item_func_hybrid_field_type(thd, a, b, a), m_cache(NULL) { arg_count--; } + void cleanup() + { + Item_func_hybrid_field_type::cleanup(); + arg_count= 2; // See the comment to the constructor + } bool date_op(MYSQL_TIME *ltime, uint fuzzydate); double real_op(); longlong int_op(); From c948559135aed3c48d49e2fdcca31be3ffde1ec1 Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Wed, 20 Jul 2016 13:43:53 -0400 Subject: [PATCH 172/295] MDEV-10408 run the tokudb_rpl.rpl_rfr_disable_on_expl_pk_absence test. Add control files. Fixup result file for mariadb --- storage/tokudb/mysql-test/tokudb_rpl/include/have_tokudb.inc | 1 + .../tokudb_rpl/r/rpl_rfr_disable_on_expl_pk_absence.result | 3 --- storage/tokudb/mysql-test/tokudb_rpl/suite.opt | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/include/have_tokudb.inc create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/suite.opt diff --git a/storage/tokudb/mysql-test/tokudb_rpl/include/have_tokudb.inc b/storage/tokudb/mysql-test/tokudb_rpl/include/have_tokudb.inc new file mode 100644 index 0000000000000..12b29a22d2c65 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/include/have_tokudb.inc @@ -0,0 +1 @@ +let $datadir=`select @@datadir`; diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_rfr_disable_on_expl_pk_absence.result b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_rfr_disable_on_expl_pk_absence.result index 981a833aea5db..2977dc859f5b3 100644 --- a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_rfr_disable_on_expl_pk_absence.result +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_rfr_disable_on_expl_pk_absence.result @@ -1,7 +1,4 @@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] call mtr.add_suppression("read free replication is disabled for tokudb table"); CREATE TABLE t (a int(11), b char(20)) ENGINE = TokuDB; diff --git a/storage/tokudb/mysql-test/tokudb_rpl/suite.opt b/storage/tokudb/mysql-test/tokudb_rpl/suite.opt new file mode 100644 index 0000000000000..f94d0f6d6dcc2 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/suite.opt @@ -0,0 +1 @@ +--tokudb --plugin-load-add=$HA_TOKUDB_SO --loose-tokudb-check-jemalloc=0 --sql-mode=NO_ENGINE_SUBSTITUTION From 110a9f069386302c9caa0b1a451407d6512fdda5 Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Wed, 13 Jul 2016 15:15:35 -0400 Subject: [PATCH 173/295] add mtr support files for tokudb_sys_vars tests --- .../tokudb_sys_vars/include/have_tokudb.inc | 1 + .../tokudb/mysql-test/tokudb_sys_vars/suite.opt | 1 + storage/tokudb/mysql-test/tokudb_sys_vars/suite.pm | 14 ++++++++++++++ 3 files changed, 16 insertions(+) create mode 100644 storage/tokudb/mysql-test/tokudb_sys_vars/include/have_tokudb.inc create mode 100644 storage/tokudb/mysql-test/tokudb_sys_vars/suite.opt create mode 100644 storage/tokudb/mysql-test/tokudb_sys_vars/suite.pm diff --git a/storage/tokudb/mysql-test/tokudb_sys_vars/include/have_tokudb.inc b/storage/tokudb/mysql-test/tokudb_sys_vars/include/have_tokudb.inc new file mode 100644 index 0000000000000..12b29a22d2c65 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_sys_vars/include/have_tokudb.inc @@ -0,0 +1 @@ +let $datadir=`select @@datadir`; diff --git a/storage/tokudb/mysql-test/tokudb_sys_vars/suite.opt b/storage/tokudb/mysql-test/tokudb_sys_vars/suite.opt new file mode 100644 index 0000000000000..f94d0f6d6dcc2 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_sys_vars/suite.opt @@ -0,0 +1 @@ +--tokudb --plugin-load-add=$HA_TOKUDB_SO --loose-tokudb-check-jemalloc=0 --sql-mode=NO_ENGINE_SUBSTITUTION diff --git a/storage/tokudb/mysql-test/tokudb_sys_vars/suite.pm b/storage/tokudb/mysql-test/tokudb_sys_vars/suite.pm new file mode 100644 index 0000000000000..6c52d0110fe8c --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_sys_vars/suite.pm @@ -0,0 +1,14 @@ +package My::Suite::TokuDB; +use File::Basename; +@ISA = qw(My::Suite); + +# Ensure we can run the TokuDB tests even if hugepages are enabled +$ENV{TOKU_HUGE_PAGES_OK}=1; + +#return "Not run for embedded server" if $::opt_embedded_server; +return "No TokuDB engine" unless $ENV{HA_TOKUDB_SO} or $::mysqld_variables{tokudb}; + +sub is_default { not $::opt_embedded_server } + +bless { }; + From a52a68cb7da765a807856a8c6f238446f57b8f2a Mon Sep 17 00:00:00 2001 From: Rik Prohaska Date: Wed, 13 Jul 2016 16:17:08 -0400 Subject: [PATCH 174/295] fix the tokudb_analyze_in_background_basic test to run on mariadb. mariadb does additional value checking for boolean system variables --- .../tokudb_analyze_in_background_basic.result | 24 ++++++++++--------- .../t/tokudb_analyze_in_background_basic.test | 2 ++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/storage/tokudb/mysql-test/tokudb_sys_vars/r/tokudb_analyze_in_background_basic.result b/storage/tokudb/mysql-test/tokudb_sys_vars/r/tokudb_analyze_in_background_basic.result index 53e96810eda24..02b808fe6cbd0 100644 --- a/storage/tokudb/mysql-test/tokudb_sys_vars/r/tokudb_analyze_in_background_basic.result +++ b/storage/tokudb/mysql-test/tokudb_sys_vars/r/tokudb_analyze_in_background_basic.result @@ -19,29 +19,30 @@ SELECT @@global.tokudb_analyze_in_background; @@global.tokudb_analyze_in_background 0 SET GLOBAL tokudb_analyze_in_background = -6; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of '-6' SELECT @@global.tokudb_analyze_in_background; @@global.tokudb_analyze_in_background -1 +0 SET GLOBAL tokudb_analyze_in_background = 1.6; ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_in_background' SELECT @@global.tokudb_analyze_in_background; @@global.tokudb_analyze_in_background -1 +0 SET GLOBAL tokudb_analyze_in_background = "T"; ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'T' SELECT @@global.tokudb_analyze_in_background; @@global.tokudb_analyze_in_background -1 +0 SET GLOBAL tokudb_analyze_in_background = "Y"; ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'Y' SELECT @@global.tokudb_analyze_in_background; @@global.tokudb_analyze_in_background -1 +0 SET GLOBAL tokudb_analyze_in_background = 'foobar'; ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'foobar' SELECT @@global.tokudb_analyze_in_background; @@global.tokudb_analyze_in_background -1 +0 SET SESSION tokudb_analyze_in_background = 0; SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background @@ -53,31 +54,32 @@ SELECT @@session.tokudb_analyze_in_background; SET SESSION tokudb_analyze_in_background = DEFAULT; SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background -1 +0 SET SESSION tokudb_analyze_in_background = -6; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of '-6' SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background -1 +0 SET SESSION tokudb_analyze_in_background = 1.6; ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_in_background' SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background -1 +0 SET SESSION tokudb_analyze_in_background = "T"; ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'T' SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background -1 +0 SET SESSION tokudb_analyze_in_background = "Y"; ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'Y' SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background -1 +0 SET SESSION tokudb_analyze_in_background = 'foobar'; ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'foobar' SELECT @@session.tokudb_analyze_in_background; @@session.tokudb_analyze_in_background -1 +0 SET GLOBAL tokudb_analyze_in_background = 0; SET SESSION tokudb_analyze_in_background = 1; SELECT @@global.tokudb_analyze_in_background; diff --git a/storage/tokudb/mysql-test/tokudb_sys_vars/t/tokudb_analyze_in_background_basic.test b/storage/tokudb/mysql-test/tokudb_sys_vars/t/tokudb_analyze_in_background_basic.test index dfb2a0e416dbe..84b001b196230 100644 --- a/storage/tokudb/mysql-test/tokudb_sys_vars/t/tokudb_analyze_in_background_basic.test +++ b/storage/tokudb/mysql-test/tokudb_sys_vars/t/tokudb_analyze_in_background_basic.test @@ -17,6 +17,7 @@ SELECT @@global.tokudb_analyze_in_background; SET GLOBAL tokudb_analyze_in_background = DEFAULT; SELECT @@global.tokudb_analyze_in_background; +-- error ER_WRONG_VALUE_FOR_VAR SET GLOBAL tokudb_analyze_in_background = -6; SELECT @@global.tokudb_analyze_in_background; @@ -46,6 +47,7 @@ SELECT @@session.tokudb_analyze_in_background; SET SESSION tokudb_analyze_in_background = DEFAULT; SELECT @@session.tokudb_analyze_in_background; +-- error ER_WRONG_VALUE_FOR_VAR SET SESSION tokudb_analyze_in_background = -6; SELECT @@session.tokudb_analyze_in_background; From fa4fa0ab90e4e49dde955333667cc606838d8fa9 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Fri, 30 Sep 2016 21:13:03 -0400 Subject: [PATCH 175/295] Make galera test suites default. --- mysql-test/mysql-test-run.pl | 2 ++ plugin/wsrep_info/mysql-test/wsrep_info/suite.pm | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 2bd89f5ae4939..16d8f9bb25293 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -175,6 +175,8 @@ END federated- funcs_1- funcs_2- + galera- + galera_3nodes- handler- heap- innodb- diff --git a/plugin/wsrep_info/mysql-test/wsrep_info/suite.pm b/plugin/wsrep_info/mysql-test/wsrep_info/suite.pm index 5fd761896a154..7148a9cf057f5 100644 --- a/plugin/wsrep_info/mysql-test/wsrep_info/suite.pm +++ b/plugin/wsrep_info/mysql-test/wsrep_info/suite.pm @@ -39,5 +39,7 @@ push @::global_suppressions, $ENV{PATH}="$epath:$ENV{PATH}"; $ENV{PATH}="$spath:$ENV{PATH}" unless $epath eq $spath; +sub is_default { 1 } + bless { }; From 6dbfe7f399d121dc2a040b21c96777c0a47141fb Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 3 Oct 2016 12:02:46 -0400 Subject: [PATCH 176/295] MDEV-10944: GALERA log-slave-updates FAIL after upgrading from 10.1.17 to 10.1.18 thd->variables.option_bits need to be restored after plugin_thdvar_init() during post initialization of wsrep threads. --- sql/sql_plugin.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 5b16482ae6608..ed0721feb3b52 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -4271,7 +4271,13 @@ void wsrep_plugins_post_init() { if (IF_WSREP(thd->wsrep_applier,1)) { + // Save options_bits as it will get overwritten in plugin_thdvar_init() + ulonglong option_bits_saved= thd->variables.option_bits; + plugin_thdvar_init(thd); + + // Restore option_bits + thd->variables.option_bits= option_bits_saved; } } From 3daf89ced99b41ad6144cd1d4236959e641f5592 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 5 Oct 2016 04:24:07 -0400 Subject: [PATCH 177/295] MDEV-10957: Assertion failure when dropping a myisam table with wsrep_replicate_myisam enabled Internal updates to system statistical tables could wrongly trigger an additional total-order replication if wsrep_repli -cate_myisam is enabled. Fixed by adding a check to skip total-order replication for stat tables. Test: galera.galera_var_replicate_myisam_on --- sql/sql_base.cc | 1 + sql/sql_statistics.cc | 18 ++++++++++++++++++ sql/sql_statistics.h | 1 + 3 files changed, 20 insertions(+) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index b4a3cc27d2c7f..f04bece86d268 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -4736,6 +4736,7 @@ bool open_tables(THD *thd, const DDL_options_st &options, (*start) && (*start)->table && (*start)->table->file->ht == myisam_hton && + !is_stat_table((*start)->db, (*start)->alias) && sqlcom_can_generate_row_events(thd) && thd->get_command() != COM_STMT_PREPARE) { diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 311263c39b1d5..27bc0fb4cd36b 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -3840,3 +3840,21 @@ double Histogram::point_selectivity(double pos, double avg_sel) return sel; } +/* + Check whether the table is one of the persistent statistical tables. +*/ +bool is_stat_table(const char *db, const char *table) +{ + DBUG_ASSERT(db && table); + + if (!memcmp(db, stat_tables_db_name.str, stat_tables_db_name.length)) + { + for (uint i= 0; i < STATISTICS_TABLES; i ++) + { + if (!memcmp(table, stat_table_name[i].str, stat_table_name[i].length)) + return true; + } + } + return false; +} + diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 8e5f8107849a7..20b2eb664492a 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -107,6 +107,7 @@ double get_column_range_cardinality(Field *field, key_range *min_endp, key_range *max_endp, uint range_flag); +bool is_stat_table(const char *db, const char *table); class Histogram { From 07918b48b8c959e708a6175241d59c739fb63418 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 10 Oct 2016 14:02:37 -0400 Subject: [PATCH 178/295] MDEV-10993: wsrep.mdev_10186 result depends on location of galera library Update test case. --- mysql-test/suite/wsrep/r/mdev_10186.result | 2 +- mysql-test/suite/wsrep/t/mdev_10186.test | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/wsrep/r/mdev_10186.result b/mysql-test/suite/wsrep/r/mdev_10186.result index f966c4435695e..778c064d20877 100644 --- a/mysql-test/suite/wsrep/r/mdev_10186.result +++ b/mysql-test/suite/wsrep/r/mdev_10186.result @@ -7,5 +7,5 @@ SELECT @@wsrep_on; 0 SELECT @@GLOBAL.wsrep_provider; @@GLOBAL.wsrep_provider -/usr/lib/galera/libgalera_smm.so +libgalera_smm.so SET @@GLOBAL.wsrep_cluster_address='gcomm://'; diff --git a/mysql-test/suite/wsrep/t/mdev_10186.test b/mysql-test/suite/wsrep/t/mdev_10186.test index ec5e9462821ec..98ea519263451 100644 --- a/mysql-test/suite/wsrep/t/mdev_10186.test +++ b/mysql-test/suite/wsrep/t/mdev_10186.test @@ -7,6 +7,7 @@ --echo # SELECT @@wsrep_on; +--replace_regex /.*libgalera_smm.so/libgalera_smm.so/ SELECT @@GLOBAL.wsrep_provider; SET @@GLOBAL.wsrep_cluster_address='gcomm://'; From 7a170205e0dbd7db9047ad1e25fb465b047c6c33 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 26 Oct 2016 13:19:00 -0400 Subject: [PATCH 179/295] MDEV-11152: wsrep_replicate_myisam: SELECT gets replicated using TO Fixed the 'wsrep_replicate_myisam' check to allow only limited set of commands. Added a debug assert to discover such cases. --- .../r/galera_var_replicate_myisam_on.result | 9 +++++++ .../t/galera_var_replicate_myisam_on.test | 10 ++++++++ sql/sql_base.cc | 24 ++++++++++++------- sql/sql_parse.cc | 4 ++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result b/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result index 73a0576048b0a..22ba4ca87675d 100644 --- a/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result +++ b/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result @@ -74,5 +74,14 @@ ERROR 23000: Duplicate entry '1' for key 'PRIMARY' COMMIT; DROP TABLE t1; DROP TABLE t2; +# +# MDEV-11152: wsrep_replicate_myisam: SELECT gets replicated using TO +# +CREATE TABLE t1 (i INT) ENGINE=INNODB; +INSERT INTO t1 VALUES(1); +SELECT * FROM t1; +i +1 +DROP TABLE t1; SET GLOBAL wsrep_replicate_myisam = 0; SET GLOBAL wsrep_replicate_myisam = 0; diff --git a/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test b/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test index 9cb0edf181076..90c786f0af0d5 100644 --- a/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test +++ b/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test @@ -132,6 +132,16 @@ COMMIT; DROP TABLE t1; DROP TABLE t2; +--echo # +--echo # MDEV-11152: wsrep_replicate_myisam: SELECT gets replicated using TO +--echo # +--connection node_1 +CREATE TABLE t1 (i INT) ENGINE=INNODB; +INSERT INTO t1 VALUES(1); +# This command should not get replicated. +SELECT * FROM t1; +DROP TABLE t1; + --connection node_1 --eval SET GLOBAL wsrep_replicate_myisam = $wsrep_replicate_myisam_orig diff --git a/sql/sql_base.cc b/sql/sql_base.cc index f04bece86d268..e3f755be7ae09 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -4731,14 +4731,22 @@ bool open_tables(THD *thd, const DDL_options_st &options, } } - if (WSREP_ON && - wsrep_replicate_myisam && - (*start) && - (*start)->table && - (*start)->table->file->ht == myisam_hton && - !is_stat_table((*start)->db, (*start)->alias) && - sqlcom_can_generate_row_events(thd) && - thd->get_command() != COM_STMT_PREPARE) + if (WSREP_ON && + wsrep_replicate_myisam && + (*start) && + (*start)->table && + (*start)->table->file->ht == myisam_hton && + wsrep_thd_exec_mode(thd) == LOCAL_STATE && + !is_stat_table((*start)->db, (*start)->alias) && + thd->get_command() != COM_STMT_PREPARE && + ((thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_REPLACE || + thd->lex->sql_command == SQLCOM_REPLACE_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_LOAD || + thd->lex->sql_command == SQLCOM_DELETE))) { WSREP_TO_ISOLATION_BEGIN(NULL, NULL, (*start)); } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index e37a4b9ccc17f..5169b7c557356 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2941,6 +2941,10 @@ mysql_execute_command(THD *thd) case SQLCOM_SHOW_STORAGE_ENGINES: case SQLCOM_SHOW_PROFILE: { +#ifdef WITH_WSREP + DBUG_ASSERT(thd->wsrep_exec_mode != REPL_RECV); +#endif /* WITH_WSREP */ + thd->status_var.last_query_cost= 0.0; /* From 54d3dc0e93aea78a0dbd66a89f31ed5fa633c582 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 1 Nov 2016 17:27:53 -0400 Subject: [PATCH 180/295] Fix/disable some failing galera tests. --- mysql-test/suite/galera/disabled.def | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 907f7931483b6..38bd16ed1f39f 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -30,5 +30,8 @@ galera_gcs_fragment : Incorrect arguments to SET galera_flush_local : Fails sporadically galera_binlog_stmt_autoinc : TODO: investigate galera_concurrent_ctas : Test times out, investigate +MW-286 : TODO: investigate galera_sst_xtrabackup-v2-options : TODO: Fix test case +galera_sst_xtrabackup-v2 : TODO: Fix test case +galera_sst_xtrabackup-v2_encrypt_with_key : TODO: Fix test case mysql-wsrep#33 : TODO: investigate From 82780a7c0031af3fe6d8b9bc9f46ace2876c33f6 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 2 Nov 2016 21:20:00 +0300 Subject: [PATCH 181/295] MDEV-11130 Update the list of unstable tests for 10.1 --- mysql-test/unstable-tests | 154 +++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 75 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 82713424daa7c..528c36a4a1301 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,65 +23,56 @@ # ############################################################################## +main.alter_table : Modified in 10.1.19 main.analyze_stmt_slow_query_log : MDEV-7558 - wrong result -main.bootstrap : Modified on 2016-06-18 (MDEV-9969) main.create_delayed : MDEV-10605 - failed with timeout -main.create_or_replace : Modified on 2016-06-23 (MDEV-9728) -main.ctype_recoding : Modified on 2016-06-10 (MDEV-10181) -main.ctype_utf8 : Modified on 2016-06-21 (merge) -main.ctype_utf8mb4 : Modified on 2016-06-21 (merge) +main.create_or_replace : Modified in 10.1.19 main.ctype_utf16le : MDEV-10675: timeout or extra warnings -main.events_1 : Modified on 2016-06-21 (MDEV-9524) +main.drop : Modified in 10.1.19 +main.events_restart : MDEV-11221: assertion failure main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_in : Modified on 2016-06-20 (MDEV-10020) main.func_math : Modified on 2016-08-10 (merge) main.func_misc : Modified on 2016-08-10 (merge) main.grant2 : Modified on 2016-07-18 (MDEV-8569) -main.help : Modified on 2016-06-21 (MDEV-9524) main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout -main.index_merge_innodb : MDEV-7142 - sporadic wrong execution plan +main.information_schema : Modified in 10.1.19 main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.insert_innodb : Modified on 2016-06-14 (merge from upstream) main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist main.loaddata : Modified on 2016-08-10 (merge) -main.locale : Modified on 2016-06-21 (merge) main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.multi_update : Modified on 2016-06-20 (MDEV-5973) main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) +main.mysql : Modified in 10.1.19 +main.mysql_not_windows : Modified in 10.1.19 main.mysqlcheck : Modified on 2016-08-10 (merge) -main.mysqldump : MDEV-10512 - sporadic assertion failure -main.mysqltest : MDEV-9269 - fails on Alpha +main.mysqldump-nl : Added in 10.1.19 +main.mysqltest : MDEV-9269 - fails on Alpha; also modified in 10.1.19 main.named_pipe : Modified on 2016-08-02 (MDEV-10383) +main.null : Modified in 10.1.19 main.openssl_1 : Modified on 2016-07-11 (MDEV-10211) main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan -main.parser : Modified on 2016-06-21 (merge) main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count main.ps_1general : Modified on 2016-07-12 (merge) main.range : Modified on 2016-08-10 (merge) main.range_mrr_icp : Modified on 2016-08-10 (merge) -main.query_cache : MDEV-10611 - sporadic mutex problem +main.selectivity : Modified in 10.1.19 +main.selectivity_innodb : Modified in 10.1.19 main.show_explain : MDEV-10674 - sporadic failure -main.shutdown : MDEV-10612 - sporadic crashes +main.shutdown : MDEV-10563 - sporadic crashes main.sp-prelocking : Modified on 2016-08-10 (merge) main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl : MDEV-10211 - different ciphers on some platforms main.ssl_ca : Modified on 2016-07-11 (MDEV-10211) main.ssl_compress : Modified on 2016-07-11 (MDEV-10211) main.ssl_timeout : Modified on 2016-07-11 (MDEV-10211) main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results main.status : MDEV-8510 - sporadic wrong result -main.status_user : Modified on 2016-06-20 (MDEV-8633) -main.subselect : Modified in 10.1.17 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_sj_mat : Modified in 10.1.17 -main.temp_table : Modified on 2016-06-18 (MDEV-8569) +main.subselect_sj_mat : Modified on 2016-07-27 (MDEV-10389) main.type_date : Modified on 2016-08-10 (merge) -main.type_datetime : Modified on 2016-06-16 (MDEV-9374) main.type_datetime_hires : MDEV-10687 - timeout main.view : Modified on 2016-08-10 (merge) main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) @@ -94,7 +85,6 @@ archive.discover : MDEV-10510 - table is marked as crashed #---------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed -binlog.binlog_dmls_on_tmp_tables_readonly : New test, added on 2016-05-04 (upstream) binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- @@ -107,6 +97,44 @@ connect.jdbc-postgresql : New test, added on 2016-07-13 #---------------------------------------------------------------- +encryption.create_or_replace : MDEV-9359 - Assertion failure +encryption.encrypt_and_grep : MDEV-11222 - InnoDB error +encryption.innodb-bad-key-change : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-change2 : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-change3 : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-change4 : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-change5 : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures, and uses keys2.txt modified in 10.1.19 +encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also modified in 10.1.19 +encryption.innodb_encryption_filekeys : MDEV-9962 - timeouts +encryption.innodb_encryption_tables : MDEV-10970 - Crash/assertion failure +encryption.innodb_first_page : MDEV-10689 - crashes +encryption.innodb-missing-key : Added in 10.1.19 +encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results +encryption.innodb-page_encryption : MDEV-10641 - mutex problem +encryption.innodb-page_encryption_compression : MDEV-10739 - timeouts +encryption.innodb_page_encryption_key_change : uses keys2.txt modified in 10.1.19 + +#---------------------------------------------------------------- + +extra/binlog_tests.database : Modified in 10.1.19 (merge) + +#---------------------------------------------------------------- + +federated.federatedx : MDEV-10617 - Wrong checksum, timeouts +federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips +federated.federated_partition : MDEV-10417 - Fails on Mips +federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips + +#---------------------------------------------------------------- + +funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge) +funcs_1.processlist_priv_ps : Include file modified on 2016-07-12 (merge) +funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result +funcs_2.memory_charset : MDEV-10290 - timeout + +#---------------------------------------------------------------- + galera.GAL-382 : New test, added in 10.1.17 galera.MW-252 : New test, added in 10.1.17 galera.MW-258 : New test, added in 10.1.17 @@ -144,28 +172,8 @@ galera_3nodes.galera_ist_gcache_rollover : Modified in 10.1.17 galera_3nodes.galera_pc_bootstrap : Modified in 10.1.17 galera_3nodes.galera_pc_weight : Modified in 10.1.17 -#---------------------------------------------------------------- - -encryption.create_or_replace : MDEV-9359 - Assertion failure -encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures -encryption.innodb_encryption_discard_import : MDEV-9099 - warnings, errors, crash -encryption.innodb_encryption_filekeys : MDEV-9962 - timeouts -encryption.innodb_first_page : MDEV-10689 - crashes -encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results -encryption.innodb-page_encryption : MDEV-10641 - mutex problem - -#---------------------------------------------------------------- - -federated.federatedx : MDEV-10617 - Wrong checksum, timeouts -federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips -federated.federated_partition : MDEV-10417 - Fails on Mips -federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips - -#---------------------------------------------------------------- - -funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge) -funcs_1.processlist_priv_ps : Include file modified on 2016-07-12 (merge) -funcs_2.memory_charset : MDEV-10290 - timeout +galera.* : Added to default suites in 10.1.19 +galera_3nodes.* : Added to default suites in 10.1.19 #---------------------------------------------------------------- @@ -173,20 +181,19 @@ innodb.binlog_consistent : MDEV-10618 - Server fails to start innodb.innodb-alter-table : MDEV-10619 - Testcase timeout innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) innodb.innodb_blob_truncate : MDEV-10377 - Assertion failure -innodb.innodb-bug-14068765 : MDEV-9105 - valgrind warnings, assertion failures -innodb.innodb_corrupt_bit : Modified on 2016-06-21 (merge) -innodb.innodb-bug-14084530 : MDEV-9105 - valgrind warnings, assertion failures +innodb.innodb-bug-14068765 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.innodb-bug-14084530 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan innodb.innodb-fk-warnings : Modified on 2016-07-18 (MDEV-8569) -innodb.innodb-fkcheck : Modified on 2016-06-13 (MDEV-10083) innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result -innodb.innodb-wl5522 : MDEV-9105 - valgrind warnings, assertion failures -innodb.innodb-wl5522-1 : MDEV-9105 - valgrind warnings, assertion failures -innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption -innodb.innodb-wl5522-zip : MDEV-9105 - valgrind warnings, assertion failures -innodb.xa_recovery : MDEV-10685 - warnings +innodb.innodb-wl5522 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.innodb-wl5522-1 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.innodb-wl5522-debug : modified in 10.1.19 +innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption, also modified in 10.1.19 +innodb.innodb-wl5522-zip : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.xa_recovery : Modified in 10.1.19 #---------------------------------------------------------------- @@ -194,6 +201,10 @@ innodb_zip.innodb_prefix_index_liftedlimit : MDEV-10686 - timeout #---------------------------------------------------------------- +maria.encrypt-wrong-key : uses keys2.txt modified in 10.1.19 + +#---------------------------------------------------------------- + mroonga/storage.column_datetime_32bit_2038 : Wrong result on Alpha mroonga/storage.column_datetime_32bit_before_unix_epoch : Wrong result on Alpha mroonga/storage.column_datetime_32bit_max : Wrong result on Alpha @@ -219,18 +230,10 @@ parts.partition_int_myisam : MDEV-10621 - Testcase timeout #---------------------------------------------------------------- -perfschema.digest_table_full : Modified on 2016-06-21 (merge) perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x -perfschema.rpl_gtid_func : Modified on 2016-06-21 (merge) perfschema.setup_actors : MDEV-10679 - rare crash -perfschema.sizing_low : Modified on 2016-04-26 (5.6.30 merge) perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match -perfschema.start_server_low_digest : Modified on 2016-06-21 (merge) -perfschema.statement_digest : Modified on 2016-06-21 (merge) -perfschema.statement_digest_consumers : Modified on 2016-06-21 (merge) -perfschema.statement_digest_long_query : Modified on 2016-06-21 (merge) -perfschema.table_name : New test, added on 2016-04-26 (5.6.30 merge) perfschema.threads_mysql : MDEV-10677 - sporadic wrong result #---------------------------------------------------------------- @@ -243,11 +246,6 @@ plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc #---------------------------------------------------------------- -roles.rpl_grant_revoke_current_role-8638 : New test, added on 2016-06-20 (MDEV-8638) -roles.set_role-9614 : New test, added on 2016-05-30 (MDEV-9614) - -#---------------------------------------------------------------- - rpl.last_insert_id : MDEV-10625 - warnings in error log rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips @@ -257,13 +255,13 @@ rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result +rpl.rpl_drop_db : Modified in 10.1.19 rpl.rpl_gtid_basic : MDEV-10681 - server startup problem rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_mdev9033 : MDEV-10680 - warnings rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log -rpl.rpl_ignore_table : Modified on 2016-06-22 rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x @@ -273,15 +271,17 @@ rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts rpl.rpl_parallel_optimistic : MDEV-10511 - timeout +rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout rpl.rpl_row_log_innodb : MDEV-10688 - Wrong result rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem +rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock -rpl.rpl_switch_stm_row_mixed : MDEV-10611 - Wrong usage of mutex +rpl.rpl_stop_slave_error : Modified in 10.1.19 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master @@ -300,6 +300,10 @@ spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x #---------------------------------------------------------------- +storage_engine* : Tests are not always timely maintained + +#---------------------------------------------------------------- + stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- @@ -317,7 +321,6 @@ sys_vars.wsrep_max_ws_size_basic : Modified in 10.1.17 #---------------------------------------------------------------- -tokudb.background_job_manager : MDEV-10327 - Assertion failure on server shutdown tokudb.cluster_filter : MDEV-10678 - Wrong execution plan tokudb.cluster_filter_hidden : MDEV-10678 - Wrong execution plan tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan @@ -333,13 +336,13 @@ tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown tokudb_parts.* : MDEV-9891 - massive crashes on shutdown -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown, also modified on 2016-06-10 (Merge) +rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown tokudb_backup.* : MDEV-9891 - massive crashes on shutdown tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown -tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown +tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown #---------------------------------------------------------------- @@ -347,12 +350,13 @@ unit.ma_test_loghandler : MDEV-10638 - record read not ok #---------------------------------------------------------------- -vcol.charsets : Added on 2016-06-23 vcol.not_supported : MDEV-10639 - Testcase timeout vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout #---------------------------------------------------------------- -wsrep.* : MDEV-10041 - server crashes sporadically during bootstrap +wsrep.mdev_10186 : Modified in 10.1.19 +#---------------------------------------------------------------- +wsrep_info.* : suite.pm modified in 10.1.19 From e0f48e5ce9e77711f6d8cc1397bc32e0bd3edc33 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 3 Nov 2016 16:21:48 +0000 Subject: [PATCH 182/295] MDEV-11214 Windows : MSI installation fails, if run by a service user (e.g LocalSystem) Skip permission for data directory for LogonUser, if installation runs by one of the service accounts (determined from their well-known SID). There is no real LogonUser in this case. --- win/packaging/extra.wxs.in | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in index b716bbf7e88d6..f60c0bb2020f6 100644 --- a/win/packaging/extra.wxs.in +++ b/win/packaging/extra.wxs.in @@ -463,9 +463,24 @@ + + + + + + + + + "S-1-5-18") AND (UserSID <> "S-1-5-19") AND (UserSID <> "S-1-5-20") ]]> + + - From f0d8a4d29e3dbd713dc13fb052acd0a96ab47522 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 3 Nov 2016 22:02:24 +0400 Subject: [PATCH 183/295] MDEV-11219 main.null fails in buldbot and outside with ps-protocol --- sql/item_cmpfunc.cc | 42 +++++++++++++++++++++++++++++++++++++++++- sql/item_cmpfunc.h | 4 +++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index a222335cf972a..1a2c85f66a607 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -2553,7 +2553,7 @@ Item_func_nullif::fix_length_and_dec() See also class Item_func_nullif declaration. */ if (arg_count == 2) - args[arg_count++]= args[0]; + args[arg_count++]= m_arg0 ? m_arg0 : args[0]; THD *thd= current_thd; /* @@ -2704,7 +2704,47 @@ Item_func_nullif::fix_length_and_dec() unsigned_flag= args[2]->unsigned_flag; fix_char_length(args[2]->max_char_length()); maybe_null=1; + m_arg0= args[0]; setup_args_and_comparator(thd, &cmp); + /* + A special code for EXECUTE..PREPARE. + + If args[0] did not change, then we don't remember it, as it can point + to a temporary Item object which will be destroyed between PREPARE + and EXECUTE. EXECUTE time fix_length_and_dec() will correctly set args[2] + from args[0] again. + + If args[0] changed, then it can be Item_func_conv_charset() for the + original args[0], which was permanently installed during PREPARE time + into the item tree as a wrapper for args[0], using change_item_tree(), i.e. + + NULLIF(latin1_field, 'a' COLLATE utf8_bin) + + was "rewritten" to: + + CASE WHEN CONVERT(latin1_field USING utf8) = 'a' COLLATE utf8_bin + THEN NULL + ELSE latin1_field + + - m_args0 points to Item_field corresponding to latin1_field + - args[0] points to Item_func_conv_charset + - args[0]->args[0] is equal to m_args0 + - args[1] points to Item_func_set_collation + - args[2] points is eqial to m_args0 + + In this case we remember and reuse m_arg0 during EXECUTE time as args[2]. + + QQ: How to make sure that m_args0 does not point + to something temporary which will be destoyed between PREPARE and EXECUTE. + The condition below should probably be more strict and somehow check that: + - change_item_tree() was called for the new args[0] + - m_args0 is referenced from inside args[0], e.g. as a function argument, + and therefore it is also something that won't be destroyed between + PREPARE and EXECUTE. + Any ideas? + */ + if (args[0] == m_arg0) + m_arg0= NULL; } diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 9a9d0e65ff676..9149f25b1b1bb 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1009,6 +1009,7 @@ class Item_func_nullif :public Item_func_hybrid_field_type */ Item_cache *m_cache; int compare(); + Item *m_arg0; public: /* Here we pass three arguments to the parent constructor, as NULLIF @@ -1020,7 +1021,8 @@ class Item_func_nullif :public Item_func_hybrid_field_type */ Item_func_nullif(THD *thd, Item *a, Item *b): Item_func_hybrid_field_type(thd, a, b, a), - m_cache(NULL) + m_cache(NULL), + m_arg0(NULL) { arg_count--; } void cleanup() { From d34cd353449d9902d2dd04e98d6b2d7d4fffa336 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 3 Nov 2016 22:07:31 +0400 Subject: [PATCH 184/295] Disabling tests mentioned in MDEV-11229 galera.MW-258 galera.galera_as_master fail in buildbot --- mysql-test/disabled.def | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mysql-test/disabled.def b/mysql-test/disabled.def index e82ddcf92d1bf..6de5646d05a16 100644 --- a/mysql-test/disabled.def +++ b/mysql-test/disabled.def @@ -22,3 +22,9 @@ innodb-wl5522-debug-zip : broken upstream innodb_bug12902967 : broken upstream file_contents : MDEV-6526 these files are not installed anymore max_statement_time : cannot possibly work, depends on timing +galera.MW-258 : MDEV-11229 +galera.galera_as_master : MDEV-11229 +galera.MW-44 : MDEV-11229 +galera.galera_gcs_fc_limit : MDEV-11229 +galera.galera_roles : MDEV-11229 +galera.galera_lock_table : MDEV-11229 From cbfb3f9cb17e85d63a4f3225fe3de9410f1f0d25 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Thu, 3 Nov 2016 19:30:02 -0400 Subject: [PATCH 185/295] Move disabled galera tests to galera/disabled.def --- mysql-test/disabled.def | 6 ------ mysql-test/suite/galera/disabled.def | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mysql-test/disabled.def b/mysql-test/disabled.def index 6de5646d05a16..e82ddcf92d1bf 100644 --- a/mysql-test/disabled.def +++ b/mysql-test/disabled.def @@ -22,9 +22,3 @@ innodb-wl5522-debug-zip : broken upstream innodb_bug12902967 : broken upstream file_contents : MDEV-6526 these files are not installed anymore max_statement_time : cannot possibly work, depends on timing -galera.MW-258 : MDEV-11229 -galera.galera_as_master : MDEV-11229 -galera.MW-44 : MDEV-11229 -galera.galera_gcs_fc_limit : MDEV-11229 -galera.galera_roles : MDEV-11229 -galera.galera_lock_table : MDEV-11229 diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 38bd16ed1f39f..d5016eab156f6 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -35,3 +35,10 @@ galera_sst_xtrabackup-v2-options : TODO: Fix test case galera_sst_xtrabackup-v2 : TODO: Fix test case galera_sst_xtrabackup-v2_encrypt_with_key : TODO: Fix test case mysql-wsrep#33 : TODO: investigate +MW-258 : MDEV-11229 +galera_as_master : MDEV-11229 +MW-44 : MDEV-11229 +galera_gcs_fc_limit : MDEV-11229 +galera_roles : MDEV-11229 +galera_lock_table : MDEV-11229 + From ee0094d2fd48dac0c6933a7d6ab364bf0c733aa8 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 4 Nov 2016 13:33:28 +0300 Subject: [PATCH 186/295] MDEV-10985, MDEV-10986 - sphinx tests have not been maintained --- storage/sphinx/mysql-test/sphinx/disabled.def | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 storage/sphinx/mysql-test/sphinx/disabled.def diff --git a/storage/sphinx/mysql-test/sphinx/disabled.def b/storage/sphinx/mysql-test/sphinx/disabled.def new file mode 100644 index 0000000000000..a85b8b71e5203 --- /dev/null +++ b/storage/sphinx/mysql-test/sphinx/disabled.def @@ -0,0 +1,2 @@ +sphinx : MDEV-10986, MDEV-10985 +union-5539 : MDEV-10986, MDEV-10985 From 1ddd1b5d372492edfadd220ddcf30949b0efb02c Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 4 Nov 2016 13:41:26 +0300 Subject: [PATCH 187/295] Add sphinx tests to the list, just in case they are run somewhere --- mysql-test/unstable-tests | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 528c36a4a1301..8c79ab0f29641 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -300,6 +300,10 @@ spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x #---------------------------------------------------------------- +sphinx.* : MDEV-10985, MDEV-10986 - Tests have not been maintained + +#---------------------------------------------------------------- + storage_engine* : Tests are not always timely maintained #---------------------------------------------------------------- From 5fda340dc378a8542d55d4734d68fcc918549175 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 4 Nov 2016 14:04:24 +0300 Subject: [PATCH 188/295] Remove galera suites from default list for now, tests are unstable --- mysql-test/mysql-test-run.pl | 2 -- 1 file changed, 2 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 16d8f9bb25293..2bd89f5ae4939 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -175,8 +175,6 @@ END federated- funcs_1- funcs_2- - galera- - galera_3nodes- handler- heap- innodb- From 717f212840a35b2da228d0cc2877b8e1b32204a3 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 4 Nov 2016 12:33:42 +0100 Subject: [PATCH 189/295] MDEV-10863: parallel replication tries to continue from wrong position This occured when the SQL thread (but not the IO thread) stops while GTID and parallel replication are used with multiple domain ids in the GTID position, and is restarted. In this case, the SQL needs to start some way back in the relay log, applying or skipping events within each replication domain as appropriate. The SQL threads starts at the beginning of an old relay log file, and this position may be in the middle of an event group. The bug was that such partial event group could be re-applied, causing replication corruption. This patch fixes the issue, by making sure to skip any initial events that were part of an earlier (already applied) event group. --- mysql-test/suite/rpl/r/rpl_mdev10863.result | 39 ++++++++ mysql-test/suite/rpl/t/rpl_mdev10863.test | 104 ++++++++++++++++++++ sql/slave.cc | 17 +++- 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/rpl/r/rpl_mdev10863.result create mode 100644 mysql-test/suite/rpl/t/rpl_mdev10863.test diff --git a/mysql-test/suite/rpl/r/rpl_mdev10863.result b/mysql-test/suite/rpl/r/rpl_mdev10863.result new file mode 100644 index 0000000000000..475057d4c297f --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_mdev10863.result @@ -0,0 +1,39 @@ +include/rpl_init.inc [topology=1->2] +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +SET @old_max_relay= @@GLOBAL.max_relay_log_size; +SET GLOBAL max_relay_log_size = 4096; +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY, b VARCHAR(100)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, "a"); +*** Create a long transaction that will span a relay log file. *** +SET @old_domain= @@gtid_domain_id; +SET gtid_domain_id=10; +INSERT INTO t1 VALUES (10000, "domain 10"); +SET gtid_domain_id=20; +INSERT INTO t1 VALUES (20000, "domain 20"); +SET gtid_domain_id=@old_domain; +BEGIN; +[lots of inserts omitted] +COMMIT; +BEGIN; +[lots of inserts omitted] +COMMIT; +include/stop_slave_sql.inc +START SLAVE SQL_THREAD; +include/wait_for_slave_to_start.inc +INSERT INTO t1 VALUES (100000, "More stuffs."); +INSERT INTO t1 VALUES (100001, "And even more"); +SELECT * FROM t1 WHERE a >= 100000 ORDER BY a; +a b +100000 More stuffs. +100001 And even more +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +SET GLOBAL max_relay_log_size= @old_max_relay; +include/start_slave.inc +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mdev10863.test b/mysql-test/suite/rpl/t/rpl_mdev10863.test new file mode 100644 index 0000000000000..796e770672d79 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_mdev10863.test @@ -0,0 +1,104 @@ +--source include/have_innodb.inc +--let $rpl_topology=1->2 +--source include/rpl_init.inc + +# Test various aspects of parallel replication. + +--connection server_2 +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +SET @old_max_relay= @@GLOBAL.max_relay_log_size; +SET GLOBAL max_relay_log_size = 4096; +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +--connection server_1 +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY, b VARCHAR(100)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, "a"); +--save_master_pos + +--connection server_2 +--sync_with_master + +--echo *** Create a long transaction that will span a relay log file. *** +--connection server_1 + +# Add some transactions in separate domains, that will cause the need to +# have a multi-valued restart position in the relay log for the SQL thread. +SET @old_domain= @@gtid_domain_id; +SET gtid_domain_id=10; +INSERT INTO t1 VALUES (10000, "domain 10"); +SET gtid_domain_id=20; +INSERT INTO t1 VALUES (20000, "domain 20"); +SET gtid_domain_id=@old_domain; + +BEGIN; +--echo [lots of inserts omitted] +--disable_query_log +--let $count = 500 +while ($count) { + eval INSERT INTO t1 VALUES (1000+$count, REPEAT("hulubulu??!?", 8)); + dec $count; +} +--enable_query_log +COMMIT; + +--save_master_pos + +--connection server_2 +--sync_with_master + +--connection server_1 +# Now do another one, to make the inuse_relaylog proceed to somewhere inside +# the first large transaction. + +BEGIN; +--echo [lots of inserts omitted] +--disable_query_log +--let $count = 500 +while ($count) { + eval INSERT INTO t1 VALUES (2000+$count, REPEAT("hulubulu??!?", 8)); + dec $count; +} +--enable_query_log +COMMIT; + +--save_master_pos + +--connection server_2 +--sync_with_master + + +# Stop and restart the SQL thread only. +# The bug was that the SQL thread would restart at the start +# of a relay log file, which could be in the middle of an event group. +# This way, part of that event group could be wrongly re-applied. + +--source include/stop_slave_sql.inc +START SLAVE SQL_THREAD; +--source include/wait_for_slave_to_start.inc + + +--connection server_1 +INSERT INTO t1 VALUES (100000, "More stuffs."); +INSERT INTO t1 VALUES (100001, "And even more"); +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t1 WHERE a >= 100000 ORDER BY a; + + +# Clean up. +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +SET GLOBAL max_relay_log_size= @old_max_relay; +--source include/start_slave.inc + +--connection server_1 +DROP TABLE t1; + +--source include/rpl_end.inc diff --git a/sql/slave.cc b/sql/slave.cc index a124ca6be7e34..04d62749fb188 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -4546,7 +4546,22 @@ pthread_handler_t handle_slave_sql(void *arg) serial_rgi->gtid_sub_id= 0; serial_rgi->gtid_pending= false; - rli->gtid_skip_flag = GTID_SKIP_NOT; + if (mi->using_gtid != Master_info::USE_GTID_NO && + opt_slave_parallel_threads > 0 && + rli->restart_gtid_pos.count() > 0) + { + /* + With parallel replication in GTID mode, if we have a multi-domain GTID + position, we need to start some way back in the relay log and skip any + GTID that was already applied before. Since event groups can be split + across multiple relay logs, this earlier starting point may be in the + middle of an already applied event group, so we also need to skip any + remaining part of such group. + */ + rli->gtid_skip_flag = GTID_SKIP_TRANSACTION; + } + else + rli->gtid_skip_flag = GTID_SKIP_NOT; if (init_relay_log_pos(rli, rli->group_relay_log_name, rli->group_relay_log_pos, From 909e2392b2b31888a8410d6d692fc520243c8497 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 7 Nov 2016 10:51:35 -0500 Subject: [PATCH 190/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e578c56c4e43d..7f6861d96b725 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=19 +MYSQL_VERSION_PATCH=20 From 6ae3dd6fb4c466e0facb26261349eb9c9debd6e5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 9 Nov 2016 00:10:45 +0100 Subject: [PATCH 191/295] AWS Key management plugin does not build on Centos7. Building AWS C++ SDK as external project needs CMake 2.8.12 Centos7 out of the box has 2.8.11, thus the build fails. Fixed check for required CMake version. --- plugin/aws_key_management/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin/aws_key_management/CMakeLists.txt b/plugin/aws_key_management/CMakeLists.txt index 83285ee6cc9b7..6181b7443cd26 100644 --- a/plugin/aws_key_management/CMakeLists.txt +++ b/plugin/aws_key_management/CMakeLists.txt @@ -68,7 +68,8 @@ IF(AWS_CPP_SDK_CORE AND AWS_CPP_SDK_KMS AND HAVE_AWS_HEADERS) SET(AWS_SDK_LIBS ${AWS_CPP_SDK_CORE} ${AWS_CPP_SDK_KMS}) ELSE() # Build from source, using ExternalProject_Add - IF(CMAKE_VERSION VERSION_LESS "2.8.8") + # AWS C++ SDK requires cmake 2.8.12 + IF(CMAKE_VERSION VERSION_LESS "2.8.12") SKIP_AWS_PLUGIN("CMake is too old") ENDIF() FIND_PACKAGE(Git) From 1fee0171bcc9a71eb4f1e503232682e232c0bf6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 9 Nov 2016 15:23:25 +0200 Subject: [PATCH 192/295] MDEV-10692: InnoDB: Failing assertion: lock->trx->lock.wait_lock == lock When we enter here wait_lock could be already gone i.e. NULL, that should be allowed. --- storage/innobase/lock/lock0lock.cc | 27 ++++++++++++++++++++++++++- storage/xtradb/lock/lock0lock.cc | 26 +++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 198e4ebf6f27e..c4f9b767afee4 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -900,10 +900,35 @@ lock_reset_lock_and_trx_wait( /*=========================*/ lock_t* lock) /*!< in/out: record lock */ { - ut_ad(lock->trx->lock.wait_lock == lock); ut_ad(lock_get_wait(lock)); ut_ad(lock_mutex_own()); + if (lock->trx->lock.wait_lock && + lock->trx->lock.wait_lock != lock) { + const char* stmt=NULL; + const char* stmt2=NULL; + size_t stmt_len; + trx_id_t trx_id = 0; + stmt = innobase_get_stmt(lock->trx->mysql_thd, &stmt_len); + + if (lock->trx->lock.wait_lock && + lock->trx->lock.wait_lock->trx) { + trx_id = lock->trx->lock.wait_lock->trx->id; + stmt2 = innobase_get_stmt(lock->trx->lock.wait_lock->trx->mysql_thd, &stmt_len); + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Trx id %lu is waiting a lock in statement %s" + " for this trx id %lu and statement %s wait_lock %p", + lock->trx->id, + stmt ? stmt : "NULL", + trx_id, + stmt2 ? stmt2 : "NULL", + lock->trx->lock.wait_lock); + + ut_ad(lock->trx->lock.wait_lock == lock); + } + lock->trx->lock.wait_lock = NULL; lock->type_mode &= ~LOCK_WAIT; } diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index e626821ed7eee..c56bc5e52edc6 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -912,10 +912,34 @@ lock_reset_lock_and_trx_wait( /*=========================*/ lock_t* lock) /*!< in/out: record lock */ { - ut_ad(lock->trx->lock.wait_lock == lock); ut_ad(lock_get_wait(lock)); ut_ad(lock_mutex_own()); + if (lock->trx->lock.wait_lock && + lock->trx->lock.wait_lock != lock) { + const char* stmt=NULL; + const char* stmt2=NULL; + size_t stmt_len; + trx_id_t trx_id = 0; + stmt = innobase_get_stmt(lock->trx->mysql_thd, &stmt_len); + + if (lock->trx->lock.wait_lock && + lock->trx->lock.wait_lock->trx) { + trx_id = lock->trx->lock.wait_lock->trx->id; + stmt2 = innobase_get_stmt(lock->trx->lock.wait_lock->trx->mysql_thd, &stmt_len); + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Trx id %lu is waiting a lock in statement %s" + " for this trx id %lu and statement %s wait_lock %p", + lock->trx->id, + stmt ? stmt : "NULL", + trx_id, + stmt2 ? stmt2 : "NULL", + lock->trx->lock.wait_lock); + ut_ad(lock->trx->lock.wait_lock == lock); + } + lock->trx->lock.wait_lock = NULL; lock->type_mode &= ~LOCK_WAIT; } From 1d9b043a1f5db7ff229d5200652cff7a78ea6266 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 10 Nov 2016 18:15:36 +0400 Subject: [PATCH 193/295] A join patch for MDEV-10780 and MDEV-11265 MDEV-10780 Server crashes in in create_tmp_table MDEV-11265 Access defied when CREATE VIIEW v1 AS SELECT DEFAULT(column) FROM t1 Item_default_value and Item_insert_value erroneously derive from Item_field but forgot to override some methods that apply only to true fields, so the server code mixes Item_{default|insert}_value instances with real table fields (i.e. true Item_field) in some cases. Overriding a few methods to avoid this. TODO: we should eventually derive Item_default_value (and Item_insert_value) directly from Item, as they don't really need the entire Item_field, Item_ident and Item_result_field functionality. Only the member "Field *field" related functionality is actually needed, like val_xxx(), is_null(), get_geometry_type(), charset_for_protocol(), etc. --- mysql-test/r/default.result | 41 +++++++++++++++++++++++++++++++++++ mysql-test/t/default.test | 43 +++++++++++++++++++++++++++++++++++++ sql/item.h | 6 ++++++ 3 files changed, 90 insertions(+) diff --git a/mysql-test/r/default.result b/mysql-test/r/default.result index 9afffe4c3bc1e..ad3569e7b3dc6 100644 --- a/mysql-test/r/default.result +++ b/mysql-test/r/default.result @@ -220,3 +220,44 @@ NULL 10 drop table t1, t2; End of 5.0 tests. +# +# Start of 10.0 tests +# +# +# MDEV-11265 Access defied when CREATE VIIEW v1 AS SELECT DEFAULT(column) FROM t1 +# +CREATE TABLE t1 (a INT DEFAULT 10); +INSERT INTO t1 VALUES (11); +CREATE VIEW v1 AS SELECT a AS a FROM t1; +CREATE VIEW v2 AS SELECT DEFAULT(a) AS a FROM t1; +CREATE VIEW v3 AS SELECT VALUES(a) AS a FROM t1; +SELECT * FROM v1; +a +11 +SELECT * FROM v2; +a +10 +SELECT * FROM v3; +a +NULL +UPDATE v2 SET a=123; +ERROR HY000: Column 'a' is not updatable +UPDATE v3 SET a=123; +ERROR HY000: Column 'a' is not updatable +DROP VIEW v3; +DROP VIEW v2; +DROP VIEW v1; +DROP TABLE t1; +# +# MDEV-10780 Server crashes in in create_tmp_table +# +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=MyISAM; +INSERT INTO t1 VALUES (); +INSERT INTO t1 VALUES (); +SELECT DISTINCT DEFAULT (pk) FROM t1 GROUP BY RAND() WITH ROLLUP; +DEFAULT (pk) +0 +DROP TABLE t1; +# +# End of 10.0 tests +# diff --git a/mysql-test/t/default.test b/mysql-test/t/default.test index b719cb834481c..f386222e49752 100644 --- a/mysql-test/t/default.test +++ b/mysql-test/t/default.test @@ -166,3 +166,46 @@ drop table t1, t2; --echo End of 5.0 tests. +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-11265 Access defied when CREATE VIIEW v1 AS SELECT DEFAULT(column) FROM t1 +--echo # + +CREATE TABLE t1 (a INT DEFAULT 10); +INSERT INTO t1 VALUES (11); +CREATE VIEW v1 AS SELECT a AS a FROM t1; +CREATE VIEW v2 AS SELECT DEFAULT(a) AS a FROM t1; +CREATE VIEW v3 AS SELECT VALUES(a) AS a FROM t1; +SELECT * FROM v1; +SELECT * FROM v2; +SELECT * FROM v3; +--error ER_NONUPDATEABLE_COLUMN +UPDATE v2 SET a=123; +--error ER_NONUPDATEABLE_COLUMN +UPDATE v3 SET a=123; +DROP VIEW v3; +DROP VIEW v2; +DROP VIEW v1; +DROP TABLE t1; + +--echo # +--echo # MDEV-10780 Server crashes in in create_tmp_table +--echo # + +# Note, the problem was not repeatable with a non-fresh connection. +--connect (con1,127.0.0.1,root,,test) +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=MyISAM; +INSERT INTO t1 VALUES (); +INSERT INTO t1 VALUES (); +SELECT DISTINCT DEFAULT (pk) FROM t1 GROUP BY RAND() WITH ROLLUP; +--disconnect con1 +--connection default +DROP TABLE t1; + +--echo # +--echo # End of 10.0 tests +--echo # + diff --git a/sql/item.h b/sql/item.h index e402112b72515..64870ca15a83f 100644 --- a/sql/item.h +++ b/sql/item.h @@ -4326,6 +4326,10 @@ class Item_default_value : public Item_field int save_in_field(Field *field_arg, bool no_conversions); table_map used_tables() const { return (table_map)0L; } + Field *get_tmp_table_field() { return 0; } + Item *get_tmp_table_item(THD *thd) { return this; } + Item_field *field_for_view_update() { return 0; } + bool walk(Item_processor processor, bool walk_subquery, uchar *args) { return (arg && arg->walk(processor, walk_subquery, args)) || @@ -4367,6 +4371,8 @@ class Item_insert_value : public Item_field */ table_map used_tables() const { return RAND_TABLE_BIT; } + Item_field *field_for_view_update() { return 0; } + bool walk(Item_processor processor, bool walk_subquery, uchar *args) { return arg->walk(processor, walk_subquery, args) || From 10aee66896a127da599a91f389820937cb4ba832 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 10 Nov 2016 23:47:42 +0000 Subject: [PATCH 194/295] MDEV-11248 Fix passing offset parameter to my_file_pread in read_ddl_log_file_entry --- sql/sql_table.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 2cd36ceb4de30..1936cbc56f8c5 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -684,8 +684,8 @@ static bool read_ddl_log_file_entry(uchar *file_entry_buf, DBUG_ENTER("read_ddl_log_file_entry"); DBUG_ASSERT(io_size >= size); - if (mysql_file_pread(file_id, file_entry_buf, size, io_size * entry_no, - MYF(MY_WME)) != size) + if (mysql_file_pread(file_id, file_entry_buf, size, ((my_off_t)io_size) * entry_no, + MYF(MY_FNABP)) != size) error= TRUE; DBUG_RETURN(error); } From 96b62b55149a9297f32c3aad99ece613cc3f788f Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 11 Nov 2016 20:55:03 -0800 Subject: [PATCH 195/295] Fixed bug mdev-11161. The flag TABLE_LIST::fill_me must be reset to false at the prepare phase for any materialized derived table used in the executed query. Otherwise if the optimizer decides to generate a key for such a table it is generated only for the first execution of the query. --- mysql-test/r/derived_view.result | 83 ++++++++++++++++++++++++++++++++ mysql-test/t/derived_view.test | 54 +++++++++++++++++++++ sql/sql_derived.cc | 2 + 3 files changed, 139 insertions(+) diff --git a/mysql-test/r/derived_view.result b/mysql-test/r/derived_view.result index 230660f2fcfae..d993086299eaf 100644 --- a/mysql-test/r/derived_view.result +++ b/mysql-test/r/derived_view.result @@ -2496,5 +2496,88 @@ DROP TABLE t1,t2; # # end of 5.3 tests # +# +# Bug mdev-11161: The second execution of prepared statement +# does not use generated key for materialized +# derived table / view +# (actually this is a 5.3 bug.) +# +create table t1 ( +mat_id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, +matintnum CHAR(6) NOT NULL, +test MEDIUMINT UNSIGNED NULL +); +create table t2 ( +mat_id MEDIUMINT UNSIGNED NOT NULL, +pla_id MEDIUMINT UNSIGNED NOT NULL +); +insert into t1 values +(NULL, 'a', 1), (NULL, 'b', 2), (NULL, 'c', 3), (NULL, 'd', 4), +(NULL, 'e', 5), (NULL, 'f', 6), (NULL, 'g', 7), (NULL, 'h', 8), +(NULL, 'i', 9); +insert into t2 values +(1, 100), (1, 101), (1, 102), (2, 100), (2, 103), (2, 104), +(3, 101), (3, 102), (3, 105); +explain +SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id +FROM t1 m2 +INNER JOIN +(SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum +FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id +GROUP BY mp.pla_id) d +ON d.matintnum=m2.matintnum; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY m2 ALL NULL NULL NULL NULL 9 +1 PRIMARY ref key0 key0 7 test.m2.matintnum 2 +2 DERIVED mp ALL NULL NULL NULL NULL 9 Using temporary; Using filesort +2 DERIVED m1 eq_ref PRIMARY PRIMARY 3 test.mp.mat_id 1 +prepare stmt1 from +"SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id + FROM t1 m2 + INNER JOIN + (SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum + FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id + GROUP BY mp.pla_id) d + ON d.matintnum=m2.matintnum"; +flush status; +execute stmt1; +pla_id mat_id +102 1 +101 1 +100 1 +104 2 +103 2 +105 3 +show status like '%Handler_read%'; +Variable_name Value +Handler_read_first 0 +Handler_read_key 21 +Handler_read_last 0 +Handler_read_next 6 +Handler_read_prev 0 +Handler_read_rnd 6 +Handler_read_rnd_deleted 0 +Handler_read_rnd_next 27 +flush status; +execute stmt1; +pla_id mat_id +102 1 +101 1 +100 1 +104 2 +103 2 +105 3 +show status like '%Handler_read%'; +Variable_name Value +Handler_read_first 0 +Handler_read_key 21 +Handler_read_last 0 +Handler_read_next 6 +Handler_read_prev 0 +Handler_read_rnd 6 +Handler_read_rnd_deleted 0 +Handler_read_rnd_next 27 +deallocate prepare stmt1; +drop table t1,t2; set optimizer_switch=@exit_optimizer_switch; set join_cache_level=@exit_join_cache_level; diff --git a/mysql-test/t/derived_view.test b/mysql-test/t/derived_view.test index 67899837bb2e4..d017f847af978 100644 --- a/mysql-test/t/derived_view.test +++ b/mysql-test/t/derived_view.test @@ -1827,6 +1827,60 @@ DROP TABLE t1,t2; --echo # end of 5.3 tests --echo # +--echo # +--echo # Bug mdev-11161: The second execution of prepared statement +--echo # does not use generated key for materialized +--echo # derived table / view +--echo # (actually this is a 5.3 bug.) +--echo # + +create table t1 ( + mat_id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + matintnum CHAR(6) NOT NULL, + test MEDIUMINT UNSIGNED NULL +); +create table t2 ( + mat_id MEDIUMINT UNSIGNED NOT NULL, + pla_id MEDIUMINT UNSIGNED NOT NULL +); +insert into t1 values + (NULL, 'a', 1), (NULL, 'b', 2), (NULL, 'c', 3), (NULL, 'd', 4), + (NULL, 'e', 5), (NULL, 'f', 6), (NULL, 'g', 7), (NULL, 'h', 8), + (NULL, 'i', 9); +insert into t2 values + (1, 100), (1, 101), (1, 102), (2, 100), (2, 103), (2, 104), + (3, 101), (3, 102), (3, 105); + +explain + SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id + FROM t1 m2 + INNER JOIN + (SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum + FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id + GROUP BY mp.pla_id) d + ON d.matintnum=m2.matintnum; + +prepare stmt1 from +"SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id + FROM t1 m2 + INNER JOIN + (SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum + FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id + GROUP BY mp.pla_id) d + ON d.matintnum=m2.matintnum"; + +flush status; +execute stmt1; +show status like '%Handler_read%'; + +flush status; +execute stmt1; +show status like '%Handler_read%'; + +deallocate prepare stmt1; + +drop table t1,t2; + # The following command must be the last one the file set optimizer_switch=@exit_optimizer_switch; set join_cache_level=@exit_join_cache_level; diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index c6865a7116ed3..4439559996168 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -651,6 +651,8 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived) unit->derived= derived; + derived->fill_me= FALSE; + if (!(derived->derived_result= new select_union)) DBUG_RETURN(TRUE); // out of memory From adc38ed81140701f89b40b3356b5fb7f9587c2b7 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 14 Nov 2016 08:02:35 +0100 Subject: [PATCH 196/295] Restore MY_WME flag for my_pread in read_ddl_log_entry, fix errors in buildbot --- sql/sql_table.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 1936cbc56f8c5..7d2e67b5cfd18 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -685,7 +685,7 @@ static bool read_ddl_log_file_entry(uchar *file_entry_buf, DBUG_ASSERT(io_size >= size); if (mysql_file_pread(file_id, file_entry_buf, size, ((my_off_t)io_size) * entry_no, - MYF(MY_FNABP)) != size) + MYF(MY_WME)) != size) error= TRUE; DBUG_RETURN(error); } From cf29e8c55b1f39234b5e38cee28f0241a23436fd Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 14 Nov 2016 11:02:57 -0500 Subject: [PATCH 197/295] wsrep_info plugin: Fix test case --- plugin/wsrep_info/mysql-test/wsrep_info/r/plugin.result | 4 ++-- plugin/wsrep_info/mysql-test/wsrep_info/t/plugin.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin/wsrep_info/mysql-test/wsrep_info/r/plugin.result b/plugin/wsrep_info/mysql-test/wsrep_info/r/plugin.result index 1aef7c30fc773..31d66ab8b3459 100644 --- a/plugin/wsrep_info/mysql-test/wsrep_info/r/plugin.result +++ b/plugin/wsrep_info/mysql-test/wsrep_info/r/plugin.result @@ -1,7 +1,7 @@ # On node 1 SELECT * FROM INFORMATION_SCHEMA.WSREP_STATUS; NODE_INDEX NODE_STATUS CLUSTER_STATUS CLUSTER_SIZE CLUSTER_STATE_UUID CLUSTER_STATE_SEQNO CLUSTER_CONF_ID GAP PROTOCOL_VERSION - Synced Primary 2 0 2 NO 3 + Synced Primary 2 0 NO 3 SELECT * FROM INFORMATION_SCHEMA.WSREP_MEMBERSHIP ORDER BY NAME; INDEX UUID NAME ADDRESS test-node-1
@@ -9,7 +9,7 @@ INDEX UUID NAME ADDRESS # On node 2 SELECT * FROM INFORMATION_SCHEMA.WSREP_STATUS; NODE_INDEX NODE_STATUS CLUSTER_STATUS CLUSTER_SIZE CLUSTER_STATE_UUID CLUSTER_STATE_SEQNO CLUSTER_CONF_ID GAP PROTOCOL_VERSION - Synced Primary 2 0 2 YES 3 + Synced Primary 2 0 YES 3 SELECT * FROM INFORMATION_SCHEMA.WSREP_MEMBERSHIP ORDER BY NAME; INDEX UUID NAME ADDRESS test-node-1
diff --git a/plugin/wsrep_info/mysql-test/wsrep_info/t/plugin.test b/plugin/wsrep_info/mysql-test/wsrep_info/t/plugin.test index 9b06f15b12521..9ae783a957e75 100644 --- a/plugin/wsrep_info/mysql-test/wsrep_info/t/plugin.test +++ b/plugin/wsrep_info/mysql-test/wsrep_info/t/plugin.test @@ -4,7 +4,7 @@ --echo # On node 1 --connection node_1 ---replace_column 1 5 +--replace_column 1 5 7 SELECT * FROM INFORMATION_SCHEMA.WSREP_STATUS; --replace_column 1 2 4
@@ -13,7 +13,7 @@ SELECT * FROM INFORMATION_SCHEMA.WSREP_MEMBERSHIP ORDER BY NAME; --echo # On node 2 --connection node_2 ---replace_column 1 5 +--replace_column 1 5 7 SELECT * FROM INFORMATION_SCHEMA.WSREP_STATUS; --replace_column 1 2 4
From f1fcc1fc105ad0e329b8f6dd11923f99d6edd42b Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Tue, 15 Nov 2016 23:00:11 +0100 Subject: [PATCH 198/295] Back-port Master_info::using_parallel() to 10.0. This has no functional changes, but it helps avoid merge problems from 10.0 to 10.1. In 10.0, code that checks for parallel replication uses opt_slave_parallel_threads > 0, but this check needs to be mi->using_parallel() in 10.1. By using the same check in 10.0 (with unchanged semantics), merge problems to 10.1 are avoided. --- sql/log_event.cc | 2 +- sql/rpl_mi.h | 4 ++++ sql/rpl_rli.cc | 2 +- sql/slave.cc | 21 +++++++++------------ sql/sys_vars.cc | 3 +-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/sql/log_event.cc b/sql/log_event.cc index 4a1d13a8004fb..d311297f8a833 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -12829,7 +12829,7 @@ bool rpl_get_position_info(const char **log_file_name, ulonglong *log_pos, return FALSE; #else const Relay_log_info *rli= &(active_mi->rli); - if (opt_slave_parallel_threads == 0) + if (!rli->mi->using_parallel()) { *log_file_name= rli->group_master_log_name; *log_pos= rli->group_master_log_pos + diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index a27672e4c90e3..e58df0150f522 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -75,6 +75,10 @@ class Master_info : public Slave_reporting_capability return connection_name.str == 0; } static const char *using_gtid_astext(enum enum_using_gtid arg); + bool using_parallel() + { + return opt_slave_parallel_threads > 0; + } /* the variables below are needed because we can change masters on the fly */ char master_log_name[FN_REFLEN+6]; /* Room for multi-*/ diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 4aed8cdcd9474..c570105cdf608 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1231,7 +1231,7 @@ bool Relay_log_info::is_until_satisfied(THD *thd, Log_event *ev) if (ev && ev->server_id == (uint32) global_system_variables.server_id && !replicate_same_server_id) DBUG_RETURN(FALSE); - log_name= (opt_slave_parallel_threads > 0 ? + log_name= (mi->using_parallel() ? future_event_master_log_name : group_master_log_name); log_pos= ((!ev)? group_master_log_pos : (get_flag(IN_TRANSACTION) || !ev->log_pos) ? diff --git a/sql/slave.cc b/sql/slave.cc index 04d62749fb188..e6a0ac086a8d0 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -620,8 +620,7 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) if (thread_mask & (SLAVE_SQL|SLAVE_FORCE_ALL)) { DBUG_PRINT("info",("Terminating SQL thread")); - if (opt_slave_parallel_threads > 0 && - mi->rli.abort_slave && mi->rli.stop_for_until) + if (mi->using_parallel() && mi->rli.abort_slave && mi->rli.stop_for_until) { mi->rli.stop_for_until= false; mi->rli.parallel.stop_during_until(); @@ -2726,8 +2725,7 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, else { idle= mi->rli.sql_thread_caught_up; - if (opt_slave_parallel_threads > 0 && idle && - !mi->rli.parallel.workers_idle()) + if (mi->using_parallel() && idle && !mi->rli.parallel.workers_idle()) idle= false; } if (idle) @@ -3517,7 +3515,7 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, the user might be surprised to see a claim that the slave is up to date long before those queued events are actually executed. */ - if (opt_slave_parallel_threads == 0 && + if (!rli->mi->using_parallel() && !(ev->is_artificial_event() || ev->is_relay_log_event() || (ev->when == 0))) { rli->last_master_timestamp= ev->when + (time_t) ev->exec_time; @@ -3568,7 +3566,7 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, update_state_of_relay_log(rli, ev); - if (opt_slave_parallel_threads > 0) + if (rli->mi->using_parallel()) { int res= rli->parallel.do_event(serial_rgi, ev, event_size); if (res >= 0) @@ -4546,8 +4544,7 @@ pthread_handler_t handle_slave_sql(void *arg) serial_rgi->gtid_sub_id= 0; serial_rgi->gtid_pending= false; - if (mi->using_gtid != Master_info::USE_GTID_NO && - opt_slave_parallel_threads > 0 && + if (mi->using_gtid != Master_info::USE_GTID_NO && mi->using_parallel() && rli->restart_gtid_pos.count() > 0) { /* @@ -4734,7 +4731,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, } } - if (opt_slave_parallel_threads > 0) + if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); /* Thread stopped. Print the current replication position to the log */ @@ -4760,7 +4757,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, (We want the first one to be before the printout of stop position to get the correct position printed.) */ - if (opt_slave_parallel_threads > 0) + if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); /* @@ -4784,7 +4781,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, ulong domain_count; flush_relay_log_info(rli); - if (opt_slave_parallel_threads > 0) + if (mi->using_parallel()) { /* In parallel replication GTID mode, we may stop with different domains @@ -6475,7 +6472,7 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) llstr(my_b_tell(cur_log),llbuf1), llstr(rli->event_relay_log_pos,llbuf2))); DBUG_ASSERT(my_b_tell(cur_log) >= BIN_LOG_HEADER_SIZE); - DBUG_ASSERT(opt_slave_parallel_threads > 0 || + DBUG_ASSERT(rli->mi->using_parallel() || my_b_tell(cur_log) == rli->event_relay_log_pos); } #endif diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 689d35c9cc389..089b6cd9c7e43 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -4351,8 +4351,7 @@ static bool update_slave_skip_counter(sys_var *self, THD *thd, Master_info *mi) mi->connection_name.str); return true; } - if (mi->using_gtid != Master_info::USE_GTID_NO && - opt_slave_parallel_threads > 0) + if (mi->using_gtid != Master_info::USE_GTID_NO && mi->using_parallel()) { ulong domain_count; mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); From 390f2a013b1b223bdce2cd9011a799131eaf13ce Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 16 Nov 2016 11:00:38 +0100 Subject: [PATCH 199/295] Fix incorrect reading of events from relaylog in parallel replication. The SQL thread keeps track of the position in the current relay log from which to read the next event. This position is not normally used, but a certain interaction with the IO thread can cause the SQL thread to re-open the relay log and seek to the stored position. In parallel replication, there were a couple of places where the position was not updated. This created a race where a re-open of the relay log could seek to the wrong position and start re-reading and processing events already handled once, causing various kinds of problems. Fix this by moving the position update into a single place in apply_event_and_update_pos(), which should ensure that the position is always updated in the parallel replication case. This problem was found from the testcase of MDEV-10863, but it is logically a separate problem. --- sql/rpl_parallel.cc | 1 - sql/slave.cc | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 98712f18c1b86..23f61a82a9088 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -2614,7 +2614,6 @@ rpl_parallel::do_event(rpl_group_info *serial_rgi, Log_event *ev, /* Queue the event for processing. */ - rli->event_relay_log_pos= rli->future_event_relay_log_pos; qev->ir= rli->last_inuse_relaylog; ++qev->ir->queued_count; cur_thread->enqueue(qev); diff --git a/sql/slave.cc b/sql/slave.cc index e6a0ac086a8d0..65bcdc48c6a69 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3569,6 +3569,13 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, if (rli->mi->using_parallel()) { int res= rli->parallel.do_event(serial_rgi, ev, event_size); + /* + In parallel replication, we need to update the relay log position + immediately so that it will be the correct position from which to + read the next event. + */ + if (res == 0) + rli->event_relay_log_pos= rli->future_event_relay_log_pos; if (res >= 0) DBUG_RETURN(res); /* From af05becb4c78d843f34328931704d82faaa8f213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 16 Nov 2016 13:52:41 +0200 Subject: [PATCH 200/295] MDEV-10771: Test innodb_defragment_fill_factor does not work correctly Added more columns to stabilice the test case. --- .../r/innodb_defragment_fill_factor.result | 126 ++++++++++------ .../t/innodb_defragment_fill_factor.test | 142 ++++++++++++------ 2 files changed, 176 insertions(+), 92 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result b/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result index 04e41046a6ac4..e1e616a7e6ff1 100644 --- a/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result +++ b/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result @@ -1,8 +1,8 @@ DROP TABLE if exists t1; DROP TABLE if exists t2; Testing tables with large records -CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB; -INSERT INTO t1 VALUES (1, REPEAT('A', 256)); +CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), c VARCHAR(256), KEY SECOND(a, b,c)) ENGINE=INNODB; +INSERT INTO t1 VALUES (1, REPEAT('A', 256), REPEAT('B', 256)); INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; @@ -16,9 +16,6 @@ INSERT INTO t1 (b) SELECT b from t1; optimize table t1; Table Op Msg_type Msg_text test.t1 optimize status OK -select sleep(1); -sleep(1) -0 select count(*) from t1; count(*) 927 @@ -26,51 +23,94 @@ select count(*) from t1 force index (second); count(*) 927 # A few more insertions on the page should not cause a page split. -insert into t1 values (81, REPEAT('A', 256)); -insert into t1 values (83, REPEAT('A', 256)); -insert into t1 values (87, REPEAT('A', 256)); -insert into t1 values (82, REPEAT('A', 256)); -insert into t1 values (86, REPEAT('A', 256)); -# More insertions will cause page splits -insert into t1 values (88, REPEAT('A', 256)); -insert into t1 values (85, REPEAT('A', 256)); -insert into t1 values (84, REPEAT('A', 256)); +insert into t1 values (81, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (83, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (87, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (82, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (86, REPEAT('A', 256), REPEAT('B', 256)); +# Insert more rows to cause a page split +insert into t1 values (180, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (181, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (182, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (183, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (184, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (185, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (186, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (187, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (188, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (189, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (190, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (191, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (192, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (193, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (194, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (195, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (196, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (197, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (198, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (199, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (200, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (201, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (202, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (203, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (204, REPEAT('A', 256), REPEAT('B', 256)); DROP TABLE t1; Testing table with small records -CREATE TABLE t2 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARchar(16), KEY SECOND(a,b)) ENGINE=INNODB; +CREATE TABLE t2 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(16), c VARCHAR(32), KEY SECOND(a,b,c)) ENGINE=INNODB; optimize table t2; Table Op Msg_type Msg_text test.t2 optimize status OK -select sleep(1); -sleep(1) -0 +select count(*) from t2; +count(*) +3701 select count(*) from t2 force index(second); count(*) 3701 The page should have room for about 20 insertions -insert into t2 values(1181, REPEAT('A', 16)); -insert into t2 values(1191, REPEAT('A', 16)); -insert into t2 values(1182, REPEAT('A', 16)); -insert into t2 values(1192, REPEAT('A', 16)); -insert into t2 values(1183, REPEAT('A', 16)); -insert into t2 values(1193, REPEAT('A', 16)); -insert into t2 values(1184, REPEAT('A', 16)); -insert into t2 values(1194, REPEAT('A', 16)); -insert into t2 values(1185, REPEAT('A', 16)); -insert into t2 values(1195, REPEAT('A', 16)); -insert into t2 values(1186, REPEAT('A', 16)); -insert into t2 values(1196, REPEAT('A', 16)); -insert into t2 values(1187, REPEAT('A', 16)); -insert into t2 values(1197, REPEAT('A', 16)); -insert into t2 values(1188, REPEAT('A', 16)); -insert into t2 values(1198, REPEAT('A', 16)); -insert into t2 values(1189, REPEAT('A', 16)); -insert into t2 values(1199, REPEAT('A', 16)); -insert into t2 values(1190, REPEAT('A', 16)); -insert into t2 values(1180, REPEAT('A', 16)); -More insertions will cause page split. -insert into t2 values(1280, REPEAT('A', 16)); -insert into t2 values(1290, REPEAT('A', 16)); -insert into t2 values(1281, REPEAT('A', 16)); -insert into t2 values(1291, REPEAT('A', 16)); +insert into t2 values(1181, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1191, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1182, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1192, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1183, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1193, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1184, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1194, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1185, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1195, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1186, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1196, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1187, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1197, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1188, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1198, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1189, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1199, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1190, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1180, REPEAT('A', 16), REPEAT('B',32)); +# Insert more rows to cause a page split +insert into t2 values (180, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (181, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (182, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (183, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (184, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (185, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (186, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (187, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (188, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (189, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (190, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (191, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (192, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (193, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (194, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (195, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (196, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (197, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (198, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (199, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (200, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (201, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (202, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (203, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (204, REPEAT('A', 16), REPEAT('B', 32)); DROP TABLE t2; diff --git a/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test b/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test index a46ca124f51bf..2edc8a45c028a 100644 --- a/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test +++ b/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test @@ -11,10 +11,10 @@ DROP TABLE if exists t2; --echo Testing tables with large records # Create table. -CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB; +CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), c VARCHAR(256), KEY SECOND(a, b,c)) ENGINE=INNODB; # Populate table. -INSERT INTO t1 VALUES (1, REPEAT('A', 256)); +INSERT INTO t1 VALUES (1, REPEAT('A', 256), REPEAT('B', 256)); INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; @@ -36,28 +36,24 @@ while ($size) } --enable_query_log +--source include/wait_innodb_all_purged.inc --source include/restart_mysqld.inc optimize table t1; -select sleep(1); ---source include/restart_mysqld.inc select count(*) from t1; -# After deletion & defragmentation, there are 800 records left. Each page can hold about 57 records. We fill the page 90% full, -# so there should be less than 16 pages total. --let $primary_before = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t1%' and index_name = 'PRIMARY', Value, 1) select count(*) from t1 force index (second); -# secondary index is slightly bigger than primary index so the number of pages should be similar. --let $second_before = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t1%' and index_name = 'second', Value, 1) --echo # A few more insertions on the page should not cause a page split. -insert into t1 values (81, REPEAT('A', 256)); -insert into t1 values (83, REPEAT('A', 256)); -insert into t1 values (87, REPEAT('A', 256)); -insert into t1 values (82, REPEAT('A', 256)); -insert into t1 values (86, REPEAT('A', 256)); +insert into t1 values (81, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (83, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (87, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (82, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (86, REPEAT('A', 256), REPEAT('B', 256)); --let $primary_after = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t1%' and index_name = 'PRIMARY', Value, 1) --let $second_after = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t1%' and index_name = 'second', Value, 1) @@ -69,28 +65,52 @@ if ($second_before != $second_after) { --echo Insertion caused page split on second, which should be avoided by innodb_defragment_fill_factor. } ---echo # More insertions will cause page splits -insert into t1 values (88, REPEAT('A', 256)); -insert into t1 values (85, REPEAT('A', 256)); -insert into t1 values (84, REPEAT('A', 256)); +--echo # Insert more rows to cause a page split +insert into t1 values (180, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (181, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (182, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (183, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (184, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (185, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (186, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (187, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (188, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (189, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (190, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (191, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (192, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (193, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (194, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (195, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (196, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (197, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (198, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (199, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (200, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (201, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (202, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (203, REPEAT('A', 256), REPEAT('B', 256)); +insert into t1 values (204, REPEAT('A', 256), REPEAT('B', 256)); + --let $primary_after = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t1%' and index_name = 'PRIMARY', Value, 1) + --let $second_after = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t1%' and index_name = 'second', Value, 1) if ($primary_before == $primary_after) { - --echo Too much space are reserved on primary index. + --echo Too little space is reserved on primary index. } if ($second_before == $second_after) { - --echo Too much space are reserved on second index. + --echo Too little space is reserved on second index. } DROP TABLE t1; --echo Testing table with small records -CREATE TABLE t2 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARchar(16), KEY SECOND(a,b)) ENGINE=INNODB; +CREATE TABLE t2 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(16), c VARCHAR(32), KEY SECOND(a,b,c)) ENGINE=INNODB; # Populate table. --disable_query_log -INSERT INTO t2 VALUES (1, REPEAT('A', 16)); +INSERT INTO t2 VALUES (1, REPEAT('A', 16), REPEAT('B', 32)); INSERT INTO t2 (b) SELECT b from t2; INSERT INTO t2 (b) SELECT b from t2; INSERT INTO t2 (b) SELECT b from t2; @@ -115,36 +135,36 @@ while ($size) } --enable_query_log +--source include/wait_innodb_all_purged.inc --source include/restart_mysqld.inc optimize table t2; -select sleep(1); ---source include/restart_mysqld.inc +select count(*) from t2; select count(*) from t2 force index(second); --let $second_before = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t2%' and index_name = 'second', Value, 1) --echo The page should have room for about 20 insertions -insert into t2 values(1181, REPEAT('A', 16)); -insert into t2 values(1191, REPEAT('A', 16)); -insert into t2 values(1182, REPEAT('A', 16)); -insert into t2 values(1192, REPEAT('A', 16)); -insert into t2 values(1183, REPEAT('A', 16)); -insert into t2 values(1193, REPEAT('A', 16)); -insert into t2 values(1184, REPEAT('A', 16)); -insert into t2 values(1194, REPEAT('A', 16)); -insert into t2 values(1185, REPEAT('A', 16)); -insert into t2 values(1195, REPEAT('A', 16)); -insert into t2 values(1186, REPEAT('A', 16)); -insert into t2 values(1196, REPEAT('A', 16)); -insert into t2 values(1187, REPEAT('A', 16)); -insert into t2 values(1197, REPEAT('A', 16)); -insert into t2 values(1188, REPEAT('A', 16)); -insert into t2 values(1198, REPEAT('A', 16)); -insert into t2 values(1189, REPEAT('A', 16)); -insert into t2 values(1199, REPEAT('A', 16)); -insert into t2 values(1190, REPEAT('A', 16)); -insert into t2 values(1180, REPEAT('A', 16)); +insert into t2 values(1181, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1191, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1182, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1192, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1183, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1193, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1184, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1194, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1185, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1195, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1186, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1196, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1187, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1197, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1188, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1198, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1189, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1199, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1190, REPEAT('A', 16), REPEAT('B',32)); +insert into t2 values(1180, REPEAT('A', 16), REPEAT('B',32)); --let $second_after = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t2%' and index_name = 'second', Value, 1) @@ -152,15 +172,39 @@ if ($second_before != $second_after) { --echo Insertion caused page split on second, which should be avoided by innodb_defragment_fill_factor. } ---echo More insertions will cause page split. -insert into t2 values(1280, REPEAT('A', 16)); -insert into t2 values(1290, REPEAT('A', 16)); -insert into t2 values(1281, REPEAT('A', 16)); -insert into t2 values(1291, REPEAT('A', 16)); +--echo # Insert more rows to cause a page split +insert into t2 values (180, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (181, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (182, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (183, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (184, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (185, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (186, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (187, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (188, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (189, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (190, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (191, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (192, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (193, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (194, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (195, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (196, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (197, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (198, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (199, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (200, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (201, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (202, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (203, REPEAT('A', 16), REPEAT('B', 32)); +insert into t2 values (204, REPEAT('A', 16), REPEAT('B', 32)); --let $second_after = query_get_value(select count(*) as Value from information_schema.innodb_buffer_page where table_name like '%t2%' and index_name = 'second', Value, 1) + if ($second_before == $second_after) { - --echo Too much space are reserved on second index. + --echo Too little space is reserved on second index. } -DROP TABLE t2; \ No newline at end of file +DROP TABLE t2; + +--source include/wait_innodb_all_purged.inc From 42a398b59b5c2d08a9cc8c03d6d082a095a2cd77 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 17 Nov 2016 12:04:39 +0400 Subject: [PATCH 201/295] Fixing a typo in the patch for MDEV-10780, which caused default.test failure. Fixing the "connect" command to use "localhost" instead of "127.0.0.1" to make it work with both "mtr" and "mtr --embedded". --- mysql-test/t/default.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/t/default.test b/mysql-test/t/default.test index f386222e49752..432df9d44524f 100644 --- a/mysql-test/t/default.test +++ b/mysql-test/t/default.test @@ -196,7 +196,7 @@ DROP TABLE t1; --echo # # Note, the problem was not repeatable with a non-fresh connection. ---connect (con1,127.0.0.1,root,,test) +--connect (con1,localhost,root,,test) CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=MyISAM; INSERT INTO t1 VALUES (); INSERT INTO t1 VALUES (); From 03ddc19ab21d585f4aac16ad91e5cba39687cd31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 17 Nov 2016 15:15:20 +0200 Subject: [PATCH 202/295] MDEV-6424: MariaDB server crashes with assertion failure in file ha_innodb.c line 11652 This is not a fix, this is instrumentation to find out is MySQL frm dictionary and InnoDB data dictionary really out-of-sync when this assertion is fired, or is there some other reason (bug). --- storage/innobase/handler/ha_innodb.cc | 76 ++++++++++++++++++++++++--- storage/xtradb/handler/ha_innodb.cc | 76 ++++++++++++++++++++++++--- 2 files changed, 140 insertions(+), 12 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index af4b523872af7..59ebc411db0b9 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1998,11 +1998,16 @@ innobase_get_stmt( THD* thd, /*!< in: MySQL thread handle */ size_t* length) /*!< out: length of the SQL statement */ { - LEX_STRING* stmt; - - stmt = thd_query_string(thd); - *length = stmt->length; - return(stmt->str); + const char* query = NULL; + LEX_STRING *stmt = NULL; + if (thd) { + stmt = thd_query_string(thd); + if (stmt) { + *length = stmt->length; + query = stmt->str; + } + } + return (query); } /**********************************************************************//** @@ -6616,7 +6621,66 @@ build_template_field( UNIV_MEM_INVALID(templ, sizeof *templ); templ->col_no = i; templ->clust_rec_field_no = dict_col_get_clust_pos(col, clust_index); - ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + + /* If clustered index record field is not found, lets print out + field names and all the rest to understand why field is not found. */ + if (templ->clust_rec_field_no == ULINT_UNDEFINED) { + const char* tb_col_name = dict_table_get_col_name(clust_index->table, i); + dict_field_t* field=NULL; + size_t size = 0; + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + if (ifield && !memcmp(tb_col_name, ifield->name, + strlen(tb_col_name))) { + field = ifield; + break; + } + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Looking for field %lu name %s from table %s", + i, + (tb_col_name ? tb_col_name : "NULL"), + clust_index->table->name); + + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + ib_logf(IB_LOG_LEVEL_INFO, + "InnoDB Table %s field %lu name %s", + clust_index->table->name, + j, + (ifield ? ifield->name : "NULL")); + } + + for(ulint j=0; j < table->s->stored_fields; j++) { + ib_logf(IB_LOG_LEVEL_INFO, + "MySQL table %s field %lu name %s", + table->s->table_name.str, + j, + table->field[j]->field_name); + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Clustered record field for column %lu" + " not found table n_user_defined %d" + " index n_user_defined %d" + " InnoDB table %s field name %s" + " MySQL table %s field name %s n_fields %d" + " query %s", + i, + clust_index->n_user_defined_cols, + clust_index->table->n_cols - DATA_N_SYS_COLS, + clust_index->table->name, + (field ? field->name : "NULL"), + table->s->table_name.str, + (tb_col_name ? tb_col_name : "NULL"), + table->s->stored_fields, + innobase_get_stmt(current_thd, &size)); + + ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + } if (dict_index_is_clust(index)) { templ->rec_field_no = templ->clust_rec_field_no; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index e95fc365be8d7..705913d496205 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -2355,11 +2355,16 @@ innobase_get_stmt( THD* thd, /*!< in: MySQL thread handle */ size_t* length) /*!< out: length of the SQL statement */ { - LEX_STRING* stmt; - - stmt = thd_query_string(thd); - *length = stmt->length; - return(stmt->str); + const char* query = NULL; + LEX_STRING *stmt = NULL; + if (thd) { + stmt = thd_query_string(thd); + if (stmt) { + *length = stmt->length; + query = stmt->str; + } + } + return (query); } /**********************************************************************//** @@ -7517,7 +7522,66 @@ build_template_field( UNIV_MEM_INVALID(templ, sizeof *templ); templ->col_no = i; templ->clust_rec_field_no = dict_col_get_clust_pos(col, clust_index); - ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + + /* If clustered index record field is not found, lets print out + field names and all the rest to understand why field is not found. */ + if (templ->clust_rec_field_no == ULINT_UNDEFINED) { + const char* tb_col_name = dict_table_get_col_name(clust_index->table, i); + dict_field_t* field=NULL; + size_t size = 0; + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + if (ifield && !memcmp(tb_col_name, ifield->name, + strlen(tb_col_name))) { + field = ifield; + break; + } + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Looking for field %lu name %s from table %s", + i, + (tb_col_name ? tb_col_name : "NULL"), + clust_index->table->name); + + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + ib_logf(IB_LOG_LEVEL_INFO, + "InnoDB Table %s field %lu name %s", + clust_index->table->name, + j, + (ifield ? ifield->name : "NULL")); + } + + for(ulint j=0; j < table->s->stored_fields; j++) { + ib_logf(IB_LOG_LEVEL_INFO, + "MySQL table %s field %lu name %s", + table->s->table_name.str, + j, + table->field[j]->field_name); + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Clustered record field for column %lu" + " not found table n_user_defined %d" + " index n_user_defined %d" + " InnoDB table %s field name %s" + " MySQL table %s field name %s n_fields %d" + " query %s", + i, + clust_index->n_user_defined_cols, + clust_index->table->n_cols - DATA_N_SYS_COLS, + clust_index->table->name, + (field ? field->name : "NULL"), + table->s->table_name.str, + (tb_col_name ? tb_col_name : "NULL"), + table->s->stored_fields, + innobase_get_stmt(current_thd, &size)); + + ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + } if (dict_index_is_clust(index)) { templ->rec_field_no = templ->clust_rec_field_no; From 57058cb6772ec4205c1b0bade761f5bf65772748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 22 Nov 2016 16:38:36 +0200 Subject: [PATCH 203/295] MDEV-10377: innodb.innodb_blob_truncate fails in buildbot: Failing assertion: page_type == 34354 || page_type == 37401 || page_type == 17855 || page_type == 2 || page_type == 3 || ... Page type FIL_PAGE_TYPE_ZBLOB2 was missing from assertion. --- storage/innobase/include/fil0fil.ic | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index 5654d8f617886..ceebf6c1ab3f0 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -131,6 +131,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || + page_type == FIL_PAGE_TYPE_ZBLOB2 || page_type == FIL_PAGE_TYPE_COMPRESSED))) { uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); @@ -165,6 +166,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || + page_type == FIL_PAGE_TYPE_ZBLOB2 || page_type == FIL_PAGE_TYPE_COMPRESSED); return false; } From d145d1b6ee42ffa7f9ca2ce02478a31a09f3fe99 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 23 Nov 2016 12:29:38 +0100 Subject: [PATCH 204/295] fix bogus stalls in the lock tree for low concurrency applications Merge into the MariaDB tree the pull request from Rich Prohaska for PerconaFT. These changes are needed to get parallel replication to work with TokuDB. Once the pull request is accepted by Percona and the new upstream version enters MariaDB, this commit can be superseded. Original commit message from Rich Prohaska: 1. Fix the release before wait race The release before wait race occurs when a lock is released by transaction A after transaction B tried to acquire it but before transaction B has a chance to register it's pending lock request. There are several ways to fix this problem, but we want to optimize for the common situation of minimal lock conflicts, which is what the lock acquisition algorithm currently does. Our solution to the release before wait race is for transaction B to retry its lock request after its lock request has been added to the pending lock set. 2. Fix the retry race The retry race occurs in the current lock retry algorithm which assumes that if some transaction is running lock retry, then my transaction does not also need to run it. There is a chance that some pending lock requests will be skipped, but these lock requests will eventually time out. For applications with small numbers of concurrent transactions, timeouts will frequently occur, and the application throughput will be very small. The solution to the retry race is to use a group retry algorithm. All threads run through the retry logic. Sequence numbers are used to group retries into batches such that one transaction can run the retry logic on behalf of several transactions. This amortizes the retry cost. The sequence numbers also ensure that when a transaction releases its locks, all of the pending lock requests that it is blocking are retried. 3. Implement a mechanism to find and kill a pending lock request Tags lock requests with a client id, use the client id as a key into the pending lock requests sets to find a lock request, complete the lock request with a lock timeout error. Copyright (c) 2016, Rich Prohaska All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --- .../tokudb/PerconaFT/buildheader/make_tdb.cc | 5 +- storage/tokudb/PerconaFT/ft/txn/txn.cc | 9 +- storage/tokudb/PerconaFT/ft/txn/txn.h | 5 +- .../tokudb/PerconaFT/locktree/lock_request.cc | 130 +++++++++++------- .../tokudb/PerconaFT/locktree/lock_request.h | 15 +- storage/tokudb/PerconaFT/locktree/locktree.cc | 14 +- storage/tokudb/PerconaFT/locktree/locktree.h | 8 +- storage/tokudb/PerconaFT/locktree/manager.cc | 13 ++ .../PerconaFT/locktree/tests/kill_waiter.cc | 100 ++++++++++++++ .../locktree/tests/lock_request_killed.cc | 3 +- .../locktree/tests/lock_request_not_killed.cc | 1 - .../tests/lock_request_start_release_wait.cc | 89 ++++++++++++ .../tests/lock_request_start_retry_race.cc | 31 +---- .../tests/lock_request_start_retry_race_3.cc | 127 +++++++++++++++++ .../lock_request_start_retry_wait_race_2.cc | 128 +++++++++++++++++ .../tests/test_iterate_live_transactions.cc | 9 +- .../PerconaFT/src/tests/test_stress0.cc | 3 +- storage/tokudb/PerconaFT/src/ydb.cc | 5 + storage/tokudb/PerconaFT/src/ydb_txn.cc | 8 +- storage/tokudb/tokudb_information_schema.cc | 8 +- storage/tokudb/tokudb_txn.h | 2 +- 21 files changed, 606 insertions(+), 107 deletions(-) create mode 100644 storage/tokudb/PerconaFT/locktree/tests/kill_waiter.cc create mode 100644 storage/tokudb/PerconaFT/locktree/tests/lock_request_start_release_wait.cc create mode 100644 storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc create mode 100644 storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_wait_race_2.cc diff --git a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc index 7ede78b3c0db3..cadaa48cceaa1 100644 --- a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc +++ b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc @@ -425,6 +425,7 @@ static void print_db_env_struct (void) { "bool (*set_dir_per_db)(DB_ENV *, bool new_val)", "bool (*get_dir_per_db)(DB_ENV *)", "const char *(*get_data_dir)(DB_ENV *env)", + "void (*kill_waiter)(DB_ENV *, void *extra)", NULL}; sort_and_dump_fields("db_env", true, extra); @@ -545,8 +546,8 @@ static void print_db_txn_struct (void) { "int (*abort_with_progress)(DB_TXN*, TXN_PROGRESS_POLL_FUNCTION, void*)", "int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *, uint32_t flags)", "uint64_t (*id64) (DB_TXN*)", - "void (*set_client_id)(DB_TXN *, uint64_t client_id)", - "uint64_t (*get_client_id)(DB_TXN *)", + "void (*set_client_id)(DB_TXN *, uint64_t client_id, void *client_extra)", + "void (*get_client_id)(DB_TXN *, uint64_t *client_id, void **client_extra)", "bool (*is_prepared)(DB_TXN *)", "DB_TXN *(*get_child)(DB_TXN *)", "uint64_t (*get_start_time)(DB_TXN *)", diff --git a/storage/tokudb/PerconaFT/ft/txn/txn.cc b/storage/tokudb/PerconaFT/ft/txn/txn.cc index dd03073a3ec4e..9e48d0d05dd8b 100644 --- a/storage/tokudb/PerconaFT/ft/txn/txn.cc +++ b/storage/tokudb/PerconaFT/ft/txn/txn.cc @@ -269,6 +269,7 @@ static txn_child_manager tcm; .state = TOKUTXN_LIVE, .num_pin = 0, .client_id = 0, + .client_extra = nullptr, .start_time = time(NULL), }; @@ -705,12 +706,14 @@ bool toku_txn_has_spilled_rollback(TOKUTXN txn) { return txn_has_spilled_rollback_logs(txn); } -uint64_t toku_txn_get_client_id(TOKUTXN txn) { - return txn->client_id; +void toku_txn_get_client_id(TOKUTXN txn, uint64_t *client_id, void **client_extra) { + *client_id = txn->client_id; + *client_extra = txn->client_extra; } -void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id) { +void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id, void *client_extra) { txn->client_id = client_id; + txn->client_extra = client_extra; } time_t toku_txn_get_start_time(struct tokutxn *txn) { diff --git a/storage/tokudb/PerconaFT/ft/txn/txn.h b/storage/tokudb/PerconaFT/ft/txn/txn.h index 51a4602215076..34a76aa9cadaa 100644 --- a/storage/tokudb/PerconaFT/ft/txn/txn.h +++ b/storage/tokudb/PerconaFT/ft/txn/txn.h @@ -193,6 +193,7 @@ struct tokutxn { uint32_t num_pin; // number of threads (all hot indexes) that want this // txn to not transition to commit or abort uint64_t client_id; + void *client_extra; time_t start_time; }; typedef struct tokutxn *TOKUTXN; @@ -293,8 +294,8 @@ void toku_txn_unpin_live_txn(struct tokutxn *txn); bool toku_txn_has_spilled_rollback(struct tokutxn *txn); -uint64_t toku_txn_get_client_id(struct tokutxn *txn); -void toku_txn_set_client_id(struct tokutxn *txn, uint64_t client_id); +void toku_txn_get_client_id(struct tokutxn *txn, uint64_t *client_id, void **client_extra); +void toku_txn_set_client_id(struct tokutxn *txn, uint64_t client_id, void *client_extra); time_t toku_txn_get_start_time(struct tokutxn *txn); diff --git a/storage/tokudb/PerconaFT/locktree/lock_request.cc b/storage/tokudb/PerconaFT/locktree/lock_request.cc index 22b6da9afc4ee..1bc613533dba7 100644 --- a/storage/tokudb/PerconaFT/locktree/lock_request.cc +++ b/storage/tokudb/PerconaFT/locktree/lock_request.cc @@ -65,6 +65,7 @@ void lock_request::create(void) { toku_cond_init(&m_wait_cond, nullptr); m_start_test_callback = nullptr; + m_start_before_pending_test_callback = nullptr; m_retry_test_callback = nullptr; } @@ -79,7 +80,7 @@ void lock_request::destroy(void) { } // set the lock request parameters. this API allows a lock request to be reused. -void lock_request::set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT *right_key, lock_request::type lock_type, bool big_txn) { +void lock_request::set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT *right_key, lock_request::type lock_type, bool big_txn, void *extra) { invariant(m_state != state::PENDING); m_lt = lt; m_txnid = txnid; @@ -91,6 +92,7 @@ void lock_request::set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT m_state = state::INITIALIZED; m_info = lt ? lt->get_lock_request_info() : nullptr; m_big_txn = big_txn; + m_extra = extra; } // get rid of any stored left and right key copies and @@ -173,6 +175,7 @@ int lock_request::start(void) { m_state = state::PENDING; m_start_time = toku_current_time_microsec() / 1000; m_conflicting_txnid = conflicts.get(0); + if (m_start_before_pending_test_callback) m_start_before_pending_test_callback(); toku_mutex_lock(&m_info->mutex); insert_into_lock_requests(); if (deadlock_exists(conflicts)) { @@ -203,7 +206,18 @@ int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*kil toku_mutex_lock(&m_info->mutex); + // check again, this time locking out other retry calls + if (m_state == state::PENDING) { + retry(); + } + while (m_state == state::PENDING) { + // check if this thread is killed + if (killed_callback && killed_callback()) { + remove_from_lock_requests(); + complete(DB_LOCK_NOTGRANTED); + continue; + } // compute next wait time uint64_t t_wait; @@ -221,7 +235,7 @@ int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*kil invariant(r == 0 || r == ETIMEDOUT); t_now = toku_current_time_microsec(); - if (m_state == state::PENDING && (t_now >= t_end || (killed_callback && killed_callback()))) { + if (m_state == state::PENDING && t_now >= t_end) { m_info->counters.timeout_count += 1; // if we're still pending and we timed out, then remove our @@ -274,13 +288,15 @@ TXNID lock_request::get_conflicting_txnid(void) const { } int lock_request::retry(void) { + invariant(m_state == state::PENDING); int r; - invariant(m_state == state::PENDING); + txnid_set conflicts; + conflicts.create(); if (m_type == type::WRITE) { - r = m_lt->acquire_write_lock(m_txnid, m_left_key, m_right_key, nullptr, m_big_txn); + r = m_lt->acquire_write_lock(m_txnid, m_left_key, m_right_key, &conflicts, m_big_txn); } else { - r = m_lt->acquire_read_lock(m_txnid, m_left_key, m_right_key, nullptr, m_big_txn); + r = m_lt->acquire_read_lock(m_txnid, m_left_key, m_right_key, &conflicts, m_big_txn); } // if the acquisition succeeded then remove ourselves from the @@ -290,59 +306,77 @@ int lock_request::retry(void) { complete(r); if (m_retry_test_callback) m_retry_test_callback(); // test callback toku_cond_broadcast(&m_wait_cond); + } else { + m_conflicting_txnid = conflicts.get(0); } + conflicts.destroy(); return r; } -void lock_request::retry_all_lock_requests(locktree *lt) { +void lock_request::retry_all_lock_requests(locktree *lt, void (*after_retry_all_test_callback)(void)) { lt_lock_request_info *info = lt->get_lock_request_info(); - // if a thread reads this bit to be true, then it should go ahead and - // take the locktree mutex and retry lock requests. we use this bit - // to prevent every single thread from waiting on the locktree mutex - // in order to retry requests, especially when no requests actually exist. - // - // it is important to note that this bit only provides an optimization. - // it is not problematic for it to be true when it should be false, - // but it can be problematic for it to be false when it should be true. - // therefore, the lock request code must ensures that when lock requests - // are added to this locktree, the bit is set. - // see lock_request::insert_into_lock_requests() - if (!info->should_retry_lock_requests) { + info->retry_want++; + + // if there are no pending lock requests than there is nothing to do + // the unlocked data race on pending_is_empty is OK since lock requests + // are retried after added to the pending set. + if (info->pending_is_empty) return; - } toku_mutex_lock(&info->mutex); - // let other threads know that they need not retry lock requests at this time. - // - // the motivation here is that if a bunch of threads have already released - // their locks in the rangetree, then its probably okay for only one thread - // to iterate over the list of requests and retry them. otherwise, at high - // thread counts and a large number of pending lock requests, you could - // end up wasting a lot of cycles. - info->should_retry_lock_requests = false; - - size_t i = 0; - while (i < info->pending_lock_requests.size()) { - lock_request *request; - int r = info->pending_lock_requests.fetch(i, &request); - invariant_zero(r); - - // retry the lock request. if it didn't succeed, - // move on to the next lock request. otherwise - // the request is gone from the list so we may - // read the i'th entry for the next one. - r = request->retry(); - if (r != 0) { - i++; + // here is the group retry algorithm. + // get the latest retry_want count and use it as the generation number of this retry operation. + // if this retry generation is > the last retry generation, then do the lock retries. otherwise, + // no lock retries are needed. + unsigned long long retry_gen = info->retry_want.load(); + if (retry_gen > info->retry_done) { + + // retry all of the pending lock requests. + for (size_t i = 0; i < info->pending_lock_requests.size(); ) { + lock_request *request; + int r = info->pending_lock_requests.fetch(i, &request); + invariant_zero(r); + + // retry this lock request. if it didn't succeed, + // move on to the next lock request. otherwise + // the request is gone from the list so we may + // read the i'th entry for the next one. + r = request->retry(); + if (r != 0) { + i++; + } } + if (after_retry_all_test_callback) after_retry_all_test_callback(); + info->retry_done = retry_gen; } - // future threads should only retry lock requests if some still exist - info->should_retry_lock_requests = info->pending_lock_requests.size() > 0; + toku_mutex_unlock(&info->mutex); +} +void *lock_request::get_extra(void) const { + return m_extra; +} + +void lock_request::kill_waiter(void) { + remove_from_lock_requests(); + complete(DB_LOCK_NOTGRANTED); + toku_cond_broadcast(&m_wait_cond); +} + +void lock_request::kill_waiter(locktree *lt, void *extra) { + lt_lock_request_info *info = lt->get_lock_request_info(); + toku_mutex_lock(&info->mutex); + for (size_t i = 0; i < info->pending_lock_requests.size(); i++) { + lock_request *request; + int r = info->pending_lock_requests.fetch(i, &request); + if (r == 0 && request->get_extra() == extra) { + request->kill_waiter(); + break; + } + } toku_mutex_unlock(&info->mutex); } @@ -364,9 +398,7 @@ void lock_request::insert_into_lock_requests(void) { invariant(r == DB_NOTFOUND); r = m_info->pending_lock_requests.insert_at(this, idx); invariant_zero(r); - - // ensure that this bit is true, now that at least one lock request is in the set - m_info->should_retry_lock_requests = true; + m_info->pending_is_empty = false; } // remove this lock request from the locktree's set. must hold the mutex. @@ -378,6 +410,8 @@ void lock_request::remove_from_lock_requests(void) { invariant(request == this); r = m_info->pending_lock_requests.delete_at(idx); invariant_zero(r); + if (m_info->pending_lock_requests.size() == 0) + m_info->pending_is_empty = true; } int lock_request::find_by_txnid(lock_request * const &request, const TXNID &txnid) { @@ -395,6 +429,10 @@ void lock_request::set_start_test_callback(void (*f)(void)) { m_start_test_callback = f; } +void lock_request::set_start_before_pending_test_callback(void (*f)(void)) { + m_start_before_pending_test_callback = f; +} + void lock_request::set_retry_test_callback(void (*f)(void)) { m_retry_test_callback = f; } diff --git a/storage/tokudb/PerconaFT/locktree/lock_request.h b/storage/tokudb/PerconaFT/locktree/lock_request.h index 48d1279cde27b..ab69253bcec05 100644 --- a/storage/tokudb/PerconaFT/locktree/lock_request.h +++ b/storage/tokudb/PerconaFT/locktree/lock_request.h @@ -78,7 +78,7 @@ class lock_request { // effect: Resets the lock request parameters, allowing it to be reused. // requires: Lock request was already created at some point - void set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT *right_key, type lock_type, bool big_txn); + void set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT *right_key, type lock_type, bool big_txn, void *extra = nullptr); // effect: Tries to acquire a lock described by this lock request. // returns: The return code of locktree::acquire_[write,read]_lock() @@ -109,12 +109,18 @@ class lock_request { // effect: Retries all of the lock requests for the given locktree. // Any lock requests successfully restarted is completed and woken up. // The rest remain pending. - static void retry_all_lock_requests(locktree *lt); + static void retry_all_lock_requests(locktree *lt, void (*after_retry_test_callback)(void) = nullptr); void set_start_test_callback(void (*f)(void)); + void set_start_before_pending_test_callback(void (*f)(void)); void set_retry_test_callback(void (*f)(void)); -private: + void *get_extra(void) const; + + void kill_waiter(void); + static void kill_waiter(locktree *lt, void *extra); + +private: enum state { UNINITIALIZED, INITIALIZED, @@ -152,6 +158,8 @@ class lock_request { // locktree that this lock request is for. struct lt_lock_request_info *m_info; + void *m_extra; + // effect: tries again to acquire the lock described by this lock request // returns: 0 if retrying the request succeeded and is now complete int retry(void); @@ -187,6 +195,7 @@ class lock_request { static int find_by_txnid(lock_request * const &request, const TXNID &txnid); void (*m_start_test_callback)(void); + void (*m_start_before_pending_test_callback)(void); void (*m_retry_test_callback)(void); friend class lock_request_unit_test; diff --git a/storage/tokudb/PerconaFT/locktree/locktree.cc b/storage/tokudb/PerconaFT/locktree/locktree.cc index d3596d47eeb49..11f8a4e5ff7e9 100644 --- a/storage/tokudb/PerconaFT/locktree/locktree.cc +++ b/storage/tokudb/PerconaFT/locktree/locktree.cc @@ -81,20 +81,14 @@ void locktree::create(locktree_manager *mgr, DICTIONARY_ID dict_id, const compar m_sto_end_early_time = 0; m_lock_request_info.pending_lock_requests.create(); + m_lock_request_info.pending_is_empty = true; ZERO_STRUCT(m_lock_request_info.mutex); toku_mutex_init(&m_lock_request_info.mutex, nullptr); - m_lock_request_info.should_retry_lock_requests = false; + m_lock_request_info.retry_want = m_lock_request_info.retry_done = 0; ZERO_STRUCT(m_lock_request_info.counters); - // Threads read the should retry bit without a lock - // for performance. It's ok to read the wrong value. - // - If you think you should but you shouldn't, you waste a little time. - // - If you think you shouldn't but you should, then some other thread - // will come around to do the work of retrying requests instead of you. - TOKU_VALGRIND_HG_DISABLE_CHECKING( - &m_lock_request_info.should_retry_lock_requests, - sizeof(m_lock_request_info.should_retry_lock_requests)); - TOKU_DRD_IGNORE_VAR(m_lock_request_info.should_retry_lock_requests); + TOKU_VALGRIND_HG_DISABLE_CHECKING(&m_lock_request_info.pending_is_empty, sizeof(m_lock_request_info.pending_is_empty)); + TOKU_DRD_IGNORE_VAR(m_lock_request_info.pending_is_empty); } void locktree::destroy(void) { diff --git a/storage/tokudb/PerconaFT/locktree/locktree.h b/storage/tokudb/PerconaFT/locktree/locktree.h index 710f9e7db0618..64171c51b23bf 100644 --- a/storage/tokudb/PerconaFT/locktree/locktree.h +++ b/storage/tokudb/PerconaFT/locktree/locktree.h @@ -38,6 +38,8 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #pragma once +#include + #include #include #include @@ -80,9 +82,11 @@ namespace toku { // Lock request state for some locktree struct lt_lock_request_info { omt pending_lock_requests; + std::atomic_bool pending_is_empty; toku_mutex_t mutex; - bool should_retry_lock_requests; lt_counters counters; + std::atomic_ullong retry_want; + unsigned long long retry_done; }; // The locktree manager manages a set of locktrees, one for each open dictionary. @@ -159,6 +163,8 @@ namespace toku { // Add time t to the escalator's wait time statistics void add_escalator_wait_time(uint64_t t); + void kill_waiter(void *extra); + private: static const uint64_t DEFAULT_MAX_LOCK_MEMORY = 64L * 1024 * 1024; diff --git a/storage/tokudb/PerconaFT/locktree/manager.cc b/storage/tokudb/PerconaFT/locktree/manager.cc index 4708cdf4a5a3c..91ff7c5a007c8 100644 --- a/storage/tokudb/PerconaFT/locktree/manager.cc +++ b/storage/tokudb/PerconaFT/locktree/manager.cc @@ -483,4 +483,17 @@ void locktree_manager::get_status(LTM_STATUS statp) { *statp = ltm_status; } +void locktree_manager::kill_waiter(void *extra) { + mutex_lock(); + int r = 0; + size_t num_locktrees = m_locktree_map.size(); + for (size_t i = 0; i < num_locktrees; i++) { + locktree *lt; + r = m_locktree_map.fetch(i, <); + invariant_zero(r); + lock_request::kill_waiter(lt, extra); + } + mutex_unlock(); +} + } /* namespace toku */ diff --git a/storage/tokudb/PerconaFT/locktree/tests/kill_waiter.cc b/storage/tokudb/PerconaFT/locktree/tests/kill_waiter.cc new file mode 100644 index 0000000000000..8d93c0bbbab21 --- /dev/null +++ b/storage/tokudb/PerconaFT/locktree/tests/kill_waiter.cc @@ -0,0 +1,100 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: + +// test the lock manager kill waiter function + +#include "locktree.h" +#include "lock_request.h" +#include "test.h" +#include "locktree_unit_test.h" +#include +#include + +namespace toku { + +const uint64_t my_lock_wait_time = 1000 * 1000; +const uint64_t my_killed_time = 500 * 1000; +const int n_locks = 4; + +static int my_killed_callback(void) { + if (1) fprintf(stderr, "%s:%u %s\n", __FILE__, __LINE__, __FUNCTION__); + return 0; +} + +static void locktree_release_lock(locktree *lt, TXNID txn_id, const DBT *left, const DBT *right) { + range_buffer buffer; + buffer.create(); + buffer.append(left, right); + lt->release_locks(txn_id, &buffer); + buffer.destroy(); +} + +static void wait_lock(lock_request *lr, std::atomic_int *done) { + int r = lr->wait(my_lock_wait_time, my_killed_time, my_killed_callback); + assert(r == DB_LOCK_NOTGRANTED); + *done = 1; +} + +static void test_kill_waiter(void) { + int r; + + locktree_manager mgr; + mgr.create(nullptr, nullptr, nullptr, nullptr); + + DICTIONARY_ID dict_id = { 1 }; + locktree *lt = mgr.get_lt(dict_id, dbt_comparator, nullptr); + + const DBT *one = get_dbt(1); + + lock_request locks[n_locks]; + std::thread waiters[n_locks-1]; + for (int i = 0; i < n_locks; i++) { + locks[i].create(); + locks[i].set(lt, i+1, one, one, lock_request::type::WRITE, false, &waiters[i]); + } + + // txn 'n_locks' grabs the lock + r = locks[n_locks-1].start(); + assert_zero(r); + + for (int i = 0; i < n_locks-1; i++) { + r = locks[i].start(); + assert(r == DB_LOCK_NOTGRANTED); + } + + std::atomic_int done[n_locks-1]; + for (int i = 0; i < n_locks-1; i++) { + done[i] = 0; + waiters[i] = std::thread(wait_lock, &locks[i], &done[i]); + } + + for (int i = 0; i < n_locks-1; i++) { + assert(!done[i]); + } + + sleep(1); + for (int i = 0; i < n_locks-1; i++) { + mgr.kill_waiter(&waiters[i]); + while (!done[i]) sleep(1); + waiters[i].join(); + for (int j = i+1; j < n_locks-1; j++) + assert(!done[j]); + } + + locktree_release_lock(lt, n_locks, one, one); + + for (int i = 0; i < n_locks; i++) { + locks[i].destroy(); + } + + mgr.release_lt(lt); + mgr.destroy(); +} + +} /* namespace toku */ + +int main(void) { + toku::test_kill_waiter(); + return 0; +} + diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_killed.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_killed.cc index efd4092906bcb..ec464444271fe 100644 --- a/storage/tokudb/PerconaFT/locktree/tests/lock_request_killed.cc +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_killed.cc @@ -51,8 +51,9 @@ static uint64_t t_do_kill; static int my_killed_callback(void) { uint64_t t_now = toku_current_time_microsec(); + if (t_now == t_last_kill) + return 0; assert(t_now >= t_last_kill); - assert(t_now - t_last_kill >= my_killed_time * 1000 / 2); // div by 2 for valgrind which is not very accurate t_last_kill = t_now; killed_calls++; if (t_now >= t_do_kill) diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_not_killed.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_not_killed.cc index 702e2e2626c1b..647b4d3c418b4 100644 --- a/storage/tokudb/PerconaFT/locktree/tests/lock_request_not_killed.cc +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_not_killed.cc @@ -52,7 +52,6 @@ static uint64_t t_last_kill; static int my_killed_callback(void) { uint64_t t_now = toku_current_time_microsec(); assert(t_now >= t_last_kill); - assert(t_now - t_last_kill >= my_killed_time * 1000 / 2); // div by 2 for valgrind which is not very accurate t_last_kill = t_now; killed_calls++; return 0; diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_release_wait.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_release_wait.cc new file mode 100644 index 0000000000000..eb19ceb70e5db --- /dev/null +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_release_wait.cc @@ -0,0 +1,89 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: + +// test the race between start, release, and wait. since start does not put its +// lock request into the pending set, the blocking txn could release its lock before +// the first txn waits. this will block the first txn because its lock request is +// not known when the lock is released. the bug fix is to try again when lock retries +// are locked out. + +#include "locktree.h" +#include "lock_request.h" +#include "test.h" +#include "locktree_unit_test.h" +#include +#include + +namespace toku { + +const uint64_t my_lock_wait_time = 1000 * 1000; // ms +const uint64_t my_killed_time = 1 * 1000; // ms + +static uint64_t t_wait; + +static int my_killed_callback(void) { + uint64_t t_now = toku_current_time_microsec(); + assert(t_now >= t_wait); + if (t_now - t_wait >= my_killed_time*1000) + abort(); + return 0; +} + +static void locktree_release_lock(locktree *lt, TXNID txn_id, const DBT *left, const DBT *right) { + range_buffer buffer; + buffer.create(); + buffer.append(left, right); + lt->release_locks(txn_id, &buffer); + buffer.destroy(); +} + +static void test_start_release_wait(void) { + int r; + + locktree_manager mgr; + mgr.create(nullptr, nullptr, nullptr, nullptr); + + DICTIONARY_ID dict_id = { 1 }; + locktree *lt = mgr.get_lt(dict_id, dbt_comparator, nullptr); + + const DBT *one = get_dbt(1); + + // a locks one + lock_request a; + a.create(); + a.set(lt, 1, one, one, lock_request::type::WRITE, false); + r = a.start(); + assert(r == 0); + + // b tries to lock one, fails + lock_request b; + b.create(); + b.set(lt, 2, one, one, lock_request::type::WRITE, false); + r = b.start(); + assert(r == DB_LOCK_NOTGRANTED); + + // a releases its lock + locktree_release_lock(lt, 1, one, one); + + // b waits for one, gets locks immediately + t_wait = toku_current_time_microsec(); + r = b.wait(my_lock_wait_time, my_killed_time, my_killed_callback); + assert(r == 0); + + // b releases its lock so we can exit cleanly + locktree_release_lock(lt, 2, one, one); + + a.destroy(); + b.destroy(); + + mgr.release_lt(lt); + mgr.destroy(); +} + +} /* namespace toku */ + +int main(void) { + toku::test_start_release_wait(); + return 0; +} + diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race.cc index 3b653e9c6ef8a..88493ec9ce0d5 100644 --- a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race.cc +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race.cc @@ -37,6 +37,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." #include +#include #include "test.h" #include "locktree.h" #include "lock_request.h" @@ -47,15 +48,6 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. namespace toku { -struct locker_arg { - locktree *_lt; - TXNID _id; - const DBT *_key; - - locker_arg(locktree *lt, TXNID id, const DBT *key) : _lt(lt), _id(id), _key(key) { - } -}; - static void locker_callback(void) { usleep(10000); } @@ -97,20 +89,13 @@ static void run_locker(locktree *lt, TXNID txnid, const DBT *key) { toku_pthread_yield(); if ((i % 10) == 0) - std::cout << toku_pthread_self() << " " << i << std::endl; + std::cout << std::this_thread::get_id() << " " << i << std::endl; } } -static void *locker(void *v_arg) { - locker_arg *arg = static_cast(v_arg); - run_locker(arg->_lt, arg->_id, arg->_key); - return arg; -} - } /* namespace toku */ int main(void) { - int r; toku::locktree lt; DICTIONARY_ID dict_id = { 1 }; @@ -119,18 +104,12 @@ int main(void) { const DBT *one = toku::get_dbt(1); const int n_workers = 2; - toku_pthread_t ids[n_workers]; + std::thread worker[n_workers]; for (int i = 0; i < n_workers; i++) { - toku::locker_arg *arg = new toku::locker_arg(<, i, one); - r = toku_pthread_create(&ids[i], nullptr, toku::locker, arg); - assert_zero(r); + worker[i] = std::thread(toku::run_locker, <, i, one); } for (int i = 0; i < n_workers; i++) { - void *ret; - r = toku_pthread_join(ids[i], &ret); - assert_zero(r); - toku::locker_arg *arg = static_cast(ret); - delete arg; + worker[i].join(); } lt.release_reference(); diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc new file mode 100644 index 0000000000000..92122861819db --- /dev/null +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc @@ -0,0 +1,127 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see . + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see . +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include +#include +#include +#include "test.h" +#include "locktree.h" +#include "lock_request.h" + +// Suppose that 3 threads are running a lock acquire, release, retry sequence. There is +// a race in the retry algorithm with 2 threads running lock retry simultaneously. The +// first thread to run retry sets a flag that will cause the second thread to skip the +// lock retries. If the first thread progressed past the contended lock, then the second +// threa will HANG until its lock timer pops, even when the contended lock is no longer held. + +// This test exposes this problem as a test hang. The group retry algorithm fixes the race +// in the lock request retry algorihm and this test should no longer hang. + +namespace toku { + +// use 1000 when after_retry_all is implemented, otherwise use 100000 +static const int n_tests = 1000; // 100000; + +static void after_retry_all(void) { + usleep(10000); +} + +static void run_locker(locktree *lt, TXNID txnid, const DBT *key, pthread_barrier_t *b) { + for (int i = 0; i < n_tests; i++) { + int r; + r = pthread_barrier_wait(b); assert(r == 0 || r == PTHREAD_BARRIER_SERIAL_THREAD); + + lock_request request; + request.create(); + + request.set(lt, txnid, key, key, lock_request::type::WRITE, false); + + // try to acquire the lock + r = request.start(); + if (r == DB_LOCK_NOTGRANTED) { + // wait for the lock to be granted + r = request.wait(1000 * 1000); + } + + if (r == 0) { + // release the lock + range_buffer buffer; + buffer.create(); + buffer.append(key, key); + lt->release_locks(txnid, &buffer); + buffer.destroy(); + + // retry pending lock requests + lock_request::retry_all_lock_requests(lt, after_retry_all); + } + + request.destroy(); + memset(&request, 0xab, sizeof request); + + toku_pthread_yield(); + if ((i % 10) == 0) + std::cout << std::this_thread::get_id() << " " << i << std::endl; + } +} + +} /* namespace toku */ + +int main(void) { + + toku::locktree lt; + DICTIONARY_ID dict_id = { 1 }; + lt.create(nullptr, dict_id, toku::dbt_comparator); + + const DBT *one = toku::get_dbt(1); + + const int n_workers = 3; + std::thread worker[n_workers]; + pthread_barrier_t b; + int r = pthread_barrier_init(&b, nullptr, n_workers); assert(r == 0); + for (int i = 0; i < n_workers; i++) { + worker[i] = std::thread(toku::run_locker, <, i, one, &b); + } + for (int i = 0; i < n_workers; i++) { + worker[i].join(); + } + r = pthread_barrier_destroy(&b); assert(r == 0); + lt.release_reference(); + lt.destroy(); + return 0; +} + diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_wait_race_2.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_wait_race_2.cc new file mode 100644 index 0000000000000..a2ceff99edb72 --- /dev/null +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_wait_race_2.cc @@ -0,0 +1,128 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see . + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see . +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include +#include +#include +#include "test.h" +#include "locktree.h" +#include "lock_request.h" + +// Suppose that 2 threads are running a lock acquire, release, retry sequence. There is a +// race between the acquire and the release with 2 threads. If thread 1 acquires a lock, +// and thread 2 tries to acquire the same lock and fails, thread 1 may release its lock and retry +// pending lock requests BEFORE thread 2 adds itself to the pending lock requests. If this +// happens, then thread 2 will HANG until its lock timer expires even when the lock it is +// waiting for is FREE. + +// This test exposes this problem as a test hang. If the race is fixed, then the test runs to +// completion. + +namespace toku { + +static void start_before_pending(void) { + usleep(10000); +} + +static void run_locker(locktree *lt, TXNID txnid, const DBT *key, pthread_barrier_t *b) { + for (int i = 0; i < 100000; i++) { + int r; + r = pthread_barrier_wait(b); assert(r == 0 || r == PTHREAD_BARRIER_SERIAL_THREAD); + + lock_request request; + request.create(); + request.set(lt, txnid, key, key, lock_request::type::WRITE, false); + + // if the callback is included, then the race is easy to reproduce. Otherwise, several + // test runs may be required before the race happens. + if (1) request.set_start_before_pending_test_callback(start_before_pending); + + // try to acquire the lock + r = request.start(); + if (r == DB_LOCK_NOTGRANTED) { + // wait for the lock to be granted + r = request.wait(1000 * 1000); + } + + if (r == 0) { + // release the lock + range_buffer buffer; + buffer.create(); + buffer.append(key, key); + lt->release_locks(txnid, &buffer); + buffer.destroy(); + + // retry pending lock requests + lock_request::retry_all_lock_requests(lt); + } + + request.destroy(); + memset(&request, 0xab, sizeof request); + + toku_pthread_yield(); + if ((i % 10) == 0) + std::cout << std::this_thread::get_id() << " " << i << std::endl; + } +} + +} /* namespace toku */ + +int main(void) { + + toku::locktree lt; + DICTIONARY_ID dict_id = { 1 }; + lt.create(nullptr, dict_id, toku::dbt_comparator); + + const DBT *one = toku::get_dbt(1); + + const int n_workers = 2; + std::thread worker[n_workers]; + pthread_barrier_t b; + int r = pthread_barrier_init(&b, nullptr, n_workers); assert(r == 0); + for (int i = 0; i < n_workers; i++) { + worker[i] = std::thread(toku::run_locker, <, i, one, &b); + } + for (int i = 0; i < n_workers; i++) { + worker[i].join(); + } + r = pthread_barrier_destroy(&b); assert(r == 0); + lt.release_reference(); + lt.destroy(); + return 0; +} + diff --git a/storage/tokudb/PerconaFT/src/tests/test_iterate_live_transactions.cc b/storage/tokudb/PerconaFT/src/tests/test_iterate_live_transactions.cc index c5561cdf90f05..23c79620cd81b 100644 --- a/storage/tokudb/PerconaFT/src/tests/test_iterate_live_transactions.cc +++ b/storage/tokudb/PerconaFT/src/tests/test_iterate_live_transactions.cc @@ -55,7 +55,8 @@ static int iterate_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { uint64_t txnid = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); + uint64_t client_id; void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); iterate_extra *info = reinterpret_cast(extra); DB *db; DBT left_key, right_key; @@ -93,13 +94,13 @@ int test_main(int UU(argc), char *const UU(argv[])) { r = env->open(env, TOKU_TEST_FILENAME, env_flags, 0755); CKERR(r); r = env->txn_begin(env, NULL, &txn1, 0); CKERR(r); - txn1->set_client_id(txn1, 0); + txn1->set_client_id(txn1, 0, NULL); txnid1 = txn1->id64(txn1); r = env->txn_begin(env, NULL, &txn2, 0); CKERR(r); - txn2->set_client_id(txn2, 1); + txn2->set_client_id(txn2, 1, NULL); txnid2 = txn2->id64(txn2); r = env->txn_begin(env, NULL, &txn3, 0); CKERR(r); - txn3->set_client_id(txn3, 2); + txn3->set_client_id(txn3, 2, NULL); txnid3 = txn3->id64(txn3); { diff --git a/storage/tokudb/PerconaFT/src/tests/test_stress0.cc b/storage/tokudb/PerconaFT/src/tests/test_stress0.cc index 88140dd173184..037ffdd312ddd 100644 --- a/storage/tokudb/PerconaFT/src/tests/test_stress0.cc +++ b/storage/tokudb/PerconaFT/src/tests/test_stress0.cc @@ -93,7 +93,8 @@ static int iterate_txns(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { uint64_t txnid = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); + uint64_t client_id; void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); invariant_null(extra); invariant(txnid > 0); invariant(client_id == 0); diff --git a/storage/tokudb/PerconaFT/src/ydb.cc b/storage/tokudb/PerconaFT/src/ydb.cc index 3341f6d76c624..d51ee81700f8e 100644 --- a/storage/tokudb/PerconaFT/src/ydb.cc +++ b/storage/tokudb/PerconaFT/src/ydb.cc @@ -2620,6 +2620,10 @@ static void env_set_killed_callback(DB_ENV *env, uint64_t default_killed_time_ms env->i->killed_callback = killed_callback; } +static void env_kill_waiter(DB_ENV *env, void *extra) { + env->i->ltm.kill_waiter(extra); +} + static void env_do_backtrace(DB_ENV *env) { if (env->i->errcall) { db_env_do_backtrace_errfunc((toku_env_err_func) toku_env_err, (const void *) env); @@ -2719,6 +2723,7 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) { USENV(set_dir_per_db); USENV(get_dir_per_db); USENV(get_data_dir); + USENV(kill_waiter); #undef USENV // unlocked methods diff --git a/storage/tokudb/PerconaFT/src/ydb_txn.cc b/storage/tokudb/PerconaFT/src/ydb_txn.cc index ae1f93011d1de..40b479055f224 100644 --- a/storage/tokudb/PerconaFT/src/ydb_txn.cc +++ b/storage/tokudb/PerconaFT/src/ydb_txn.cc @@ -323,12 +323,12 @@ int locked_txn_abort(DB_TXN *txn) { return r; } -static void locked_txn_set_client_id(DB_TXN *txn, uint64_t client_id) { - toku_txn_set_client_id(db_txn_struct_i(txn)->tokutxn, client_id); +static void locked_txn_set_client_id(DB_TXN *txn, uint64_t client_id, void *client_extra) { + toku_txn_set_client_id(db_txn_struct_i(txn)->tokutxn, client_id, client_extra); } -static uint64_t locked_txn_get_client_id(DB_TXN *txn) { - return toku_txn_get_client_id(db_txn_struct_i(txn)->tokutxn); +static void locked_txn_get_client_id(DB_TXN *txn, uint64_t *client_id, void **client_extra) { + toku_txn_get_client_id(db_txn_struct_i(txn)->tokutxn, client_id, client_extra); } static int toku_txn_discard(DB_TXN *txn, uint32_t flags) { diff --git a/storage/tokudb/tokudb_information_schema.cc b/storage/tokudb/tokudb_information_schema.cc index b3d77eef2d968..86af3f14f9190 100644 --- a/storage/tokudb/tokudb_information_schema.cc +++ b/storage/tokudb/tokudb_information_schema.cc @@ -75,7 +75,9 @@ int trx_callback( void *extra) { uint64_t txn_id = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); + uint64_t client_id; + void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); uint64_t start_time = txn->get_start_time(txn); trx_extra_t* e = reinterpret_cast(extra); THD* thd = e->thd; @@ -314,7 +316,9 @@ int locks_callback( void* extra) { uint64_t txn_id = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); + uint64_t client_id; + void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); locks_extra_t* e = reinterpret_cast(extra); THD* thd = e->thd; TABLE* table = e->table; diff --git a/storage/tokudb/tokudb_txn.h b/storage/tokudb/tokudb_txn.h index 67bf591d088d7..d025541540309 100644 --- a/storage/tokudb/tokudb_txn.h +++ b/storage/tokudb/tokudb_txn.h @@ -116,7 +116,7 @@ inline int txn_begin( int r = env->txn_begin(env, parent, txn, flags); if (r == 0 && thd) { DB_TXN* this_txn = *txn; - this_txn->set_client_id(this_txn, thd_get_thread_id(thd)); + this_txn->set_client_id(this_txn, thd_get_thread_id(thd), thd); } TOKUDB_TRACE_FOR_FLAGS( TOKUDB_DEBUG_TXN, From 660a2928a535e36e5dda846677dce4ba96508cd7 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 23 Nov 2016 16:46:33 +0100 Subject: [PATCH 205/295] Fix optimistic parallel replication for TokuDB. Make TokuDB report row lock waits with thd_rpl_deadlock_check(). This allows parallel replication to properly detect conflicts, and kill and retry the offending transaction. --- .../tokudb/PerconaFT/buildheader/make_tdb.cc | 2 + storage/tokudb/PerconaFT/ftcxx/db_env.hpp | 12 +++ .../tokudb/PerconaFT/locktree/lock_request.cc | 45 ++++++++-- .../tokudb/PerconaFT/locktree/lock_request.h | 12 ++- .../tests/lock_request_start_retry_race_3.cc | 2 +- storage/tokudb/PerconaFT/src/ydb-internal.h | 1 + storage/tokudb/PerconaFT/src/ydb.cc | 7 ++ storage/tokudb/PerconaFT/src/ydb_row_lock.cc | 15 +++- storage/tokudb/hatoku_hton.cc | 87 +++++++++++++++++-- 9 files changed, 163 insertions(+), 20 deletions(-) diff --git a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc index cadaa48cceaa1..0145d63183951 100644 --- a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc +++ b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc @@ -405,6 +405,7 @@ static void print_db_env_struct (void) { "int (*set_lock_timeout) (DB_ENV *env, uint64_t default_lock_wait_time_msec, uint64_t (*get_lock_wait_time_cb)(uint64_t default_lock_wait_time))", "int (*get_lock_timeout) (DB_ENV *env, uint64_t *lock_wait_time_msec)", "int (*set_lock_timeout_callback) (DB_ENV *env, lock_timeout_callback callback)", + "int (*set_lock_wait_callback) (DB_ENV *env, lock_wait_callback callback)", "int (*txn_xa_recover) (DB_ENV*, TOKU_XA_XID list[/*count*/], long count, /*out*/ long *retp, uint32_t flags)", "int (*get_txn_from_xid) (DB_ENV*, /*in*/ TOKU_XA_XID *, /*out*/ DB_TXN **)", "DB* (*get_db_for_directory) (DB_ENV*)", @@ -751,6 +752,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) { printf("void toku_dbt_array_resize(DBT_ARRAY *dbts, uint32_t size) %s;\n", VISIBLE); printf("typedef void (*lock_timeout_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid);\n"); + printf("typedef void (*lock_wait_callback)(void *arg, uint64_t requesting_txnid, uint64_t blocking_txnid);\n"); printf("typedef int (*iterate_row_locks_callback)(DB **db, DBT *left_key, DBT *right_key, void *extra);\n"); printf("typedef int (*iterate_transactions_callback)(DB_TXN *dbtxn, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n"); printf("typedef int (*iterate_requests_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid, uint64_t start_time, void *extra);\n"); diff --git a/storage/tokudb/PerconaFT/ftcxx/db_env.hpp b/storage/tokudb/PerconaFT/ftcxx/db_env.hpp index 071614b87e907..15b5ce55f7217 100644 --- a/storage/tokudb/PerconaFT/ftcxx/db_env.hpp +++ b/storage/tokudb/PerconaFT/ftcxx/db_env.hpp @@ -202,6 +202,7 @@ namespace ftcxx { typedef uint64_t (*get_lock_wait_time_cb_func)(uint64_t); get_lock_wait_time_cb_func _get_lock_wait_time_cb; lock_timeout_callback _lock_timeout_callback; + lock_wait_callback _lock_wait_needed_callback; uint64_t (*_loader_memory_size_callback)(void); uint32_t _cachesize_gbytes; @@ -231,6 +232,7 @@ namespace ftcxx { _lock_wait_time_msec(0), _get_lock_wait_time_cb(nullptr), _lock_timeout_callback(nullptr), + _lock_wait_needed_callback(nullptr), _loader_memory_size_callback(nullptr), _cachesize_gbytes(0), _cachesize_bytes(0), @@ -296,6 +298,11 @@ namespace ftcxx { handle_ft_retval(r); } + if (_lock_wait_needed_callback) { + r = env->set_lock_wait_callback(env, _lock_wait_needed_callback); + handle_ft_retval(r); + } + if (_loader_memory_size_callback) { env->set_loader_memory_size(env, _loader_memory_size_callback); } @@ -419,6 +426,11 @@ namespace ftcxx { return *this; } + DBEnvBuilder& set_lock_wait_callback(lock_wait_callback callback) { + _lock_wait_needed_callback = callback; + return *this; + } + DBEnvBuilder& set_loader_memory_size(uint64_t (*callback)(void)) { _loader_memory_size_callback = callback; return *this; diff --git a/storage/tokudb/PerconaFT/locktree/lock_request.cc b/storage/tokudb/PerconaFT/locktree/lock_request.cc index 1bc613533dba7..943362e1b9d35 100644 --- a/storage/tokudb/PerconaFT/locktree/lock_request.cc +++ b/storage/tokudb/PerconaFT/locktree/lock_request.cc @@ -199,7 +199,8 @@ int lock_request::wait(uint64_t wait_time_ms) { return wait(wait_time_ms, 0, nullptr); } -int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*killed_callback)(void)) { +int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*killed_callback)(void), + void (*lock_wait_callback)(void *, TXNID, TXNID)) { uint64_t t_now = toku_current_time_microsec(); uint64_t t_start = t_now; uint64_t t_end = t_start + wait_time_ms * 1000; @@ -208,7 +209,13 @@ int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*kil // check again, this time locking out other retry calls if (m_state == state::PENDING) { - retry(); + GrowableArray conflicts_collector; + conflicts_collector.init(); + retry(&conflicts_collector); + if (m_state == state::PENDING) { + report_waits(&conflicts_collector, lock_wait_callback); + } + conflicts_collector.deinit(); } while (m_state == state::PENDING) { @@ -287,7 +294,7 @@ TXNID lock_request::get_conflicting_txnid(void) const { return m_conflicting_txnid; } -int lock_request::retry(void) { +int lock_request::retry(GrowableArray *conflicts_collector) { invariant(m_state == state::PENDING); int r; @@ -308,13 +315,14 @@ int lock_request::retry(void) { toku_cond_broadcast(&m_wait_cond); } else { m_conflicting_txnid = conflicts.get(0); + add_conflicts_to_waits(&conflicts, conflicts_collector); } conflicts.destroy(); return r; } -void lock_request::retry_all_lock_requests(locktree *lt, void (*after_retry_all_test_callback)(void)) { +void lock_request::retry_all_lock_requests(locktree *lt, void (*lock_wait_callback)(void *, TXNID, TXNID), void (*after_retry_all_test_callback)(void)) { lt_lock_request_info *info = lt->get_lock_request_info(); info->retry_want++; @@ -327,6 +335,9 @@ void lock_request::retry_all_lock_requests(locktree *lt, void (*after_retry_all_ toku_mutex_lock(&info->mutex); + GrowableArray conflicts_collector; + conflicts_collector.init(); + // here is the group retry algorithm. // get the latest retry_want count and use it as the generation number of this retry operation. // if this retry generation is > the last retry generation, then do the lock retries. otherwise, @@ -344,7 +355,7 @@ void lock_request::retry_all_lock_requests(locktree *lt, void (*after_retry_all_ // move on to the next lock request. otherwise // the request is gone from the list so we may // read the i'th entry for the next one. - r = request->retry(); + r = request->retry(&conflicts_collector); if (r != 0) { i++; } @@ -354,6 +365,30 @@ void lock_request::retry_all_lock_requests(locktree *lt, void (*after_retry_all_ } toku_mutex_unlock(&info->mutex); + + report_waits(&conflicts_collector, lock_wait_callback); + conflicts_collector.deinit(); +} + +void lock_request::add_conflicts_to_waits(txnid_set *conflicts, + GrowableArray *wait_conflicts) { + size_t num_conflicts = conflicts->size(); + for (size_t i = 0; i < num_conflicts; i++) { + wait_conflicts->push(m_txnid); + wait_conflicts->push(conflicts->get(i)); + } +} + +void lock_request::report_waits(GrowableArray *wait_conflicts, + void (*lock_wait_callback)(void *, TXNID, TXNID)) { + if (!lock_wait_callback) + return; + size_t num_conflicts = wait_conflicts->get_size(); + for (size_t i = 0; i < num_conflicts; i += 2) { + TXNID blocked_txnid = wait_conflicts->fetch_unchecked(i); + TXNID blocking_txnid = wait_conflicts->fetch_unchecked(i+1); + (*lock_wait_callback)(nullptr, blocked_txnid, blocking_txnid); + } } void *lock_request::get_extra(void) const { diff --git a/storage/tokudb/PerconaFT/locktree/lock_request.h b/storage/tokudb/PerconaFT/locktree/lock_request.h index ab69253bcec05..1fa94ef5b962e 100644 --- a/storage/tokudb/PerconaFT/locktree/lock_request.h +++ b/storage/tokudb/PerconaFT/locktree/lock_request.h @@ -89,7 +89,8 @@ class lock_request { // returns: The return code of locktree::acquire_[write,read]_lock() // or simply DB_LOCK_NOTGRANTED if the wait time expired. int wait(uint64_t wait_time_ms); - int wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*killed_callback)(void)); + int wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*killed_callback)(void), + void (*lock_wait_callback)(void *, TXNID, TXNID) = nullptr); // return: left end-point of the lock range const DBT *get_left_key(void) const; @@ -109,7 +110,7 @@ class lock_request { // effect: Retries all of the lock requests for the given locktree. // Any lock requests successfully restarted is completed and woken up. // The rest remain pending. - static void retry_all_lock_requests(locktree *lt, void (*after_retry_test_callback)(void) = nullptr); + static void retry_all_lock_requests(locktree *lt, void (*lock_wait_callback)(void *, TXNID, TXNID) = nullptr, void (*after_retry_test_callback)(void) = nullptr); void set_start_test_callback(void (*f)(void)); void set_start_before_pending_test_callback(void (*f)(void)); @@ -162,7 +163,7 @@ class lock_request { // effect: tries again to acquire the lock described by this lock request // returns: 0 if retrying the request succeeded and is now complete - int retry(void); + int retry(GrowableArray *conflict_collector); void complete(int complete_r); @@ -194,6 +195,11 @@ class lock_request { static int find_by_txnid(lock_request * const &request, const TXNID &txnid); + // Report list of conflicts to lock wait callback. + static void report_waits(GrowableArray *wait_conflicts, + void (*lock_wait_callback)(void *, TXNID, TXNID)); + void add_conflicts_to_waits(txnid_set *conflicts, GrowableArray *wait_conflicts); + void (*m_start_test_callback)(void); void (*m_start_before_pending_test_callback)(void); void (*m_retry_test_callback)(void); diff --git a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc index 92122861819db..8f0d86c9f6437 100644 --- a/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc +++ b/storage/tokudb/PerconaFT/locktree/tests/lock_request_start_retry_race_3.cc @@ -87,7 +87,7 @@ static void run_locker(locktree *lt, TXNID txnid, const DBT *key, pthread_barrie buffer.destroy(); // retry pending lock requests - lock_request::retry_all_lock_requests(lt, after_retry_all); + lock_request::retry_all_lock_requests(lt, nullptr, after_retry_all); } request.destroy(); diff --git a/storage/tokudb/PerconaFT/src/ydb-internal.h b/storage/tokudb/PerconaFT/src/ydb-internal.h index d40f7795b0b86..a1eb43a67c56d 100644 --- a/storage/tokudb/PerconaFT/src/ydb-internal.h +++ b/storage/tokudb/PerconaFT/src/ydb-internal.h @@ -105,6 +105,7 @@ struct __toku_db_env_internal { TOKULOGGER logger; toku::locktree_manager ltm; lock_timeout_callback lock_wait_timeout_callback; // Called when a lock request times out waiting for a lock. + lock_wait_callback lock_wait_needed_callback; // Called when a lock request requires a wait. DB *directory; // Maps dnames to inames DB *persistent_environment; // Stores environment settings, can be used for upgrade diff --git a/storage/tokudb/PerconaFT/src/ydb.cc b/storage/tokudb/PerconaFT/src/ydb.cc index d51ee81700f8e..f99b8dfa5da40 100644 --- a/storage/tokudb/PerconaFT/src/ydb.cc +++ b/storage/tokudb/PerconaFT/src/ydb.cc @@ -1804,6 +1804,12 @@ env_set_lock_timeout_callback(DB_ENV *env, lock_timeout_callback callback) { return 0; } +static int +env_set_lock_wait_callback(DB_ENV *env, lock_wait_callback callback) { + env->i->lock_wait_needed_callback = callback; + return 0; +} + static void format_time(const time_t *timer, char *buf) { ctime_r(timer, buf); @@ -2704,6 +2710,7 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) { USENV(get_lock_timeout); USENV(set_lock_timeout); USENV(set_lock_timeout_callback); + USENV(set_lock_wait_callback); USENV(set_redzone); USENV(log_flush); USENV(log_archive); diff --git a/storage/tokudb/PerconaFT/src/ydb_row_lock.cc b/storage/tokudb/PerconaFT/src/ydb_row_lock.cc index 913e1a44faf3d..597e6311eb840 100644 --- a/storage/tokudb/PerconaFT/src/ydb_row_lock.cc +++ b/storage/tokudb/PerconaFT/src/ydb_row_lock.cc @@ -193,7 +193,10 @@ int toku_db_start_range_lock(DB *db, DB_TXN *txn, const DBT *left_key, const DBT toku::lock_request::type lock_type, toku::lock_request *request) { DB_TXN *txn_anc = txn_oldest_ancester(txn); TXNID txn_anc_id = txn_anc->id64(txn_anc); - request->set(db->i->lt, txn_anc_id, left_key, right_key, lock_type, toku_is_big_txn(txn_anc)); + uint64_t client_id; + void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); + request->set(db->i->lt, txn_anc_id, left_key, right_key, lock_type, toku_is_big_txn(txn_anc), client_extra); const int r = request->start(); if (r == 0) { @@ -221,7 +224,8 @@ int toku_db_wait_range_lock(DB *db, DB_TXN *txn, toku::lock_request *request) { uint64_t killed_time_msec = env->i->default_killed_time_msec; if (env->i->get_killed_time_callback) killed_time_msec = env->i->get_killed_time_callback(killed_time_msec); - const int r = request->wait(wait_time_msec, killed_time_msec, env->i->killed_callback); + const int r = request->wait(wait_time_msec, killed_time_msec, env->i->killed_callback, + env->i->lock_wait_needed_callback); if (r == 0) { db_txn_note_row_lock(db, txn_anc, left_key, right_key); } else if (r == DB_LOCK_NOTGRANTED) { @@ -248,7 +252,10 @@ void toku_db_grab_write_lock (DB *db, DBT *key, TOKUTXN tokutxn) { // This lock request must succeed, so we do not want to wait toku::lock_request request; request.create(); - request.set(db->i->lt, txn_anc_id, key, key, toku::lock_request::type::WRITE, toku_is_big_txn(txn_anc)); + uint64_t client_id; + void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); + request.set(db->i->lt, txn_anc_id, key, key, toku::lock_request::type::WRITE, toku_is_big_txn(txn_anc), client_extra); int r = request.start(); invariant_zero(r); db_txn_note_row_lock(db, txn_anc, key, key); @@ -268,7 +275,7 @@ void toku_db_release_lt_key_ranges(DB_TXN *txn, txn_lt_key_ranges *ranges) { // all of our locks have been released, so first try to wake up // pending lock requests, then release our reference on the lt - toku::lock_request::retry_all_lock_requests(lt); + toku::lock_request::retry_all_lock_requests(lt, txn->mgrp->i->lock_wait_needed_callback); // Release our reference on this locktree toku::locktree_manager *ltm = &txn->mgrp->i->ltm; diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 67adc480914d9..1154840bf3c06 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -55,6 +55,7 @@ static bool tokudb_show_status( static void tokudb_handle_fatal_signal(handlerton* hton, THD* thd, int sig); #endif static int tokudb_close_connection(handlerton* hton, THD* thd); +static void tokudb_kill_query(handlerton *hton, THD *thd, enum thd_kill_levels level); static int tokudb_commit(handlerton* hton, THD* thd, bool all); static int tokudb_rollback(handlerton* hton, THD* thd, bool all); #if TOKU_INCLUDE_XA @@ -147,6 +148,11 @@ static void tokudb_lock_timeout_callback( const DBT* right_key, uint64_t blocking_txnid); +static void tokudb_lock_wait_needed_callback( + void* arg, + uint64_t requesting_txnid, + uint64_t blocking_txnid); + #define ASSERT_MSGLEN 1024 void toku_hton_assert_fail( @@ -331,6 +337,7 @@ static int tokudb_init_func(void *p) { tokudb_hton->create = tokudb_create_handler; tokudb_hton->close_connection = tokudb_close_connection; + tokudb_hton->kill_query = tokudb_kill_query; tokudb_hton->savepoint_offset = sizeof(SP_INFO_T); tokudb_hton->savepoint_set = tokudb_savepoint; @@ -532,6 +539,7 @@ static int tokudb_init_func(void *p) { db_env->set_lock_timeout_callback(db_env, tokudb_lock_timeout_callback); db_env->set_dir_per_db(db_env, tokudb::sysvars::dir_per_db); + db_env->set_lock_wait_callback(db_env, tokudb_lock_wait_needed_callback); db_env->set_loader_memory_size( db_env, @@ -754,6 +762,12 @@ static int tokudb_close_connection(handlerton* hton, THD* thd) { return error; } +void tokudb_kill_query(handlerton *hton, THD *thd, enum thd_kill_levels level) { + TOKUDB_DBUG_ENTER(""); + db_env->kill_waiter(db_env, thd); + DBUG_VOID_RETURN; +} + bool tokudb_flush_logs(handlerton * hton) { TOKUDB_DBUG_ENTER(""); int error; @@ -873,9 +887,9 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { tokudb_sync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; TOKUDB_TRACE_FOR_FLAGS( TOKUDB_DEBUG_TXN, - "commit trx %u txn %p syncflag %u", + "commit trx %u txn %p %" PRIu64 " syncflag %u", all, - this_txn, + this_txn, this_txn->id64(this_txn), syncflag); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_commit_before", DBUG_SUICIDE();); @@ -904,9 +918,9 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { if (this_txn) { TOKUDB_TRACE_FOR_FLAGS( TOKUDB_DEBUG_TXN, - "rollback %u txn %p", + "rollback %u txn %p %" PRIu64, all, - this_txn); + this_txn, this_txn->id64(this_txn)); tokudb_cleanup_handlers(trx, this_txn); abort_txn_with_progress(this_txn, thd); *txn = NULL; @@ -952,9 +966,9 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { uint32_t syncflag = tokudb_sync_on_prepare() ? 0 : DB_TXN_NOSYNC; TOKUDB_TRACE_FOR_FLAGS( TOKUDB_DEBUG_XA, - "doing txn prepare:%d:%p", + "doing txn prepare:%d:%p %" PRIu64, all, - txn); + txn, txn->id64(txn)); // a TOKU_XA_XID is identical to a MYSQL_XID TOKU_XA_XID thd_xid; thd_get_xid(thd, (MYSQL_XID*) &thd_xid); @@ -1570,7 +1584,9 @@ static int tokudb_search_txn_callback( void* extra) { uint64_t txn_id = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); + uint64_t client_id; + void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); struct tokudb_search_txn_extra* e = reinterpret_cast(extra); if (e->match_txn_id == txn_id) { @@ -1748,6 +1764,63 @@ static void tokudb_lock_timeout_callback( } } +extern "C" int thd_rpl_deadlock_check(MYSQL_THD thd, MYSQL_THD other_thd); + +struct tokudb_search_txn_thd { + bool match_found; + uint64_t match_txn_id; + THD *match_client_thd; +}; + +static int tokudb_search_txn_thd_callback( + DB_TXN* txn, + iterate_row_locks_callback iterate_locks, + void* locks_extra, + void* extra) { + + uint64_t txn_id = txn->id64(txn); + uint64_t client_id; + void *client_extra; + txn->get_client_id(txn, &client_id, &client_extra); + struct tokudb_search_txn_thd* e = + reinterpret_cast(extra); + if (e->match_txn_id == txn_id) { + e->match_found = true; + e->match_client_thd = reinterpret_cast(client_extra); + return 1; + } + return 0; +} + +static bool tokudb_txn_id_to_thd( + uint64_t txnid, + THD **out_thd) { + + struct tokudb_search_txn_thd e = { + false, + txnid, + 0 + }; + db_env->iterate_live_transactions(db_env, tokudb_search_txn_thd_callback, &e); + if (e.match_found) { + *out_thd = e.match_client_thd; + } + return e.match_found; +} + +static void tokudb_lock_wait_needed_callback( + void *arg, + uint64_t requesting_txnid, + uint64_t blocking_txnid) { + + THD *requesting_thd; + THD *blocking_thd; + if (tokudb_txn_id_to_thd(requesting_txnid, &requesting_thd) && + tokudb_txn_id_to_thd(blocking_txnid, &blocking_thd)) { + thd_rpl_deadlock_check (requesting_thd, blocking_thd); + } +} + // Retrieves variables for information_schema.global_status. // Names (columnname) are automatically converted to upper case, // and prefixed with "TOKUDB_" From 021f78f6956d2f33cfbee3b3a2a20e9c822e0c98 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 23 Nov 2016 16:44:03 +0100 Subject: [PATCH 206/295] Use thd_kill_level() over old thd_killed() in TokuDB. --- storage/tokudb/ha_tokudb.cc | 16 ++++++++-------- storage/tokudb/ha_tokudb_admin.cc | 8 ++++---- storage/tokudb/hatoku_hton.h | 4 ++-- storage/tokudb/tokudb_information_schema.cc | 12 ++++++------ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 7e9e6100c6e04..24682fe40d65f 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -533,7 +533,7 @@ typedef struct index_read_info { static int ai_poll_fun(void *extra, float progress) { LOADER_CONTEXT context = (LOADER_CONTEXT)extra; - if (thd_killed(context->thd)) { + if (thd_kill_level(context->thd)) { sprintf(context->write_status_msg, "The process has been killed, aborting add index."); return ER_ABORTING_CONNECTION; } @@ -548,7 +548,7 @@ static int ai_poll_fun(void *extra, float progress) { static int loader_poll_fun(void *extra, float progress) { LOADER_CONTEXT context = (LOADER_CONTEXT)extra; - if (thd_killed(context->thd)) { + if (thd_kill_level(context->thd)) { sprintf(context->write_status_msg, "The process has been killed, aborting bulk load."); return ER_ABORTING_CONNECTION; } @@ -3435,7 +3435,7 @@ int ha_tokudb::end_bulk_insert(bool abort) { ai_metadata_update_required = false; loader_error = 0; if (loader) { - if (!abort_loader && !thd_killed(thd)) { + if (!abort_loader && !thd_kill_level(thd)) { DBUG_EXECUTE_IF("tokudb_end_bulk_insert_sleep", { const char *orig_proc_info = tokudb_thd_get_proc_info(thd); thd_proc_info(thd, "DBUG sleep"); @@ -3445,7 +3445,7 @@ int ha_tokudb::end_bulk_insert(bool abort) { error = loader->close(loader); loader = NULL; if (error) { - if (thd_killed(thd)) { + if (thd_kill_level(thd)) { my_error(ER_QUERY_INTERRUPTED, MYF(0)); } goto cleanup; @@ -3580,7 +3580,7 @@ int ha_tokudb::is_index_unique(bool* is_unique, DB_TXN* txn, DB* db, KEY* key_in share->row_count(), key_info->name); thd_proc_info(thd, status_msg); - if (thd_killed(thd)) { + if (thd_kill_level(thd)) { my_error(ER_QUERY_INTERRUPTED, MYF(0)); error = ER_QUERY_INTERRUPTED; goto cleanup; @@ -5245,7 +5245,7 @@ int ha_tokudb::fill_range_query_buf( // otherwise, if we simply see that the current key is no match, // we tell the cursor to continue and don't store // the key locally - if (result == ICP_OUT_OF_RANGE || thd_killed(thd)) { + if (result == ICP_OUT_OF_RANGE || thd_kill_level(thd)) { icp_went_out_of_range = true; error = 0; DEBUG_SYNC(ha_thd(), "tokudb_icp_asc_scan_out_of_range"); @@ -5613,7 +5613,7 @@ int ha_tokudb::get_next( static_cast(thd_get_ha_data(thd, tokudb_hton)); trx->stmt_progress.queried++; track_progress(thd); - if (thd_killed(thd)) + if (thd_kill_level(thd)) error = ER_ABORTING_CONNECTION; } cleanup: @@ -8351,7 +8351,7 @@ int ha_tokudb::tokudb_add_index( (long long unsigned)share->row_count()); #endif - if (thd_killed(thd)) { + if (thd_kill_level(thd)) { error = ER_ABORTING_CONNECTION; goto cleanup; } diff --git a/storage/tokudb/ha_tokudb_admin.cc b/storage/tokudb/ha_tokudb_admin.cc index 6d8e7173c8d8b..e1443101bb69f 100644 --- a/storage/tokudb/ha_tokudb_admin.cc +++ b/storage/tokudb/ha_tokudb_admin.cc @@ -225,7 +225,7 @@ int recount_rows_t::analyze_recount_rows_progress( _ticks = 0; uint64_t now = tokudb::time::microsec(); _total_elapsed_time = now - _recount_start; - if ((_thd && thd_killed(_thd)) || cancelled()) { + if ((_thd && thd_kill_level(_thd)) || cancelled()) { // client killed return ER_ABORTING_CONNECTION; } @@ -540,7 +540,7 @@ int standard_t::analyze_key_progress(void) { uint64_t now = tokudb::time::microsec(); _total_elapsed_time = now - _analyze_start; _key_elapsed_time = now - _analyze_key_start; - if ((_thd && thd_killed(_thd)) || cancelled()) { + if ((_thd && thd_kill_level(_thd)) || cancelled()) { // client killed return ER_ABORTING_CONNECTION; } else if (_time_limit > 0 && @@ -876,7 +876,7 @@ typedef struct hot_optimize_context { static int hot_optimize_progress_fun(void *extra, float progress) { HOT_OPTIMIZE_CONTEXT context = (HOT_OPTIMIZE_CONTEXT)extra; - if (thd_killed(context->thd)) { + if (thd_kill_level(context->thd)) { sprintf( context->write_status_msg, "The process has been killed, aborting hot optimize."); @@ -1003,7 +1003,7 @@ struct check_context { static int ha_tokudb_check_progress(void* extra, float progress) { struct check_context* context = (struct check_context*)extra; int result = 0; - if (thd_killed(context->thd)) + if (thd_kill_level(context->thd)) result = ER_ABORTING_CONNECTION; return result; } diff --git a/storage/tokudb/hatoku_hton.h b/storage/tokudb/hatoku_hton.h index ade7be128a506..d126ff4339f26 100644 --- a/storage/tokudb/hatoku_hton.h +++ b/storage/tokudb/hatoku_hton.h @@ -172,12 +172,12 @@ inline uint64_t tokudb_get_killed_time_callback(uint64_t default_killed_time) { inline int tokudb_killed_callback(void) { THD *thd = current_thd; - return thd_killed(thd); + return thd_kill_level(thd); } inline bool tokudb_killed_thd_callback(void *extra, uint64_t deleted_rows) { THD *thd = static_cast(extra); - return thd_killed(thd) != 0; + return thd_kill_level(thd) != 0; } diff --git a/storage/tokudb/tokudb_information_schema.cc b/storage/tokudb/tokudb_information_schema.cc index 86af3f14f9190..2b4128cc03a27 100644 --- a/storage/tokudb/tokudb_information_schema.cc +++ b/storage/tokudb/tokudb_information_schema.cc @@ -87,7 +87,7 @@ int trx_callback( uint64_t tnow = (uint64_t) ::time(NULL); table->field[2]->store(tnow >= start_time ? tnow - start_time : 0, false); int error = schema_table_store_record(thd, table); - if (!error && thd_killed(thd)) + if (!error && thd_kill_level(thd)) error = ER_QUERY_INTERRUPTED; return error; } @@ -221,7 +221,7 @@ int lock_waits_callback( int error = schema_table_store_record(thd, table); - if (!error && thd_killed(thd)) + if (!error && thd_kill_level(thd)) error = ER_QUERY_INTERRUPTED; return error; @@ -365,7 +365,7 @@ int locks_callback( error = schema_table_store_record(thd, table); - if (!error && thd_killed(thd)) + if (!error && thd_kill_level(thd)) error = ER_QUERY_INTERRUPTED; } return error; @@ -497,7 +497,7 @@ int report_file_map(TABLE* table, THD* thd) { error = schema_table_store_record(thd, table); } - if (!error && thd_killed(thd)) + if (!error && thd_kill_level(thd)) error = ER_QUERY_INTERRUPTED; } if (error == DB_NOTFOUND) { @@ -702,7 +702,7 @@ int report_fractal_tree_info(TABLE* table, THD* thd) { if (error) error = 0; // ignore read uncommitted errors } - if (!error && thd_killed(thd)) + if (!error && thd_kill_level(thd)) error = ER_QUERY_INTERRUPTED; } if (error == DB_NOTFOUND) { @@ -993,7 +993,7 @@ int report_fractal_tree_block_map(TABLE* table, THD* thd) { table, thd); } - if (!error && thd_killed(thd)) + if (!error && thd_kill_level(thd)) error = ER_QUERY_INTERRUPTED; } if (error == DB_NOTFOUND) { From 1d8eafbeafbc4a77153d6ceb974dbbeff543e648 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 24 Nov 2016 15:55:55 +0400 Subject: [PATCH 207/295] Removing the unused function my_bincmp() from strings/ctype-ucs2.c --- strings/ctype-ucs2.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c index f2bcf69bbc604..c080e344b352e 100644 --- a/strings/ctype-ucs2.c +++ b/strings/ctype-ucs2.c @@ -53,17 +53,6 @@ static unsigned long lfactor[9]= #ifdef HAVE_CHARSET_mb2_or_mb4 -static inline int -my_bincmp(const uchar *s, const uchar *se, - const uchar *t, const uchar *te) -{ - int slen= (int) (se - s), tlen= (int) (te - t); - int len= MY_MIN(slen, tlen); - int cmp= memcmp(s, t, len); - return cmp ? cmp : slen - tlen; -} - - static size_t my_caseup_str_mb2_or_mb4(CHARSET_INFO * cs __attribute__((unused)), char * s __attribute__((unused))) From 8da33e3a868e7ab4472823f8f3bfab3d5cd1833e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 25 Nov 2016 06:09:00 +0200 Subject: [PATCH 208/295] MDEV-11349 (1/2) Fix some clang 4.0 warnings In InnoDB and XtraDB functions that declare pointer parameters as nonnull, remove nullness checks, because GCC would optimize them away anyway. Use #ifdef instead of #if when checking for a configuration flag. Clang says that left shifts of negative values are undefined. So, use ~0U instead of ~0 in a number of macros. Some functions that were defined as UNIV_INLINE were declared as UNIV_INTERN. Consistently use the same type of linkage. ibuf_merge_or_delete_for_page() could pass bitmap_page=NULL to buf_page_print(), conflicting with the __attribute__((nonnull)). --- storage/innobase/btr/btr0cur.cc | 4 ++-- storage/innobase/dict/dict0dict.cc | 7 +------ storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/ibuf/ibuf0ibuf.cc | 10 ++++++++-- storage/innobase/include/buf0buf.h | 2 +- storage/innobase/include/dict0mem.h | 12 ++++++------ storage/innobase/include/fil0pagecompress.h | 18 +++++++++--------- storage/innobase/include/fsp0fsp.h | 8 ++++---- storage/innobase/lock/lock0lock.cc | 1 - storage/innobase/row/row0merge.cc | 5 +++-- storage/innobase/row/row0upd.cc | 3 --- storage/innobase/sync/sync0sync.cc | 3 --- storage/xtradb/btr/btr0cur.cc | 8 +++----- storage/xtradb/buf/buf0buf.cc | 9 +-------- storage/xtradb/dict/dict0dict.cc | 7 +------ storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/ibuf/ibuf0ibuf.cc | 10 ++++++++-- storage/xtradb/include/dict0mem.h | 12 ++++++------ storage/xtradb/include/fil0pagecompress.h | 18 +++++++++--------- storage/xtradb/include/fsp0fsp.h | 15 +++++++-------- storage/xtradb/include/page0page.ic | 3 --- storage/xtradb/lock/lock0lock.cc | 1 - storage/xtradb/page/page0zip.cc | 4 ---- storage/xtradb/row/row0merge.cc | 5 +++-- storage/xtradb/row/row0upd.cc | 3 --- storage/xtradb/sync/sync0sync.cc | 3 --- 26 files changed, 74 insertions(+), 101 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 18f193094e215..77b915bce927a 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3038,7 +3038,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && (thr != NULL)) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr)->id, index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -3186,7 +3186,7 @@ btr_cur_del_mark_set_sec_rec( rec = btr_cur_get_rec(cursor); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && (thr != NULL)) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr)->id, cursor->index, "del mark "); rec_print(stderr, rec, cursor->index); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 0310e5e1d66a6..4685ba3085d90 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -6317,15 +6317,10 @@ dict_set_corrupted_index_cache_only( /* Mark the table as corrupted only if the clustered index is corrupted */ if (dict_index_is_clust(index)) { - dict_table_t* corrupt_table; - - corrupt_table = (table != NULL) ? table : index->table; ut_ad((index->table != NULL) || (table != NULL) || index->table == table); - if (corrupt_table) { - corrupt_table->corrupted = TRUE; - } + table->corrupted = TRUE; } index->type |= DICT_CORRUPT; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 79d8ae0dc81a6..553d48aa48f3f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18659,7 +18659,7 @@ wsrep_fake_trx_id( trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&trx_sys->mutex); - (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); + wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } #endif /* WITH_WSREP */ diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index fcf0388d80a89..08f2967a4957a 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -4690,8 +4690,14 @@ ibuf_merge_or_delete_for_page( bitmap_page = ibuf_bitmap_get_map_page(space, page_no, zip_size, &mtr); - buf_page_print(bitmap_page, 0, - BUF_PAGE_PRINT_NO_CRASH); + if (bitmap_page == NULL) + { + fputs("InnoDB: cannot retrieve bitmap page\n", + stderr); + } else { + buf_page_print(bitmap_page, 0, + BUF_PAGE_PRINT_NO_CRASH); + } ibuf_mtr_commit(&mtr); fputs("\nInnoDB: Dump of the page:\n", stderr); diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index c737f3a6f1dd7..d7de9d81ca2f4 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1920,7 +1920,7 @@ class HazardPointer { /** Buffer pool instance */ const buf_pool_t* m_buf_pool; -#if UNIV_DEBUG +#ifdef UNIV_DEBUG /** mutex that protects access to the m_hp. */ const ib_mutex_t* m_mutex; #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index f2d72de39e5ce..902b960eaa386 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -159,7 +159,7 @@ DEFAULT=0, ON = 1, OFF = 2 + DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY) /** A mask of all the known/used bits in table flags */ -#define DICT_TF_BIT_MASK (~(~0 << DICT_TF_BITS)) +#define DICT_TF_BIT_MASK (~(~0U << DICT_TF_BITS)) /** Zero relative shift position of the COMPACT field */ #define DICT_TF_POS_COMPACT 0 @@ -208,23 +208,23 @@ DEFAULT=0, ON = 1, OFF = 2 << DICT_TF_POS_DATA_DIR) /** Bit mask of the PAGE_COMPRESSION field */ #define DICT_TF_MASK_PAGE_COMPRESSION \ - ((~(~0 << DICT_TF_WIDTH_PAGE_COMPRESSION)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_COMPRESSION)) \ << DICT_TF_POS_PAGE_COMPRESSION) /** Bit mask of the PAGE_COMPRESSION_LEVEL field */ #define DICT_TF_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0 << DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << DICT_TF_POS_PAGE_COMPRESSION_LEVEL) /** Bit mask of the ATOMIC_WRITES field */ #define DICT_TF_MASK_ATOMIC_WRITES \ - ((~(~0 << DICT_TF_WIDTH_ATOMIC_WRITES)) \ + ((~(~0U << DICT_TF_WIDTH_ATOMIC_WRITES)) \ << DICT_TF_POS_ATOMIC_WRITES) /** Bit mask of the PAGE_ENCRYPTION field */ #define DICT_TF_MASK_PAGE_ENCRYPTION \ - ((~(~0 << DICT_TF_WIDTH_PAGE_ENCRYPTION)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_ENCRYPTION)) \ << DICT_TF_POS_PAGE_ENCRYPTION) /** Bit mask of the PAGE_ENCRYPTION_KEY field */ #define DICT_TF_MASK_PAGE_ENCRYPTION_KEY \ - ((~(~0 << DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY)) \ << DICT_TF_POS_PAGE_ENCRYPTION_KEY) /** Return the value of the COMPACT field */ diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 99a05f14ffeb8..10db59fb21800 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -34,7 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com Returns the page compression level flag of the space, or 0 if the space is not compressed. The tablespace must be cached in the memory cache. @return page compression level if page compressed, ULINT_UNDEFINED if space not found */ -UNIV_INTERN +UNIV_INLINE ulint fil_space_get_page_compression_level( /*=================================*/ @@ -43,7 +43,7 @@ fil_space_get_page_compression_level( Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ -UNIV_INTERN +UNIV_INLINE ibool fil_space_is_page_compressed( /*=========================*/ @@ -61,7 +61,7 @@ fil_space_get_page_compressed( Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ -UNIV_INTERN +UNIV_INLINE atomic_writes_t fil_space_get_atomic_writes( /*=========================*/ @@ -69,7 +69,7 @@ fil_space_get_atomic_writes( /*******************************************************************//** Find out wheather the page is index page or not @return true if page type index page, false if not */ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_index_page( /*===================*/ @@ -79,7 +79,7 @@ fil_page_is_index_page( Get the name of the compression algorithm used for page compression. @return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INTERN +UNIV_INLINE const char* fil_get_compression_alg_name( /*=========================*/ @@ -136,7 +136,7 @@ fil_node_get_space_id( /****************************************************************//** Get block size from fil node @return block size*/ -UNIV_INTERN +UNIV_INLINE ulint fil_node_get_block_size( fil_node_t* node); /*!< in: Node where to get block @@ -144,7 +144,7 @@ fil_node_get_block_size( /*******************************************************************//** Find out wheather the page is page compressed @return true if page is page compressed*/ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_compressed( /*===================*/ @@ -153,7 +153,7 @@ fil_page_is_compressed( /*******************************************************************//** Find out wheather the page is page compressed @return true if page is page compressed*/ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_compressed_encrypted( /*=============================*/ @@ -162,7 +162,7 @@ fil_page_is_compressed_encrypted( /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method*/ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_lzo_compressed( /*=======================*/ diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index b9ff05b4bd40c..abcd5721a4713 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -129,19 +129,19 @@ dictionary */ << FSP_FLAGS_POS_DATA_DIR) /** Bit mask of the DATA_DIR field */ #define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0 << FSP_FLAGS_WIDTH_DATA_DIR)) \ + ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ << FSP_FLAGS_POS_DATA_DIR_ORACLE) /** Bit mask of the PAGE_COMPRESSION field */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) /** Bit mask of the PAGE_COMPRESSION_LEVEL field */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) /** Bit mask of the ATOMIC_WRITES field */ #define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ + ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ << FSP_FLAGS_POS_ATOMIC_WRITES) /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index c4f9b767afee4..9f15c7b3b372b 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -2366,7 +2366,6 @@ lock_queue_validate( ulint space; ulint page_no; ulint rec_fold; - hash_table_t* hash; hash_cell_t* cell; lock_t* next; bool wait_lock = false; diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index bdedd0da85c33..12b3cf1f57287 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4192,9 +4192,10 @@ row_merge_build_indexes( (total_static_cost + total_dynamic_cost) * PCT_COST_MERGESORT_INDEX * 100; - bufend = innobase_convert_name(buf, sizeof buf, + bufend = innobase_convert_name( + buf, sizeof buf, indexes[i]->name, strlen(indexes[i]->name), - trx ? trx->mysql_thd : NULL, + trx->mysql_thd, FALSE); buf[bufend - buf]='\0'; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 60b490228b18a..54bf50cba3d1e 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -2789,9 +2789,6 @@ row_upd( { dberr_t err = DB_SUCCESS; - ut_ad(node != NULL); - ut_ad(thr != NULL); - if (UNIV_LIKELY(node->in_mysql_interface)) { /* We do not get the cmpl_info value from the MySQL diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 3e3ce353724d4..e4e5bd7109ff3 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -222,9 +222,6 @@ UNIV_INTERN mysql_pfs_key_t mutex_list_mutex_key; /** Latching order checks start when this is set TRUE */ UNIV_INTERN ibool sync_order_checks_on = FALSE; -/** Number of slots reserved for each OS thread in the sync level array */ -static const ulint SYNC_THREAD_N_LEVELS = 10000; - /** Array for tracking sync levels per thread. */ typedef std::vector sync_arr_t; diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index 5449397f832c1..afe2c18159417 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -1862,9 +1862,7 @@ btr_cur_upd_lock_and_undo( const rec_t* rec; dberr_t err; - ut_ad((thr != NULL) || (flags & BTR_NO_LOCKING_FLAG)); - - if (UNIV_UNLIKELY(thr && thr_get_trx(thr)->fake_changes)) { + if (UNIV_UNLIKELY(thr_get_trx(thr)->fake_changes)) { /* skip LOCK, UNDO */ return(DB_SUCCESS); } @@ -3211,7 +3209,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && (thr != NULL)) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr)->id, index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -3369,7 +3367,7 @@ btr_cur_del_mark_set_sec_rec( rec = btr_cur_get_rec(cursor); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && (thr != NULL)) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr)->id, cursor->index, "del mark "); rec_print(stderr, rec, cursor->index); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 8cf8ab3975e24..50a13da262c1f 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -946,13 +946,6 @@ buf_page_print( #endif /* !UNIV_HOTBACKUP */ ulint size = zip_size; - if (!read_buf) { - fprintf(stderr, - " InnoDB: Not dumping page as (in memory) pointer " - "is NULL\n"); - return; - } - if (!size) { size = UNIV_PAGE_SIZE; } @@ -4702,7 +4695,7 @@ buf_page_io_complete( if (io_type == BUF_IO_READ) { ulint read_page_no; ulint read_space_id; - byte* frame = NULL; + byte* frame; if (!buf_page_decrypt_after_read(bpage)) { /* encryption error! */ diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 2f0b09ec41996..ae928306db2ac 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -6317,15 +6317,10 @@ dict_set_corrupted_index_cache_only( /* Mark the table as corrupted only if the clustered index is corrupted */ if (dict_index_is_clust(index)) { - dict_table_t* corrupt_table; - - corrupt_table = (table != NULL) ? table : index->table; ut_ad((index->table != NULL) || (table != NULL) || index->table == table); - if (corrupt_table) { - corrupt_table->corrupted = TRUE; - } + table->corrupted = TRUE; } index->type |= DICT_CORRUPT; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 01014a553ec0d..fd3ef64467380 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -20248,7 +20248,7 @@ wsrep_fake_trx_id( trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&trx_sys->mutex); - (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); + wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } #endif /* WITH_WSREP */ diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index 13597d384335f..c1d735eecdd95 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -4732,8 +4732,14 @@ ibuf_merge_or_delete_for_page( bitmap_page = ibuf_bitmap_get_map_page(space, page_no, zip_size, &mtr); - buf_page_print(bitmap_page, 0, - BUF_PAGE_PRINT_NO_CRASH); + if (bitmap_page == NULL) + { + fputs("InnoDB: cannot retrieve bitmap page\n", + stderr); + } else { + buf_page_print(bitmap_page, 0, + BUF_PAGE_PRINT_NO_CRASH); + } ibuf_mtr_commit(&mtr); fputs("\nInnoDB: Dump of the page:\n", stderr); diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 29f5b8fe8145d..96c85cd8a9902 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -164,7 +164,7 @@ DEFAULT=0, ON = 1, OFF = 2 + DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY) /** A mask of all the known/used bits in table flags */ -#define DICT_TF_BIT_MASK (~(~0 << DICT_TF_BITS)) +#define DICT_TF_BIT_MASK (~(~0U << DICT_TF_BITS)) /** Zero relative shift position of the COMPACT field */ #define DICT_TF_POS_COMPACT 0 @@ -215,23 +215,23 @@ DEFAULT=0, ON = 1, OFF = 2 << DICT_TF_POS_DATA_DIR) /** Bit mask of the PAGE_COMPRESSION field */ #define DICT_TF_MASK_PAGE_COMPRESSION \ - ((~(~0 << DICT_TF_WIDTH_PAGE_COMPRESSION)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_COMPRESSION)) \ << DICT_TF_POS_PAGE_COMPRESSION) /** Bit mask of the PAGE_COMPRESSION_LEVEL field */ #define DICT_TF_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0 << DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << DICT_TF_POS_PAGE_COMPRESSION_LEVEL) /** Bit mask of the ATOMIC_WRITES field */ #define DICT_TF_MASK_ATOMIC_WRITES \ - ((~(~0 << DICT_TF_WIDTH_ATOMIC_WRITES)) \ + ((~(~0U << DICT_TF_WIDTH_ATOMIC_WRITES)) \ << DICT_TF_POS_ATOMIC_WRITES) /** Bit mask of the PAGE_ENCRYPTION field */ #define DICT_TF_MASK_PAGE_ENCRYPTION \ - ((~(~0L << DICT_TF_WIDTH_PAGE_ENCRYPTION)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_ENCRYPTION)) \ << DICT_TF_POS_PAGE_ENCRYPTION) /** Bit mask of the PAGE_ENCRYPTION_KEY field */ #define DICT_TF_MASK_PAGE_ENCRYPTION_KEY \ - ((~(~0L << DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY)) \ + ((~(~0U << DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY)) \ << DICT_TF_POS_PAGE_ENCRYPTION_KEY) /** Return the value of the COMPACT field */ diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 99a05f14ffeb8..10db59fb21800 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -34,7 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com Returns the page compression level flag of the space, or 0 if the space is not compressed. The tablespace must be cached in the memory cache. @return page compression level if page compressed, ULINT_UNDEFINED if space not found */ -UNIV_INTERN +UNIV_INLINE ulint fil_space_get_page_compression_level( /*=================================*/ @@ -43,7 +43,7 @@ fil_space_get_page_compression_level( Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ -UNIV_INTERN +UNIV_INLINE ibool fil_space_is_page_compressed( /*=========================*/ @@ -61,7 +61,7 @@ fil_space_get_page_compressed( Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ -UNIV_INTERN +UNIV_INLINE atomic_writes_t fil_space_get_atomic_writes( /*=========================*/ @@ -69,7 +69,7 @@ fil_space_get_atomic_writes( /*******************************************************************//** Find out wheather the page is index page or not @return true if page type index page, false if not */ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_index_page( /*===================*/ @@ -79,7 +79,7 @@ fil_page_is_index_page( Get the name of the compression algorithm used for page compression. @return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INTERN +UNIV_INLINE const char* fil_get_compression_alg_name( /*=========================*/ @@ -136,7 +136,7 @@ fil_node_get_space_id( /****************************************************************//** Get block size from fil node @return block size*/ -UNIV_INTERN +UNIV_INLINE ulint fil_node_get_block_size( fil_node_t* node); /*!< in: Node where to get block @@ -144,7 +144,7 @@ fil_node_get_block_size( /*******************************************************************//** Find out wheather the page is page compressed @return true if page is page compressed*/ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_compressed( /*===================*/ @@ -153,7 +153,7 @@ fil_page_is_compressed( /*******************************************************************//** Find out wheather the page is page compressed @return true if page is page compressed*/ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_compressed_encrypted( /*=============================*/ @@ -162,7 +162,7 @@ fil_page_is_compressed_encrypted( /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method*/ -UNIV_INTERN +UNIV_INLINE ibool fil_page_is_lzo_compressed( /*=======================*/ diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 9212328f72488..551a8c9ef97a0 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -128,21 +128,20 @@ dictionary */ << FSP_FLAGS_POS_DATA_DIR) /** Bit mask of the DATA_DIR field */ #define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0 << FSP_FLAGS_WIDTH_DATA_DIR)) \ + ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ << FSP_FLAGS_POS_DATA_DIR_ORACLE) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) /** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) /** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ +#define FSP_FLAGS_MASK_ATOMIC_WRITES \ + ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ << FSP_FLAGS_POS_ATOMIC_WRITES) - /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ ((flags & FSP_FLAGS_MASK_POST_ANTELOPE) \ diff --git a/storage/xtradb/include/page0page.ic b/storage/xtradb/include/page0page.ic index efa5e855eb76f..5cf92fd5d8d98 100644 --- a/storage/xtradb/include/page0page.ic +++ b/storage/xtradb/include/page0page.ic @@ -275,9 +275,6 @@ page_is_leaf( /*=========*/ const page_t* page) /*!< in: page */ { - if (!page) { - return(FALSE); - } return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); } diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index c56bc5e52edc6..13edfb2e7ea71 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -2156,7 +2156,6 @@ lock_queue_validate( ulint space; ulint page_no; ulint rec_fold; - hash_table_t* hash; hash_cell_t* cell; lock_t* next; bool wait_lock = false; diff --git a/storage/xtradb/page/page0zip.cc b/storage/xtradb/page/page0zip.cc index 63d6fe225c25b..04340c0f3d2ea 100644 --- a/storage/xtradb/page/page0zip.cc +++ b/storage/xtradb/page/page0zip.cc @@ -1250,10 +1250,6 @@ page_zip_compress( anytime. */ my_bool cmp_per_index_enabled = srv_cmp_per_index_enabled; - if (!page) { - return(FALSE); - } - ut_a(page_is_comp(page)); ut_a(fil_page_get_type(page) == FIL_PAGE_INDEX); ut_ad(page_simple_validate_new((page_t*) page)); diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 9f154a712f453..22ae1eac4527b 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -4203,9 +4203,10 @@ row_merge_build_indexes( (total_static_cost + total_dynamic_cost) * PCT_COST_MERGESORT_INDEX * 100; - bufend = innobase_convert_name(buf, sizeof buf, + bufend = innobase_convert_name( + buf, sizeof buf, indexes[i]->name, strlen(indexes[i]->name), - trx ? trx->mysql_thd : NULL, + trx->mysql_thd, FALSE); buf[bufend - buf]='\0'; diff --git a/storage/xtradb/row/row0upd.cc b/storage/xtradb/row/row0upd.cc index 924b4ccca0d85..6288251da77fc 100644 --- a/storage/xtradb/row/row0upd.cc +++ b/storage/xtradb/row/row0upd.cc @@ -2822,9 +2822,6 @@ row_upd( { dberr_t err = DB_SUCCESS; - ut_ad(node != NULL); - ut_ad(thr != NULL); - if (UNIV_LIKELY(node->in_mysql_interface)) { /* We do not get the cmpl_info value from the MySQL diff --git a/storage/xtradb/sync/sync0sync.cc b/storage/xtradb/sync/sync0sync.cc index 7458a05222414..5c4b45eb3c099 100644 --- a/storage/xtradb/sync/sync0sync.cc +++ b/storage/xtradb/sync/sync0sync.cc @@ -223,9 +223,6 @@ UNIV_INTERN mysql_pfs_key_t mutex_list_mutex_key; /** Latching order checks start when this is set TRUE */ UNIV_INTERN ibool sync_order_checks_on = FALSE; -/** Number of slots reserved for each OS thread in the sync level array */ -static const ulint SYNC_THREAD_N_LEVELS = 10000; - /** Array for tracking sync levels per thread. */ typedef std::vector sync_arr_t; From 3bec0b327c77456c826040fd4591844d37c58e79 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 23 Nov 2016 16:45:31 +0100 Subject: [PATCH 209/295] Parallel replication test case for TokuDB. --- .../r/rpl_parallel_optimistic.result | 494 ++++++++++++++++++ .../tokudb_rpl/t/rpl_parallel_optimistic.test | 478 +++++++++++++++++ 2 files changed, 972 insertions(+) create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/r/rpl_parallel_optimistic.result create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/t/rpl_parallel_optimistic.test diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_parallel_optimistic.result b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_parallel_optimistic.result new file mode 100644 index 0000000000000..8f662f3db060d --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_parallel_optimistic.result @@ -0,0 +1,494 @@ +include/master-slave.inc +[connection master] +ALTER TABLE mysql.gtid_slave_pos ENGINE=TokuDB; +CREATE TABLE t1 (a int PRIMARY KEY, b INT, UNIQUE KEY (b)) ENGINE=TokuDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +CHANGE MASTER TO master_use_gtid=slave_pos; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; +INSERT INTO t1 VALUES(1,1); +BEGIN; +INSERT INTO t1 VALUES(2,2); +INSERT INTO t1 VALUES(3,3); +COMMIT; +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,2); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,6); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,4); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,5); +DELETE FROM t1 WHERE a=3; +INSERT INTO t1 VALUES(3,3); +DELETE FROM t1 WHERE a=1; +INSERT INTO t1 VALUES(1,4); +DELETE FROM t1 WHERE a=3; +INSERT INTO t1 VALUES(3,3); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,6); +include/save_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 4 +2 6 +3 3 +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 4 +2 6 +3 3 +*** Test a bunch of non-transactional/DDL event groups. *** +include/stop_slave.inc +INSERT INTO t1 VALUES (4,8); +INSERT INTO t1 VALUES (5,9); +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=TokuDB; +INSERT INTO t2 VALUES (1); +CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=MyISAM; +ALTER TABLE t2 ADD b INT; +INSERT INTO t2 VALUES (2,2); +ALTER TABLE t2 DROP b; +INSERT INTO t2 VALUES (3); +ALTER TABLE t2 ADD c INT; +INSERT INTO t2 VALUES (4,5); +INSERT INTO t2 VALUES (5,5); +INSERT INTO t3 VALUES (1); +UPDATE t2 SET c=NULL WHERE a=4; +ALTER TABLE t2 ADD UNIQUE (c); +INSERT INTO t2 VALUES (6,6); +UPDATE t2 SET c=c+100 WHERE a=2; +INSERT INTO t3(a) VALUES (2); +DELETE FROM t3 WHERE a=2; +INSERT INTO t3(a) VALUES (2); +DELETE FROM t3 WHERE a=2; +ALTER TABLE t3 CHANGE a c INT NOT NULL; +INSERT INTO t3(c) VALUES (2); +DELETE FROM t3 WHERE c=2; +INSERT INTO t3 SELECT a+200 FROM t2; +DELETE FROM t3 WHERE c >= 200; +INSERT INTO t3 SELECT a+200 FROM t2; +include/save_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 4 +2 6 +3 3 +4 8 +5 9 +SELECT * FROM t2 ORDER BY a; +a c +1 NULL +2 NULL +3 NULL +4 NULL +5 5 +6 6 +SELECT * FROM t3 ORDER BY c; +c +1 +201 +202 +203 +204 +205 +206 +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 4 +2 6 +3 3 +4 8 +5 9 +SELECT * FROM t2 ORDER BY a; +a c +1 NULL +2 NULL +3 NULL +4 NULL +5 5 +6 6 +SELECT * FROM t3 ORDER BY c; +c +1 +201 +202 +203 +204 +205 +206 +*** Test @@skip_parallel_replication. *** +include/stop_slave.inc +UPDATE t1 SET b=10 WHERE a=3; +SET SESSION skip_parallel_replication=1; +UPDATE t1 SET b=20 WHERE a=3; +UPDATE t1 SET b=30 WHERE a=3; +UPDATE t1 SET b=50 WHERE a=3; +UPDATE t1 SET b=80 WHERE a=3; +UPDATE t1 SET b=130 WHERE a=3; +UPDATE t1 SET b=210 WHERE a=3; +UPDATE t1 SET b=340 WHERE a=3; +UPDATE t1 SET b=550 WHERE a=3; +UPDATE t1 SET b=890 WHERE a=3; +SET SESSION skip_parallel_replication=0; +SELECT * FROM t1 ORDER BY a; +a b +1 4 +2 6 +3 890 +4 8 +5 9 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 4 +2 6 +3 890 +4 8 +5 9 +status +Ok, no retry +*** Test that we do not replicate in parallel transactions that had row lock waits on the master *** +include/stop_slave.inc +BEGIN; +UPDATE t1 SET b=b+1 WHERE a=3; +SET debug_sync='thd_report_wait_for SIGNAL waiting1'; +UPDATE t1 SET b=1001 WHERE a=3; +SET debug_sync='now WAIT_FOR waiting1'; +BEGIN; +UPDATE t1 SET b=1002 WHERE a=5; +SET debug_sync='thd_report_wait_for SIGNAL waiting2'; +UPDATE t1 SET b=102 WHERE a=3; +SET debug_sync='now WAIT_FOR waiting2'; +UPDATE t1 SET b=1000 WHERE a=1; +SET debug_sync='thd_report_wait_for SIGNAL waiting3'; +UPDATE t1 SET b=1003 WHERE a=5; +SET debug_sync='now WAIT_FOR waiting3'; +SET debug_sync='thd_report_wait_for SIGNAL waiting4'; +UPDATE t1 SET b=1004 WHERE a=3; +SET debug_sync='now WAIT_FOR waiting4'; +SET debug_sync='thd_report_wait_for SIGNAL waiting5'; +UPDATE t1 SET b=1005 WHERE a=5; +SET debug_sync='now WAIT_FOR waiting5'; +SET debug_sync='thd_report_wait_for SIGNAL waiting6'; +UPDATE t1 SET b=1006 WHERE a=1; +SET debug_sync='now WAIT_FOR waiting6'; +SET debug_sync='thd_report_wait_for SIGNAL waiting7'; +UPDATE t1 SET b=1007 WHERE a=5; +SET debug_sync='now WAIT_FOR waiting7'; +SET debug_sync='thd_report_wait_for SIGNAL waiting8'; +UPDATE t1 SET b=1008 WHERE a=3; +SET debug_sync='now WAIT_FOR waiting8'; +COMMIT; +COMMIT; +SET debug_sync='RESET'; +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT IF(@master_value=@slave_value, "Slave data matches master", CONCAT("ERROR: Slave had different data '", @slave_value, "' than master's '", @master_value, "'!")) as check_result; +check_result +Slave data matches master +status +Ok, no retry +*** Test that we replicate correctly when using READ COMMITTED and binlog_format=MIXED on the slave *** +include/stop_slave.inc +SET @old_format= @@GLOBAL.binlog_format; +SET GLOBAL binlog_format= MIXED; +SET @old_isolation= @@GLOBAL.tx_isolation; +SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +DROP TABLE t1, t2; +CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=TokuDB; +CREATE TABLE t2 (a int PRIMARY KEY, b INT) ENGINE=TokuDB; +INSERT INTO t1 VALUES (1,0), (2,0), (3,0); +INSERT INTO t2 VALUES (1,0), (2,0); +INSERT INTO t1 SELECT 4, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 4, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 5, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 5, COUNT(*) FROM t1; +INSERT INTO t2 SELECT 6, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 6, COUNT(*) FROM t2; +INSERT INTO t1 SELECT 7, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 7, COUNT(*) FROM t1; +INSERT INTO t2 SELECT 8, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 8, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 9, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 9, COUNT(*) FROM t2; +INSERT INTO t1 SELECT 10, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 10, COUNT(*) FROM t1; +SELECT * FROM t1 ORDER BY a; +a b +1 0 +2 0 +3 0 +4 2 +5 3 +6 5 +7 5 +8 7 +9 8 +10 8 +SELECT * FROM t2 ORDER BY a; +a b +1 0 +2 0 +4 4 +5 5 +6 5 +7 7 +8 7 +9 8 +10 10 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 0 +2 0 +3 0 +4 2 +5 3 +6 5 +7 5 +8 7 +9 8 +10 8 +SELECT * FROM t2 ORDER BY a; +a b +1 0 +2 0 +4 4 +5 5 +6 5 +7 7 +8 7 +9 8 +10 10 +include/stop_slave.inc +SET GLOBAL binlog_format= @old_format; +SET GLOBAL tx_isolation= @old_isolation; +include/start_slave.inc +*** MDEV-7888: ANALYZE TABLE does wakeup_subsequent_commits(), causing wrong binlog order and parallel replication hang *** +DROP TABLE t1, t2, t3; +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=TokuDB; +CREATE TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=TokuDB; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1,1), (2,1), (3,1), (4,1), (5,1); +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_analyze_table_sleep'; +ALTER TABLE t2 COMMENT "123abc"; +ANALYZE TABLE t2; +Table Op Msg_type Msg_text +test.t2 analyze status OK +INSERT INTO t1 VALUES (1,2); +INSERT INTO t1 VALUES (2,2); +INSERT INTO t1 VALUES (3,2); +INSERT INTO t1 VALUES (4,2); +INSERT INTO t3 VALUES (1,3); +ALTER TABLE t2 COMMENT "hello, world"; +BEGIN; +INSERT INTO t1 VALUES (5,4); +INSERT INTO t1 VALUES (6,4); +INSERT INTO t1 VALUES (7,4); +INSERT INTO t1 VALUES (8,4); +INSERT INTO t1 VALUES (9,4); +INSERT INTO t1 VALUES (10,4); +INSERT INTO t1 VALUES (11,4); +INSERT INTO t1 VALUES (12,4); +INSERT INTO t1 VALUES (13,4); +INSERT INTO t1 VALUES (14,4); +INSERT INTO t1 VALUES (15,4); +INSERT INTO t1 VALUES (16,4); +INSERT INTO t1 VALUES (17,4); +INSERT INTO t1 VALUES (18,4); +INSERT INTO t1 VALUES (19,4); +INSERT INTO t1 VALUES (20,4); +COMMIT; +INSERT INTO t1 VALUES (21,5); +INSERT INTO t1 VALUES (22,5); +SELECT * FROM t1 ORDER BY a; +a b +1 2 +2 2 +3 2 +4 2 +5 4 +6 4 +7 4 +8 4 +9 4 +10 4 +11 4 +12 4 +13 4 +14 4 +15 4 +16 4 +17 4 +18 4 +19 4 +20 4 +21 5 +22 5 +SELECT * FROM t2 ORDER BY a; +a b +1 1 +2 1 +3 1 +4 1 +5 1 +SELECT * FROM t3 ORDER BY a; +a b +1 3 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 2 +2 2 +3 2 +4 2 +5 4 +6 4 +7 4 +8 4 +9 4 +10 4 +11 4 +12 4 +13 4 +14 4 +15 4 +16 4 +17 4 +18 4 +19 4 +20 4 +21 5 +22 5 +SELECT * FROM t2 ORDER BY a; +a b +1 1 +2 1 +3 1 +4 1 +5 1 +SELECT * FROM t3 ORDER BY a; +a b +1 3 +include/stop_slave.inc +SET GLOBAL debug_dbug= @old_debug; +include/start_slave.inc +*** MDEV-7929: record_gtid() for non-transactional event group calls wakeup_subsequent_commits() too early, causing slave hang. *** +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_record_gtid_serverid_100_sleep'; +ALTER TABLE t3 COMMENT "DDL statement 1"; +INSERT INTO t1 VALUES (30,0); +INSERT INTO t1 VALUES (31,0); +INSERT INTO t1 VALUES (32,0); +INSERT INTO t1 VALUES (33,0); +INSERT INTO t1 VALUES (34,0); +INSERT INTO t1 VALUES (35,0); +INSERT INTO t1 VALUES (36,0); +SET @old_server_id= @@SESSION.server_id; +SET SESSION server_id= 100; +ANALYZE TABLE t2; +Table Op Msg_type Msg_text +test.t2 analyze status OK +SET SESSION server_id= @old_server_id; +INSERT INTO t1 VALUES (37,0); +ALTER TABLE t3 COMMENT "DDL statement 2"; +INSERT INTO t1 VALUES (38,0); +INSERT INTO t1 VALUES (39,0); +ALTER TABLE t3 COMMENT "DDL statement 3"; +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; +a b +30 0 +31 0 +32 0 +33 0 +34 0 +35 0 +36 0 +37 0 +38 0 +39 0 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; +a b +30 0 +31 0 +32 0 +33 0 +34 0 +35 0 +36 0 +37 0 +38 0 +39 0 +include/stop_slave.inc +SET GLOBAL debug_dbug= @old_debug; +include/start_slave.inc +*** MDEV-8113: ALTER TABLE causes slave hang in optimistic parallel replication *** +include/stop_slave.inc +ALTER TABLE t2 ADD c INT; +INSERT INTO t2 (a,b) VALUES (50, 0); +INSERT INTO t2 (a,b) VALUES (51, 1); +INSERT INTO t2 (a,b) VALUES (52, 2); +INSERT INTO t2 (a,b) VALUES (53, 3); +INSERT INTO t2 (a,b) VALUES (54, 4); +INSERT INTO t2 (a,b) VALUES (55, 5); +INSERT INTO t2 (a,b) VALUES (56, 6); +INSERT INTO t2 (a,b) VALUES (57, 7); +INSERT INTO t2 (a,b) VALUES (58, 8); +INSERT INTO t2 (a,b) VALUES (59, 9); +ALTER TABLE t2 DROP COLUMN c; +SELECT * FROM t2 WHERE a >= 50 ORDER BY a; +a b +50 0 +51 1 +52 2 +53 3 +54 4 +55 5 +56 6 +57 7 +58 8 +59 9 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 50 ORDER BY a; +a b +50 0 +51 1 +52 2 +53 3 +54 4 +55 5 +56 6 +57 7 +58 8 +59 9 +include/stop_slave.inc +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +DROP TABLE t1, t2, t3; +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_parallel_optimistic.test b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_parallel_optimistic.test new file mode 100644 index 0000000000000..c7c7de9d90233 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_parallel_optimistic.test @@ -0,0 +1,478 @@ +--source include/have_tokudb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc + +--connection master +ALTER TABLE mysql.gtid_slave_pos ENGINE=TokuDB; +CREATE TABLE t1 (a int PRIMARY KEY, b INT, UNIQUE KEY (b)) ENGINE=TokuDB; +--save_master_pos + +--connection slave +--sync_with_master +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +CHANGE MASTER TO master_use_gtid=slave_pos; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; + + +--connection master + +INSERT INTO t1 VALUES(1,1); +BEGIN; +INSERT INTO t1 VALUES(2,2); +INSERT INTO t1 VALUES(3,3); +COMMIT; + +# Do a bunch of INSERT/DELETE on the same rows, bound to conflict. +# We will get a lot of rollbacks, probably, but they should be handled without +# any visible errors. + +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,2); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,6); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,4); +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,5); + +DELETE FROM t1 WHERE a=3; +INSERT INTO t1 VALUES(3,3); +DELETE FROM t1 WHERE a=1; +INSERT INTO t1 VALUES(1,4); +DELETE FROM t1 WHERE a=3; +INSERT INTO t1 VALUES(3,3); + +DELETE FROM t1 WHERE a=2; +INSERT INTO t1 VALUES (2,6); +--source include/save_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +#SHOW STATUS LIKE 'Slave_retried_transactions'; + + +--echo *** Test a bunch of non-transactional/DDL event groups. *** + +--connection slave +--source include/stop_slave.inc + +--connection master + +INSERT INTO t1 VALUES (4,8); +INSERT INTO t1 VALUES (5,9); +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=TokuDB; +INSERT INTO t2 VALUES (1); +CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=MyISAM; +ALTER TABLE t2 ADD b INT; +INSERT INTO t2 VALUES (2,2); +ALTER TABLE t2 DROP b; +INSERT INTO t2 VALUES (3); +ALTER TABLE t2 ADD c INT; +INSERT INTO t2 VALUES (4,5); +INSERT INTO t2 VALUES (5,5); +INSERT INTO t3 VALUES (1); +UPDATE t2 SET c=NULL WHERE a=4; +ALTER TABLE t2 ADD UNIQUE (c); +INSERT INTO t2 VALUES (6,6); +UPDATE t2 SET c=c+100 WHERE a=2; +INSERT INTO t3(a) VALUES (2); +DELETE FROM t3 WHERE a=2; +INSERT INTO t3(a) VALUES (2); +DELETE FROM t3 WHERE a=2; +ALTER TABLE t3 CHANGE a c INT NOT NULL; +INSERT INTO t3(c) VALUES (2); +DELETE FROM t3 WHERE c=2; +INSERT INTO t3 SELECT a+200 FROM t2; +DELETE FROM t3 WHERE c >= 200; +INSERT INTO t3 SELECT a+200 FROM t2; +--source include/save_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY c; + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY c; +#SHOW STATUS LIKE 'Slave_retried_transactions'; + + +--echo *** Test @@skip_parallel_replication. *** + +--connection slave +--source include/stop_slave.inc +--let $retry1= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1) + +--connection master +# We do a bunch of conflicting transactions on the master with +# skip_parallel_replication set to true, and check that we do not +# get any retries on the slave. + +UPDATE t1 SET b=10 WHERE a=3; +SET SESSION skip_parallel_replication=1; +UPDATE t1 SET b=20 WHERE a=3; +UPDATE t1 SET b=30 WHERE a=3; +UPDATE t1 SET b=50 WHERE a=3; +UPDATE t1 SET b=80 WHERE a=3; +UPDATE t1 SET b=130 WHERE a=3; +UPDATE t1 SET b=210 WHERE a=3; +UPDATE t1 SET b=340 WHERE a=3; +UPDATE t1 SET b=550 WHERE a=3; +UPDATE t1 SET b=890 WHERE a=3; +SET SESSION skip_parallel_replication=0; +SELECT * FROM t1 ORDER BY a; +--source include/save_master_gtid.inc + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +--let $retry2= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1) +--disable_query_log +eval SELECT IF($retry1=$retry2, "Ok, no retry", + CONCAT("ERROR: ", $retry2-$retry1, " retries during replication (was ", + $retry1, " now ", $retry2, ")")) AS status; +--enable_query_log + + +--echo *** Test that we do not replicate in parallel transactions that had row lock waits on the master *** + +--connection slave +--source include/stop_slave.inc +--let $retry1= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1) + +--connection master +# Setup a bunch of transactions that all needed to wait. +--connect (m1,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m2,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m3,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m4,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m5,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m6,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m7,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +--connect (m8,127.0.0.1,root,,test,$SERVER_MYPORT_1,) + +--connection default +BEGIN; UPDATE t1 SET b=b+1 WHERE a=3; + +--connection m1 +SET debug_sync='thd_report_wait_for SIGNAL waiting1'; +send UPDATE t1 SET b=1001 WHERE a=3; +--connection default +SET debug_sync='now WAIT_FOR waiting1'; + +--connection m2 +BEGIN; +UPDATE t1 SET b=1002 WHERE a=5; +SET debug_sync='thd_report_wait_for SIGNAL waiting2'; +send UPDATE t1 SET b=102 WHERE a=3; +--connection default +SET debug_sync='now WAIT_FOR waiting2'; + +UPDATE t1 SET b=1000 WHERE a=1; +--connection m3 +SET debug_sync='thd_report_wait_for SIGNAL waiting3'; +send UPDATE t1 SET b=1003 WHERE a=5; +--connection default +SET debug_sync='now WAIT_FOR waiting3'; + +--connection m4 +SET debug_sync='thd_report_wait_for SIGNAL waiting4'; +send UPDATE t1 SET b=1004 WHERE a=3; +--connection default +SET debug_sync='now WAIT_FOR waiting4'; + +--connection m5 +SET debug_sync='thd_report_wait_for SIGNAL waiting5'; +send UPDATE t1 SET b=1005 WHERE a=5; +--connection default +SET debug_sync='now WAIT_FOR waiting5'; + +--connection m6 +SET debug_sync='thd_report_wait_for SIGNAL waiting6'; +send UPDATE t1 SET b=1006 WHERE a=1; +--connection default +SET debug_sync='now WAIT_FOR waiting6'; + +--connection m7 +SET debug_sync='thd_report_wait_for SIGNAL waiting7'; +send UPDATE t1 SET b=1007 WHERE a=5; +--connection default +SET debug_sync='now WAIT_FOR waiting7'; + +--connection m8 +SET debug_sync='thd_report_wait_for SIGNAL waiting8'; +send UPDATE t1 SET b=1008 WHERE a=3; +--connection default +SET debug_sync='now WAIT_FOR waiting8'; + +--connection default +COMMIT; +--connection m1 +REAP; +--connection m2 +REAP; +COMMIT; +--connection m3 +REAP; +--connection m4 +REAP; +--connection m5 +REAP; +--connection m6 +REAP; +--connection m7 +REAP; +--connection m8 +REAP; +--connection default +SET debug_sync='RESET'; + +--source include/save_master_gtid.inc +# It is not deterministic in which order the parallel conflicting +# updates will run. Eg. for key a=5, we could get 1003, 1005, or +# 1007. As long as we get the same on the slave, it is ok. +--let $master_value= `SELECT GROUP_CONCAT(CONCAT("(", a, ",", b, ")") ORDER BY a, b SEPARATOR "/") FROM t1` + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +--let $slave_value= `SELECT GROUP_CONCAT(CONCAT("(", a, ",", b, ")") ORDER BY a, b SEPARATOR "/") FROM t1` +--disable_query_log +eval SET @master_value="$master_value"; +eval SET @slave_value="$slave_value"; +--enable_query_log +SELECT IF(@master_value=@slave_value, "Slave data matches master", CONCAT("ERROR: Slave had different data '", @slave_value, "' than master's '", @master_value, "'!")) as check_result; +--let $retry2= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1) +--disable_query_log +eval SELECT IF($retry1=$retry2, "Ok, no retry", + CONCAT("ERROR: ", $retry2-$retry1, " retries during replication (was ", + $retry1, " now ", $retry2, ")")) AS status; +--enable_query_log + + +--echo *** Test that we replicate correctly when using READ COMMITTED and binlog_format=MIXED on the slave *** + +--connection slave +--source include/stop_slave.inc +SET @old_format= @@GLOBAL.binlog_format; +# Use MIXED format; we cannot binlog ROW events on slave in STATEMENT format. +SET GLOBAL binlog_format= MIXED; +SET @old_isolation= @@GLOBAL.tx_isolation; +SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED; +# Reset the worker threads to make the new settings take effect. +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; + +--connection master +DROP TABLE t1, t2; +CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=TokuDB; +CREATE TABLE t2 (a int PRIMARY KEY, b INT) ENGINE=TokuDB; +INSERT INTO t1 VALUES (1,0), (2,0), (3,0); +INSERT INTO t2 VALUES (1,0), (2,0); +INSERT INTO t1 SELECT 4, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 4, COUNT(*) FROM t1; + +INSERT INTO t1 SELECT 5, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 5, COUNT(*) FROM t1; + +INSERT INTO t2 SELECT 6, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 6, COUNT(*) FROM t2; + +INSERT INTO t1 SELECT 7, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 7, COUNT(*) FROM t1; + +INSERT INTO t2 SELECT 8, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 8, COUNT(*) FROM t2; + +INSERT INTO t2 SELECT 9, COUNT(*) FROM t1; +INSERT INTO t1 SELECT 9, COUNT(*) FROM t2; + +INSERT INTO t1 SELECT 10, COUNT(*) FROM t2; +INSERT INTO t2 SELECT 10, COUNT(*) FROM t1; + +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL binlog_format= @old_format; +SET GLOBAL tx_isolation= @old_isolation; +--source include/start_slave.inc + + +--echo *** MDEV-7888: ANALYZE TABLE does wakeup_subsequent_commits(), causing wrong binlog order and parallel replication hang *** + +--connection master +DROP TABLE t1, t2, t3; +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=TokuDB; +CREATE TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=TokuDB; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1,1), (2,1), (3,1), (4,1), (5,1); +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_analyze_table_sleep'; + +--connection master +# The bug was that ANALYZE TABLE would call +# wakeup_subsequent_commits() too early, allowing the following +# transaction in the same group to run ahead and binlog and free the +# GCO. Then we get wrong binlog order and later access freed GCO, +# which causes lost wakeup of following GCO and thus replication hang. +# We injected a small sleep in ANALYZE to make the race easier to hit (this +# can only cause false negatives in versions with the bug, not false positives, +# so sleep is ok here. And it's in general not possible to trigger reliably +# the race with debug_sync, since the bugfix makes the race impossible). + +ALTER TABLE t2 COMMENT "123abc"; +ANALYZE TABLE t2; +INSERT INTO t1 VALUES (1,2); +INSERT INTO t1 VALUES (2,2); +INSERT INTO t1 VALUES (3,2); +INSERT INTO t1 VALUES (4,2); +INSERT INTO t3 VALUES (1,3); +ALTER TABLE t2 COMMENT "hello, world"; +BEGIN; +INSERT INTO t1 VALUES (5,4); +INSERT INTO t1 VALUES (6,4); +INSERT INTO t1 VALUES (7,4); +INSERT INTO t1 VALUES (8,4); +INSERT INTO t1 VALUES (9,4); +INSERT INTO t1 VALUES (10,4); +INSERT INTO t1 VALUES (11,4); +INSERT INTO t1 VALUES (12,4); +INSERT INTO t1 VALUES (13,4); +INSERT INTO t1 VALUES (14,4); +INSERT INTO t1 VALUES (15,4); +INSERT INTO t1 VALUES (16,4); +INSERT INTO t1 VALUES (17,4); +INSERT INTO t1 VALUES (18,4); +INSERT INTO t1 VALUES (19,4); +INSERT INTO t1 VALUES (20,4); +COMMIT; +INSERT INTO t1 VALUES (21,5); +INSERT INTO t1 VALUES (22,5); + +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY a; +--source include/save_master_gtid.inc + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL debug_dbug= @old_debug; +--source include/start_slave.inc + +--echo *** MDEV-7929: record_gtid() for non-transactional event group calls wakeup_subsequent_commits() too early, causing slave hang. *** + +--connection slave +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +# The bug was that record_gtid(), when there is no existing transaction from +# a DML event being replicated, would commit its own transaction. This wrongly +# caused wakeup_subsequent_commits(), with similar consequences as MDEV-7888 +# above. We simulate this condition with a small sleep in record_gtid() for +# a specific ANALYZE that we binlog with server id 100. +SET GLOBAL debug_dbug= '+d,inject_record_gtid_serverid_100_sleep'; + +--connection master + +ALTER TABLE t3 COMMENT "DDL statement 1"; +INSERT INTO t1 VALUES (30,0); +INSERT INTO t1 VALUES (31,0); +INSERT INTO t1 VALUES (32,0); +INSERT INTO t1 VALUES (33,0); +INSERT INTO t1 VALUES (34,0); +INSERT INTO t1 VALUES (35,0); +INSERT INTO t1 VALUES (36,0); +SET @old_server_id= @@SESSION.server_id; +SET SESSION server_id= 100; +ANALYZE TABLE t2; +SET SESSION server_id= @old_server_id; +INSERT INTO t1 VALUES (37,0); +ALTER TABLE t3 COMMENT "DDL statement 2"; +INSERT INTO t1 VALUES (38,0); +INSERT INTO t1 VALUES (39,0); +ALTER TABLE t3 COMMENT "DDL statement 3"; + +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; + +--source include/save_master_gtid.inc + + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; + + +--source include/stop_slave.inc +SET GLOBAL debug_dbug= @old_debug; +--source include/start_slave.inc + + +--echo *** MDEV-8113: ALTER TABLE causes slave hang in optimistic parallel replication *** + +--connection slave +--source include/stop_slave.inc + +--connection master +ALTER TABLE t2 ADD c INT; +INSERT INTO t2 (a,b) VALUES (50, 0); +INSERT INTO t2 (a,b) VALUES (51, 1); +INSERT INTO t2 (a,b) VALUES (52, 2); +INSERT INTO t2 (a,b) VALUES (53, 3); +INSERT INTO t2 (a,b) VALUES (54, 4); +INSERT INTO t2 (a,b) VALUES (55, 5); +INSERT INTO t2 (a,b) VALUES (56, 6); +INSERT INTO t2 (a,b) VALUES (57, 7); +INSERT INTO t2 (a,b) VALUES (58, 8); +INSERT INTO t2 (a,b) VALUES (59, 9); +ALTER TABLE t2 DROP COLUMN c; +SELECT * FROM t2 WHERE a >= 50 ORDER BY a; +--source include/save_master_gtid.inc + +--connection slave +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 50 ORDER BY a; + + +# Clean up. + +--connection slave +--source include/stop_slave.inc +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc + +--connection master +DROP TABLE t1, t2, t3; + +--source include/rpl_end.inc From a68d1352b60bfc3a424fd290a4f5a1beae1bb71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 25 Nov 2016 06:28:02 +0200 Subject: [PATCH 210/295] MDEV-11349 (2/2) Fix some bogus-looking Valgrind warnings buf_block_init(): Initialize buf_page_t::flush_type. For some reason, Valgrind 3.12.0 would seem to flag some bits in adjacent bitfields as uninitialized, even though only the two bits of flush_type were left uninitialized. Initialize the field to get rid of many warnings. buf_page_init_low(): Initialize buf_page_t::old. For some reason, Valgrind 3.12.0 would seem to flag all 32 bits uninitialized when buf_page_init_for_read() invokes buf_LRU_add_block(bpage, TRUE). This would trigger bogus warnings for buf_page_t::freed_page_clock being uninitialized. (The V-bits would later claim that only "old" is initialized in the 32-bit word.) Perhaps recent compilers (GCC 6.2.1 and clang 4.0.0) generate more optimized x86_64 code for bitfield operations, confusing Valgrind? mach_write_to_1(), mach_write_to_2(), mach_write_to_3(): Rewrite the assertions that ensure that the most significant bits are zero. Apparently, clang 4.0.0 would optimize expressions of the form ((n | 0xFF) <= 0x100) to (n <= 0x100). The redundant 0xFF was added in the first place in order to suppress a Valgrind warning. (Valgrind would warn about comparing uninitialized values even in the case when the uninitialized bits do not affect the result of the comparison.) --- storage/innobase/buf/buf0buf.cc | 2 ++ storage/innobase/include/mach0data.ic | 6 +++--- storage/xtradb/buf/buf0buf.cc | 1 + storage/xtradb/include/mach0data.ic | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 7eca60417a1fd..a05f3715cb991 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1124,6 +1124,7 @@ buf_block_init( block->frame = frame; block->page.buf_pool_index = buf_pool_index(buf_pool); + block->page.flush_type = BUF_FLUSH_LRU; block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; @@ -3784,6 +3785,7 @@ buf_page_init_low( bpage->flush_type = BUF_FLUSH_LRU; bpage->io_fix = BUF_IO_NONE; bpage->buf_fix_count = 0; + bpage->old = 0; bpage->freed_page_clock = 0; bpage->access_time = 0; bpage->newest_modification = 0; diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index fe55adaf0021c..881b2b6055f82 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -38,7 +38,7 @@ mach_write_to_1( ulint n) /*!< in: ulint integer to be stored, >= 0, < 256 */ { ut_ad(b); - ut_ad((n | 0xFFUL) <= 0xFFUL); + ut_ad((n & ~0xFFUL) == 0); b[0] = (byte) n; } @@ -68,7 +68,7 @@ mach_write_to_2( ulint n) /*!< in: ulint integer to be stored */ { ut_ad(b); - ut_ad((n | 0xFFFFUL) <= 0xFFFFUL); + ut_ad((n & ~0xFFFFUL) == 0); b[0] = (byte)(n >> 8); b[1] = (byte)(n); @@ -131,7 +131,7 @@ mach_write_to_3( ulint n) /*!< in: ulint integer to be stored */ { ut_ad(b); - ut_ad((n | 0xFFFFFFUL) <= 0xFFFFFFUL); + ut_ad((n & ~0xFFFFFFUL) == 0); b[0] = (byte)(n >> 16); b[1] = (byte)(n >> 8); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 50a13da262c1f..ebfc813f13daa 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -1189,6 +1189,7 @@ buf_block_init( block->frame = frame; block->page.buf_pool_index = buf_pool_index(buf_pool); + block->page.flush_type = BUF_FLUSH_LRU; block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; diff --git a/storage/xtradb/include/mach0data.ic b/storage/xtradb/include/mach0data.ic index 27b9f62b55234..bf2c735b0da34 100644 --- a/storage/xtradb/include/mach0data.ic +++ b/storage/xtradb/include/mach0data.ic @@ -38,7 +38,7 @@ mach_write_to_1( ulint n) /*!< in: ulint integer to be stored, >= 0, < 256 */ { ut_ad(b); - ut_ad((n | 0xFFUL) <= 0xFFUL); + ut_ad((n & ~0xFFUL) == 0); b[0] = (byte) n; } @@ -67,7 +67,7 @@ mach_write_to_2( ulint n) /*!< in: ulint integer to be stored */ { ut_ad(b); - ut_ad((n | 0xFFFFUL) <= 0xFFFFUL); + ut_ad((n & ~0xFFFFUL) == 0); b[0] = (byte)(n >> 8); b[1] = (byte)(n); @@ -115,7 +115,7 @@ mach_write_to_3( ulint n) /*!< in: ulint integer to be stored */ { ut_ad(b); - ut_ad((n | 0xFFFFFFUL) <= 0xFFFFFFUL); + ut_ad((n & ~0xFFFFFFUL) == 0); b[0] = (byte)(n >> 16); b[1] = (byte)(n >> 8); From 099ce1dda1291eb4ef789039db400ee380d75376 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 25 Nov 2016 15:59:47 +0400 Subject: [PATCH 211/295] MDEV-11348 LOAD DATA LOCAL INFILE crashes the server on loading a backslash followed by a multi-byte character The crash happened when if my_error() was called for any reasons during loading (e.g. a bad multi-byte sequence or a bad GEOMETRY value was found). The server sent both error and progress packets, so the client disconnected. The server then crashed on a assert about a wrong packet order in Debug build. The server also tried to read from a closed socket when calling READ_INFO::skip_data_till_eof(). As the crash happened only with "mysql" running in interactive mode, no tests are possible. The problem was not reproducible with "mysqltest" or "mysql" in batch mode. --- sql/sql_load.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_load.cc b/sql/sql_load.cc index a21fc692eccb0..af4b25185d079 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -521,7 +521,8 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, *enclosed, skip_lines, ignore); thd_proc_info(thd, "End bulk insert"); - thd_progress_next_stage(thd); + if (!error) + thd_progress_next_stage(thd); if (thd->locked_tables_mode <= LTM_LOCK_TABLES && table->file->ha_end_bulk_insert() && !error) { From 9976223c0063944a7fb598b8e22512a35c841f67 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 28 Nov 2016 17:28:37 +0400 Subject: [PATCH 212/295] MDEV-11171 Assertion `m_cpp_buf <= ptr && ptr <= m_cpp_buf + m_buf_length' failed in Lex_input_stream::body_utf8_append(const char*, const char*) --- mysql-test/r/parser.result | 9 +++++++++ mysql-test/t/parser.test | 10 ++++++++++ sql/sql_lex.cc | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/parser.result b/mysql-test/r/parser.result index 25143f97d9ae6..9a14c0e324b41 100644 --- a/mysql-test/r/parser.result +++ b/mysql-test/r/parser.result @@ -663,3 +663,12 @@ select 1<!0, 2 > ! 0; select 0<=!0, 0 <= !0; select 1<yyGet() == 'N') + if (!lip->eof() && lip->yyGet() == 'N') { // Allow \N as shortcut for NULL yylval->lex_str.str=(char*) "\\N"; yylval->lex_str.length=2; From dd0ff30278cd7b24776ccf36a9c0d9171a569750 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 29 Nov 2016 06:51:12 +0400 Subject: [PATCH 213/295] MDEV-11343 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partially backporting MDEV-9874 from 10.2 to 10.0 READ_INFO::read_field() raised the ER_INVALID_CHARACTER_STRING error when reading an escape character followed by a multi-byte character. Raising wellformedness errors in READ_INFO::read_field() was wrong, because the main goal of READ_INFO::read_field() is to *unescape* the data which was presumably escaped using mysql_real_escape_string(), using the same character set with the one specified in "LOAD DATA INFILE ... CHARACTER SET ..." (or assumed by default). During LOAD DATA, multi-byte characters are not always scanned as a single entity! In case of escaped data, parts of a multi-byte character can be scanned on different loop iterations. So the old code erroneously tested welformedness in the middle of a multi-byte character. Moreover, the data after unescaping can go into a BLOB field, not a text field. Wellformedness tests are meaningless in this case. Ater this patch, wellformedness is only checked later, during Field::store(str,length,cs) time. The loop that scans bytes only makes sure to revert the changes made by mysql_real_escape_string(). Note, in some cases users can supply data which did not really go through mysql_real_escape_string() and was escaped by some other means, or was not escaped at all. The file reported in this MDEV contains the string "\ä", which is an example of such improperly escaped data, as - either there should be two backslashes: "\\ä" - or there should be no backslashes at all: "ä" mysql_real_escape_string() could not generate "\ä". --- include/m_ctype.h | 3 + mysql-test/r/ctype_utf8mb4.result | 23 +++ mysql-test/r/loaddata.result | 3 +- mysql-test/std_data/loaddata/mdev-11343.txt | 12 ++ mysql-test/t/ctype_utf8mb4.test | 11 ++ mysql-test/t/loaddata.test | 1 - sql/sql_load.cc | 189 +++++++++++++++----- sql/sql_string.h | 1 + 8 files changed, 196 insertions(+), 47 deletions(-) create mode 100644 mysql-test/std_data/loaddata/mdev-11343.txt diff --git a/include/m_ctype.h b/include/m_ctype.h index 5994816cbfc1e..8d9838fdde2ee 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -180,6 +180,9 @@ extern MY_UNI_CTYPE my_uni_ctype[256]; /* A helper macros for "need at least n bytes" */ #define MY_CS_TOOSMALLN(n) (-100-(n)) +#define MY_CS_IS_TOOSMALL(rc) ((rc) >= MY_CS_TOOSMALL6 && (rc) <= MY_CS_TOOSMALL) + + #define MY_SEQ_INTTAIL 1 #define MY_SEQ_SPACES 2 diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index 38814bc36d658..50382b5d5ca39 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -3356,5 +3356,28 @@ DFFFFFDFFFFF9CFFFF9DFFFF9EFFFF # End of 5.6 tests # # +# Start of 10.0 tests +# +# +# MDEV-11343 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +# +CREATE TABLE t1 (a TEXT CHARACTER SET utf8mb4); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11343.txt' INTO TABLE t1 CHARACTER SET utf8mb4; +SELECT HEX(a) FROM t1; +HEX(a) +C3A4 +C3A478 +78C3A4 +78C3A478 +EA99A0 +EA99A078 +78EA99A0 +78EA99A078 +F09F988E +F09F988E78 +78F09F988E +78F09F988E78 +DROP TABLE t1; +# # End of tests # diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index 2f2a3579eecf2..12462305dc8b2 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -552,7 +552,8 @@ CREATE DATABASE d2 CHARSET utf8; USE d2; CREATE TABLE t1 (val TEXT); LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -ERROR HY000: Invalid utf8 character string: 'Ã"RT @niouzechun: \9058\221A' +Warnings: +Warning 1366 Incorrect string value: '\xF5\x80\x81\xAE\xE7\xB9...' for column 'val' at row 1 DROP TABLE d1.t1, d2.t1; DROP DATABASE d1; DROP DATABASE d2; diff --git a/mysql-test/std_data/loaddata/mdev-11343.txt b/mysql-test/std_data/loaddata/mdev-11343.txt new file mode 100644 index 0000000000000..dded1215ffa6b --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11343.txt @@ -0,0 +1,12 @@ +\ä +\äx +x\ä +x\äx +\ê™  +\ê™ x +x\ê™  +x\ê™ x +\😎 +\😎x +x\😎 +x\😎x diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test index 6b876cc5ebaaf..cf1c103137ef6 100644 --- a/mysql-test/t/ctype_utf8mb4.test +++ b/mysql-test/t/ctype_utf8mb4.test @@ -1864,6 +1864,17 @@ set @@collation_connection=utf8mb4_bin; --echo # End of 5.6 tests --echo # +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-11343 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +--echo # +CREATE TABLE t1 (a TEXT CHARACTER SET utf8mb4); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11343.txt' INTO TABLE t1 CHARACTER SET utf8mb4; +SELECT HEX(a) FROM t1; +DROP TABLE t1; --echo # --echo # End of tests diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 7d0f3852b6690..9f2aafc8efd56 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -675,7 +675,6 @@ SELECT HEX(val) FROM t1; CREATE DATABASE d2 CHARSET utf8; USE d2; CREATE TABLE t1 (val TEXT); ---error ER_INVALID_CHARACTER_STRING LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; DROP TABLE d1.t1, d2.t1; diff --git a/sql/sql_load.cc b/sql/sql_load.cc index af4b25185d079..51a284964e120 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -79,6 +79,81 @@ class READ_INFO { NET *io_net; int level; /* for load xml */ + +#if MYSQL_VERSION_ID >= 100200 +#error This 10.0 and 10.1 specific fix should be removed in 10.2. +#error Fix read_mbtail() to use my_charlen() instead of my_charlen_tmp() +#else + int my_charlen_tmp(CHARSET_INFO *cs, const char *str, const char *end) + { + my_wc_t wc; + return cs->cset->mb_wc(cs, &wc, (const uchar *) str, (const uchar *) end); + } + + /** + Read a tail of a multi-byte character. + The first byte of the character is assumed to be already + read from the file and appended to "str". + + @returns true - if EOF happened unexpectedly + @returns false - no EOF happened: found a good multi-byte character, + or a bad byte sequence + + Note: + The return value depends only on EOF: + - read_mbtail() returns "false" is a good character was read, but also + - read_mbtail() returns "false" if an incomplete byte sequence was found + and no EOF happened. + + For example, suppose we have an ujis file with bytes 0x8FA10A, where: + - 0x8FA1 is an incomplete prefix of a 3-byte character + (it should be [8F][A1-FE][A1-FE] to make a full 3-byte character) + - 0x0A is a line demiliter + This file has some broken data, the trailing [A1-FE] is missing. + + In this example it works as follows: + - 0x8F is read from the file and put into "data" before the call + for read_mbtail() + - 0xA1 is read from the file and put into "data" by read_mbtail() + - 0x0A is kept in the read queue, so the next read iteration after + the current read_mbtail() call will normally find it and recognize as + a line delimiter + - the current call for read_mbtail() returns "false", + because no EOF happened + */ + bool read_mbtail(String *str) + { + int chlen; + if ((chlen= my_charlen_tmp(read_charset, str->end() - 1, str->end())) == 1) + return false; // Single byte character found + for (uint32 length0= str->length() - 1 ; MY_CS_IS_TOOSMALL(chlen); ) + { + int chr= GET; + if (chr == my_b_EOF) + { + DBUG_PRINT("info", ("read_mbtail: chlen=%d; unexpected EOF", chlen)); + return true; // EOF + } + str->append(chr); + chlen= my_charlen_tmp(read_charset, str->ptr() + length0, str->end()); + if (chlen == MY_CS_ILSEQ) + { + /** + It has been an incomplete (but a valid) sequence so far, + but the last byte turned it into a bad byte sequence. + Unget the very last byte. + */ + str->length(str->length() - 1); + PUSH(chr); + DBUG_PRINT("info", ("read_mbtail: ILSEQ")); + return false; // Bad byte sequence + } + } + DBUG_PRINT("info", ("read_mbtail: chlen=%d", chlen)); + return false; // Good multi-byte character + } +#endif + public: bool error,line_cuted,found_null,enclosed; uchar *row_start, /* Found row starts here */ @@ -1474,6 +1549,54 @@ inline int READ_INFO::terminator(const uchar *ptr,uint length) } +/** + Read a field. + + The data in the loaded file was presumably escaped using + - either select_export::send_data() OUTFILE + - or mysql_real_escape_string() + using the same character set with the one specified in the current + "LOAD DATA INFILE ... CHARACTER SET ..." (or the default LOAD character set). + + Note, non-escaped multi-byte characters are scanned as a single entity. + This is needed to correctly distinguish between: + - 0x5C as an escape character versus + - 0x5C as the second byte in a multi-byte sequence (big5, cp932, gbk, sjis) + + Parts of escaped multi-byte characters are scanned on different loop + iterations. See the comment about 0x5C handling in select_export::send_data() + in sql_class.cc. + + READ_INFO::read_field() does not check wellformedness. + Raising wellformedness errors or warnings in READ_INFO::read_field() + would be wrong, as the data after unescaping can go into a BLOB field, + or into a TEXT/VARCHAR field of a different character set. + The loop below only makes sure to revert escaping made by + select_export::send_data() or mysql_real_escape_string(). + Wellformedness is checked later, during Field::store(str,length,cs) time. + + Note, in some cases users can supply data which did not go through + escaping properly. For example, utf8 "\" + (backslash followed by LATIN SMALL LETTER A WITH DIAERESIS) + is improperly escaped data that could not be generated by + select_export::send_data() / mysql_real_escape_string(): + - either there should be two backslashes: "\\" + - or there should be no backslashes at all: "" + "\" and " are scanned on two different loop iterations and + store "" into the field. + + Note, adding useless escapes before multi-byte characters like in the + example above is safe in case of utf8, but is not safe in case of + character sets that have escape_with_backslash_is_dangerous==TRUE, + such as big5, cp932, gbk, sjis. This can lead to mis-interpretation of the + data. Suppose we have a big5 character "<5C>" followed by <30> (digit 0). + If we add an extra escape before this sequence, then we'll get + <5C><5C><30>. The first loop iteration will turn <5C> into . + The second loop iteration will turn <5C><30> into <30>. + So the program that generates a dump file for further use with LOAD DATA + must make sure to use escapes properly. +*/ + int READ_INFO::read_field() { int chr,found_enclosed_char; @@ -1510,7 +1633,8 @@ int READ_INFO::read_field() for (;;) { - while ( to < end_of_buff) + // Make sure we have enough space for the longest multi-byte character. + while ( to + read_charset->mbmaxlen < end_of_buff) { chr = GET; if (chr == my_b_EOF) @@ -1598,52 +1722,27 @@ int READ_INFO::read_field() } } #ifdef USE_MB - uint ml= my_mbcharlen(read_charset, chr); - if (ml == 0) - { - *to= '\0'; - my_error(ER_INVALID_CHARACTER_STRING, MYF(0), - read_charset->csname, buffer); - error= true; - return 1; - } - - if (ml > 1 && - to + ml <= end_of_buff) - { - uchar* p= to; - *to++ = chr; - - for (uint i= 1; i < ml; i++) - { - chr= GET; - if (chr == my_b_EOF) - { - /* - Need to back up the bytes already ready from illformed - multi-byte char - */ - to-= i; - goto found_eof; - } - *to++ = chr; - } - if (my_ismbchar(read_charset, - (const char *)p, - (const char *)to)) - continue; - for (uint i= 0; i < ml; i++) - PUSH(*--to); - chr= GET; - } - else if (ml > 1) - { - // Buffer is too small, exit while loop, and reallocate. - PUSH(chr); - break; - } #endif *to++ = (uchar) chr; +#if MYSQL_VERSION_ID >= 100200 +#error This 10.0 and 10.1 specific fix should be removed in 10.2 +#else + if (my_mbcharlen(read_charset, (uchar) chr) > 1) + { + /* + A known MBHEAD found. Try to scan the full multi-byte character. + Otherwise, a possible following second byte 0x5C would be + mis-interpreted as an escape on the next iteration. + (Important for big5, gbk, sjis, cp932). + */ + String tmp((char *) to - 1, read_charset->mbmaxlen, read_charset); + tmp.length(1); + bool eof= read_mbtail(&tmp); + to+= tmp.length() - 1; + if (eof) + goto found_eof; + } +#endif } /* ** We come here if buffer is too small. Enlarge it and continue diff --git a/sql/sql_string.h b/sql/sql_string.h index c287f051d9873..557d14a79f88d 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -136,6 +136,7 @@ class String inline bool is_empty() const { return (str_length == 0); } inline void mark_as_const() { Alloced_length= 0;} inline const char *ptr() const { return Ptr; } + inline const char *end() const { return Ptr + str_length; } inline char *c_ptr() { DBUG_ASSERT(!alloced || !Ptr || !Alloced_length || From 9f31949b6460e7bd27daa8cbcaeb0343836e5352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 29 Nov 2016 08:35:51 +0200 Subject: [PATCH 214/295] MDEV-10739: encryption.innodb-page_encryption_compression fails with timeout on valgrind Test moved to big_test and not run on valgrind. Test heavy especially on debug builds. --- .../suite/encryption/t/innodb-page_encryption_compression.test | 3 +++ storage/xtradb/include/fil0fil.ic | 2 ++ 2 files changed, 5 insertions(+) diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test b/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test index 1d59c39d637a9..eb293e9769315 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test @@ -1,6 +1,9 @@ -- source include/have_innodb.inc -- source include/not_embedded.inc -- source include/have_file_key_management_plugin.inc +-- source include/big_test.inc +# Test heavy not tested on valgrind +-- source include/not_valgrind.inc --disable_query_log let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 7ccc69b956161..23614a6567aa5 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -131,6 +131,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || + page_type == FIL_PAGE_TYPE_ZBLOB2 || page_type == FIL_PAGE_TYPE_COMPRESSED))) { uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); @@ -166,6 +167,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || + page_type == FIL_PAGE_TYPE_ZBLOB2 || page_type == FIL_PAGE_TYPE_COMPRESSED); return false; } From dbdef41a59ced64e21cd6c9059ce85496662212b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 29 Nov 2016 08:41:45 +0200 Subject: [PATCH 215/295] MDEV-10686: innodb_zip.innodb_prefix_index_liftedlimit failed with timeout in buildbot Test moved to big_test and not run with valgrind because of timeout. --- .../suite/innodb_zip/t/innodb_prefix_index_liftedlimit.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/suite/innodb_zip/t/innodb_prefix_index_liftedlimit.test b/mysql-test/suite/innodb_zip/t/innodb_prefix_index_liftedlimit.test index 77f55002da5f4..541c0e119bee9 100644 --- a/mysql-test/suite/innodb_zip/t/innodb_prefix_index_liftedlimit.test +++ b/mysql-test/suite/innodb_zip/t/innodb_prefix_index_liftedlimit.test @@ -15,6 +15,8 @@ --source include/have_innodb.inc --source include/have_innodb_16k.inc +--source include/big_test.inc +--source include/not_valgrind.inc # Save innodb variables --disable_query_log From b209bc3eedfabc205af89f9e4c3af86bdfc6277a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 29 Nov 2016 09:01:46 +0200 Subject: [PATCH 216/295] MDEV-10427: innodb.innodb-wl5522-debug-zip fails sporadically in buildbot Test intentionally crashes the server, thus corrupted pages possible. --- mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result | 3 +++ mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index 0e863f5849e1b..3ce55ef0409b4 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,3 +1,6 @@ +call mtr.add_suppression("InnoDB: Page for tablespace .* "); +call mtr.add_suppression("InnoSB: Warning: database page corruption or a failed .*"); +FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@innodb_file_per_table diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index 4b03ac008d29d..37b630ad47b3d 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -17,6 +17,10 @@ # allow test to run only when innodb-page-size=16 --source include/have_innodb_16k.inc +# Test intentionally crashes the server, corrupted pages possible +call mtr.add_suppression("InnoDB: Page for tablespace .* "); +call mtr.add_suppression("InnoSB: Warning: database page corruption or a failed .*"); +FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; From 748d993cca6c63d40236ac7c937630965283242d Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 29 Nov 2016 11:28:15 -0800 Subject: [PATCH 217/295] Fixed bug mdev-11364. The function Item_func_isnull::update_used_tables() must handle the case when the predicate is over not nullable column in a special way. This is actually a bug of MariaDB 5.3/5.5, but it's probably hard to demonstrate that it can cause problems there. --- mysql-test/r/selectivity.result | 21 +++++++++++++++++++++ mysql-test/r/selectivity_innodb.result | 21 +++++++++++++++++++++ mysql-test/t/selectivity.test | 21 +++++++++++++++++++++ sql/item_cmpfunc.h | 18 +++++++++++++++++- sql/sql_select.cc | 15 ++------------- 5 files changed, 82 insertions(+), 14 deletions(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 8fb5cd17c518a..61a77d135e719 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1517,3 +1517,24 @@ Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where 0 drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-11364: IS NULL over not nullable datetime column +# in mergeable derived +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=4; +set HISTOGRAM_SIZE = 255; +CREATE TABLE t1 (t TIME, d DATE NOT NULL); +INSERT INTO t1 VALUES ('10:00:00', '0000-00-00'),('11:00:00','0000-00-00'); +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; +t +10:00:00 +11:00:00 +DROP TABLE t1; +set histogram_size=@save_histogram_size; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 3d15131dbb55e..a026c2e6d929b 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1521,6 +1521,27 @@ Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where 0 drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-11364: IS NULL over not nullable datetime column +# in mergeable derived +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=4; +set HISTOGRAM_SIZE = 255; +CREATE TABLE t1 (t TIME, d DATE NOT NULL); +INSERT INTO t1 VALUES ('10:00:00', '0000-00-00'),('11:00:00','0000-00-00'); +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; +t +10:00:00 +11:00:00 +DROP TABLE t1; +set histogram_size=@save_histogram_size; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; set @tmp_oucs= @@optimizer_use_condition_selectivity; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 8efc5216ba09c..548ef295fb210 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1025,3 +1025,24 @@ drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +--echo # +--echo # Bug mdev-11364: IS NULL over not nullable datetime column +--echo # in mergeable derived +--echo # + +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=4; +set HISTOGRAM_SIZE = 255; + +CREATE TABLE t1 (t TIME, d DATE NOT NULL); +INSERT INTO t1 VALUES ('10:00:00', '0000-00-00'),('11:00:00','0000-00-00'); + +ANALYZE TABLE t1; + +SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; + +DROP TABLE t1; + +set histogram_size=@save_histogram_size; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index dbd96a89a2413..9e83b732dc748 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1387,10 +1387,26 @@ class Item_func_isnull :public Item_bool_func update_used_tables(); } const char *func_name() const { return "isnull"; } + + bool arg_is_datetime_notnull_field() + { + Item **args= arguments(); + if (args[0]->type() == Item::FIELD_ITEM) + { + Field *field=((Item_field*) args[0])->field; + + if (((field->type() == MYSQL_TYPE_DATE) || + (field->type() == MYSQL_TYPE_DATETIME)) && + (field->flags & NOT_NULL_FLAG)) + return true; + } + return false; + } + /* Optimize case of not_null_column IS NULL */ virtual void update_used_tables() { - if (!args[0]->maybe_null) + if (!args[0]->maybe_null && !arg_is_datetime_notnull_field()) { used_tables_cache= 0; /* is always false */ const_item_cache= 1; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 0cfb964307d2a..2db9a2b8482c1 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -14766,20 +14766,9 @@ bool cond_is_datetime_is_null(Item *cond) if (cond->type() == Item::FUNC_ITEM && ((Item_func*) cond)->functype() == Item_func::ISNULL_FUNC) { - Item **args= ((Item_func_isnull*) cond)->arguments(); - if (args[0]->type() == Item::FIELD_ITEM) - { - Field *field=((Item_field*) args[0])->field; - - if (((field->type() == MYSQL_TYPE_DATE) || - (field->type() == MYSQL_TYPE_DATETIME)) && - (field->flags & NOT_NULL_FLAG)) - { - return TRUE; - } - } + return ((Item_func_isnull*) cond)->arg_is_datetime_notnull_field(); } - return FALSE; + return false; } From 525e214111a069574f9aca15944ac0a4ae7e4696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 25 Oct 2016 16:47:36 +0200 Subject: [PATCH 218/295] Remove labs() warning from maria and myisam storage engines --- storage/maria/ma_test2.c | 3 --- storage/myisam/mi_test2.c | 3 --- 2 files changed, 6 deletions(-) diff --git a/storage/maria/ma_test2.c b/storage/maria/ma_test2.c index 709a190c1a73c..c34c45d42fbbb 100644 --- a/storage/maria/ma_test2.c +++ b/storage/maria/ma_test2.c @@ -27,9 +27,6 @@ #define STANDARD_LENGTH 37 #define MARIA_KEYS 6 #define MAX_PARTS 4 -#if !defined(MSDOS) && !defined(labs) -#define labs(a) abs(a) -#endif static void get_options(int argc, char *argv[]); static uint rnd(uint max_value); diff --git a/storage/myisam/mi_test2.c b/storage/myisam/mi_test2.c index be58b3c54d00d..32dabca0ef598 100644 --- a/storage/myisam/mi_test2.c +++ b/storage/myisam/mi_test2.c @@ -26,9 +26,6 @@ #define STANDARD_LENGTH 37 #define MYISAM_KEYS 6 #define MAX_PARTS 4 -#if !defined(labs) -#define labs(a) abs(a) -#endif static void get_options(int argc, char *argv[]); static uint rnd(uint max_value); From e99990c631905cd923257331ef124a2bc9276e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 28 Oct 2016 17:10:05 +0200 Subject: [PATCH 219/295] MDEV-10744: Roles are not fully case sensitive Due to the collation used on the roles_mapping_hash, key comparison would work in a case-insensitive manner. This is incorrect from the roles mapping perspective. Make use of a case-sensitive collation for that hash, the same one used for the acl_roles hash. --- .../roles/role_case_sensitive-10744.result | 58 +++++++++++++++++++ .../roles/role_case_sensitive-10744.test | 54 +++++++++++++++++ sql/sql_acl.cc | 2 +- 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/roles/role_case_sensitive-10744.result create mode 100644 mysql-test/suite/roles/role_case_sensitive-10744.test diff --git a/mysql-test/suite/roles/role_case_sensitive-10744.result b/mysql-test/suite/roles/role_case_sensitive-10744.result new file mode 100644 index 0000000000000..e9b498eff2601 --- /dev/null +++ b/mysql-test/suite/roles/role_case_sensitive-10744.result @@ -0,0 +1,58 @@ +# +# MDEV-10744 Roles are not fully case-sensitive +# +# +# Test creating two case-different roles. +# +create user test_user@'%'; +create role test_ROLE; +create role test_role; +# +# Test if mysql.user has the roles created. +# +select user, host from mysql.user where is_role='y' and user like 'test%'; +user host +test_ROLE +test_role +create database secret_db; +create table secret_db.t1 (secret varchar(100)); +insert into secret_db.t1 values ("Some Secret P4ssw0rd"); +grant select on secret_db.* to test_role; +grant test_role to test_user; +show grants for test_user; +Grants for test_user@% +GRANT test_role TO 'test_user'@'%' +GRANT USAGE ON *.* TO 'test_user'@'%' +# +# Now test the UPPER case role. +# +grant test_ROLE to test_user; +grant insert on secret_db.t1 to test_ROLE; +show grants for test_user; +Grants for test_user@% +GRANT test_role TO 'test_user'@'%' +GRANT test_ROLE TO 'test_user'@'%' +GRANT USAGE ON *.* TO 'test_user'@'%' +# +# Test users privileges when interacting with those roles; +# +show tables from secret_db; +ERROR 42000: Access denied for user 'test_user'@'%' to database 'secret_db' +set role test_ROLE; +show tables from secret_db; +Tables_in_secret_db +t1 +select * from secret_db.t1; +ERROR 42000: SELECT command denied to user 'test_user'@'localhost' for table 't1' +insert into secret_db.t1 values ("|-|4><"); +set role test_role; +select * from secret_db.t1 order by secret; +secret +Some Secret P4ssw0rd +|-|4>< +insert into secret_db.t1 values ("|_33T|-|4><"); +ERROR 42000: INSERT command denied to user 'test_user'@'localhost' for table 't1' +drop role test_ROLE; +drop role test_role; +drop user test_user; +drop database secret_db; diff --git a/mysql-test/suite/roles/role_case_sensitive-10744.test b/mysql-test/suite/roles/role_case_sensitive-10744.test new file mode 100644 index 0000000000000..281d61bce00ea --- /dev/null +++ b/mysql-test/suite/roles/role_case_sensitive-10744.test @@ -0,0 +1,54 @@ +--source include/not_embedded.inc +--echo # +--echo # MDEV-10744 Roles are not fully case-sensitive +--echo # + +--echo # +--echo # Test creating two case-different roles. +--echo # +create user test_user@'%'; +create role test_ROLE; +create role test_role; +--echo # +--echo # Test if mysql.user has the roles created. +--echo # +--sorted_result +select user, host from mysql.user where is_role='y' and user like 'test%'; + +create database secret_db; +create table secret_db.t1 (secret varchar(100)); +insert into secret_db.t1 values ("Some Secret P4ssw0rd"); + +grant select on secret_db.* to test_role; +grant test_role to test_user; +show grants for test_user; +--echo # +--echo # Now test the UPPER case role. +--echo # +grant test_ROLE to test_user; +grant insert on secret_db.t1 to test_ROLE; +show grants for test_user; +connect (test_user,localhost,test_user); + +--echo # +--echo # Test users privileges when interacting with those roles; +--echo # +--error ER_DBACCESS_DENIED_ERROR +show tables from secret_db; +set role test_ROLE; +show tables from secret_db; +--error ER_TABLEACCESS_DENIED_ERROR +select * from secret_db.t1; +insert into secret_db.t1 values ("|-|4><"); +set role test_role; +select * from secret_db.t1 order by secret; +--error ER_TABLEACCESS_DENIED_ERROR +insert into secret_db.t1 values ("|_33T|-|4><"); + +connection default; + + +drop role test_ROLE; +drop role test_role; +drop user test_user; +drop database secret_db; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index f06e9ce647c2c..be1425e294a14 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -1600,7 +1600,7 @@ my_bool acl_reload(THD *thd) my_hash_init2(&acl_roles,50, &my_charset_utf8_bin, 0, 0, 0, (my_hash_get_key) acl_role_get_key, 0, (void (*)(void *))free_acl_role, 0); - my_hash_init2(&acl_roles_mappings, 50, system_charset_info, 0, 0, 0, + my_hash_init2(&acl_roles_mappings, 50, &my_charset_utf8_bin, 0, 0, 0, (my_hash_get_key) acl_role_map_get_key, 0, 0, 0); old_mem= acl_memroot; delete_dynamic(&acl_wild_hosts); From 2fd3af44830e8df9d60f2e8a955f9ed17e744986 Mon Sep 17 00:00:00 2001 From: sensssz Date: Thu, 1 Dec 2016 13:45:23 -0500 Subject: [PATCH 220/295] MDEV-11168: InnoDB: Failing assertion: !other_lock || wsrep_thd_is_BF(lock->trx->mysql_thd, FALSE) || wsrep_thd_is_BF(other_lock->trx->mysql_thd, FALSE) Merged pull request: Fix error in lock_has_higher_priority #266 https://github.com/MariaDB/server/pull/266 Added test case. --- .../r/innodb-lock-schedule-algorithm.result | 89 +++++++++++++++ .../t/innodb-lock-schedule-algorithm.opt | 2 + .../t/innodb-lock-schedule-algorithm.test | 106 ++++++++++++++++++ storage/innobase/lock/lock0lock.cc | 41 ++++--- storage/xtradb/lock/lock0lock.cc | 26 ++--- 5 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb-lock-schedule-algorithm.result create mode 100644 mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.opt create mode 100644 mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.test diff --git a/mysql-test/suite/innodb/r/innodb-lock-schedule-algorithm.result b/mysql-test/suite/innodb/r/innodb-lock-schedule-algorithm.result new file mode 100644 index 0000000000000..069ab3a1bf787 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-lock-schedule-algorithm.result @@ -0,0 +1,89 @@ +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * from t1; +i1 +1 +2 +UPDATE t1 SET i1 = 1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +COMMIT; +COMMIT; +SELECT * FROM t1; +i1 +SELECT * FROM t2; +i2 +1 +2 +DROP TABLE t1, t2; +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * FROM t1; +i1 +1 +2 +UPDATE t1 SET i1 = 1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +COMMIT; +COMMIT; +SELECT * FROM t1; +i1 +SELECT * FROM t2; +i2 +1 +2 +DROP TABLE t1, t2; +# "restart: --loose-innodb-lock-schedule-algorithm=FCFS" +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * from t1; +i1 +1 +2 +UPDATE t1 SET i1 = 1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +COMMIT; +COMMIT; +SELECT * FROM t1; +i1 +SELECT * FROM t2; +i2 +1 +2 +DROP TABLE t1, t2; +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * FROM t1; +i1 +1 +2 +UPDATE t1 SET i1 = 1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +COMMIT; +COMMIT; +SELECT * FROM t1; +i1 +SELECT * FROM t2; +i2 +1 +2 +DROP TABLE t1, t2; diff --git a/mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.opt b/mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.opt new file mode 100644 index 0000000000000..e5d34636ccbc5 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.opt @@ -0,0 +1,2 @@ +--loose-innodb-lock-wait-timeout=1 +--loose-innodb-lock-schedule-algorithm=VATS diff --git a/mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.test b/mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.test new file mode 100644 index 0000000000000..a14c156546a1e --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-lock-schedule-algorithm.test @@ -0,0 +1,106 @@ +--source include/have_innodb.inc + +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; + +--connect (con1,localhost,root,,test) +connection con1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * from t1; +--error 1205 +UPDATE t1 SET i1 = 1; +COMMIT; + +connection default; +COMMIT; +SELECT * FROM t1; +SELECT * FROM t2; +DROP TABLE t1, t2; +disconnect con1; + +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; + +--connect (con1,localhost,root,,test) +connection con1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * FROM t1; +--error 1205 +UPDATE t1 SET i1 = 1; + +connection default; +COMMIT; + +connection con1; +COMMIT; + +connection default; +SELECT * FROM t1; +SELECT * FROM t2; +DROP TABLE t1, t2; +disconnect con1; + +--echo # "restart: --loose-innodb-lock-schedule-algorithm=FCFS" +--let $restart_parameters=--loose_innodb_lock_schedule_algorithm=FCFS +-- source include/restart_mysqld.inc + +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; + +--connect (con1,localhost,root,,test) +connection con1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * from t1; +--error 1205 +UPDATE t1 SET i1 = 1; +COMMIT; + +connection default; +COMMIT; +SELECT * FROM t1; +SELECT * FROM t2; +DROP TABLE t1, t2; +disconnect con1; + +CREATE TABLE t1 (i1 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (i2 int) ENGINE=MyISAM; +BEGIN; +DELETE FROM t1; + +--connect (con1,localhost,root,,test) +connection con1; +BEGIN; +INSERT INTO t2 VALUES (1),(2); +SELECT * FROM t1; +--error 1205 +UPDATE t1 SET i1 = 1; + +connection default; +COMMIT; + +connection con1; +COMMIT; + +connection default; +SELECT * FROM t1; +SELECT * FROM t2; +DROP TABLE t1, t2; +disconnect con1; + diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 9f15c7b3b372b..8f25366a47bbc 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -2267,8 +2267,6 @@ lock_rec_create( /*********************************************************************//** Check if lock1 has higher priority than lock2. NULL has lowest priority. -Respect the preference of the upper server layer to reduce conflict -during in-order parallel replication. If neither of them is wait lock, the first one has higher priority. If only one of them is a wait lock, it has lower priority. Otherwise, the one with an older transaction has higher priority. @@ -2282,22 +2280,13 @@ has_higher_priority( return false; } else if (lock2 == NULL) { return true; - } - // Ask the upper server layer if any of the two trx should be prefered. - int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); - if (preference == -1) { - // lock1 is preferred as a victim, so lock2 has higher priority - return false; - } else if (preference == 1) { - // lock2 is preferred as a victim, so lock1 has higher priority - return true; - } - // No preference. Compre them by wait mode and trx age. - if (!lock_get_wait(lock1)) { - return true; - } else if (!lock_get_wait(lock2)) { - return false; - } + } + // No preference. Compre them by wait mode and trx age. + if (!lock_get_wait(lock1)) { + return true; + } else if (!lock_get_wait(lock2)) { + return false; + } return lock1->trx->start_time_micro <= lock2->trx->start_time_micro; } @@ -6652,7 +6641,6 @@ lock_rec_queue_validate( if (!lock_rec_get_gap(lock) && !lock_get_wait(lock)) { -#ifndef WITH_WSREP enum lock_mode mode; if (lock_get_mode(lock) == LOCK_S) { @@ -6660,10 +6648,21 @@ lock_rec_queue_validate( } else { mode = LOCK_S; } - ut_a(!lock_rec_other_has_expl_req( - mode, 0, 0, block, heap_no, lock->trx)); + + const lock_t* other_lock + = lock_rec_other_has_expl_req( + mode, 0, 0, block, heap_no, + lock->trx); +#ifdef WITH_WSREP + ut_a(!other_lock + || wsrep_thd_is_BF(lock->trx->mysql_thd, FALSE) + || wsrep_thd_is_BF(other_lock->trx->mysql_thd, FALSE)); + +#else + ut_a(!other_lock); #endif /* WITH_WSREP */ + } else if (lock_get_wait(lock) && !lock_rec_get_gap(lock) && innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS) { // If using VATS, it's possible that a wait lock is inserted to a place in the list diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 13edfb2e7ea71..af2c823af64aa 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -2057,8 +2057,6 @@ wsrep_print_wait_locks( /*********************************************************************//** Check if lock1 has higher priority than lock2. NULL has lowest priority. -Respect the preference of the upper server layer to reduce conflict -during in-order parallel replication. If neither of them is wait lock, the first one has higher priority. If only one of them is a wait lock, it has lower priority. Otherwise, the one with an older transaction has higher priority. @@ -2073,15 +2071,6 @@ has_higher_priority( } else if (lock2 == NULL) { return true; } - // Ask the upper server layer if any of the two trx should be prefered. - int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); - if (preference == -1) { - // lock1 is preferred as a victim, so lock2 has higher priority - return false; - } else if (preference == 1) { - // lock2 is preferred as a victim, so lock1 has higher priority - return true; - } // No preference. Compre them by wait mode and trx age. if (!lock_get_wait(lock1)) { return true; @@ -6713,7 +6702,6 @@ lock_rec_queue_validate( if (!lock_rec_get_gap(lock) && !lock_get_wait(lock)) { -#ifndef WITH_WSREP enum lock_mode mode; @@ -6722,8 +6710,18 @@ lock_rec_queue_validate( } else { mode = LOCK_S; } - ut_a(!lock_rec_other_has_expl_req( - mode, 0, 0, block, heap_no, lock->trx->id)); + + const lock_t* other_lock + = lock_rec_other_has_expl_req( + mode, 0, 0, block, heap_no, + lock->trx->id); +#ifdef WITH_WSREP + ut_a(!other_lock + || wsrep_thd_is_BF(lock->trx->mysql_thd, FALSE) + || wsrep_thd_is_BF(other_lock->trx->mysql_thd, FALSE)); + +#else + ut_a(!other_lock); #endif /* WITH_WSREP */ } else if (lock_get_wait(lock) && !lock_rec_get_gap(lock) From 7436c3d6abdd7d2b253ea0de108c7db0c72bdb0a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 2 Dec 2016 10:22:18 +0100 Subject: [PATCH 221/295] 5.6.34-79.1 --- storage/xtradb/handler/ha_innodb.cc | 28 ---------------------------- storage/xtradb/include/univ.i | 2 +- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 58d638d0b0c30..041d370af4d1a 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -7765,12 +7765,6 @@ ha_innobase::write_row( error = row_insert_for_mysql((byte*) record, prebuilt); DEBUG_SYNC(user_thd, "ib_after_row_insert"); -#ifdef EXTENDED_FOR_USERSTAT - if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { - rows_changed++; - } -#endif - /* Handle duplicate key errors */ if (auto_inc_used) { ulonglong auto_inc; @@ -8284,12 +8278,6 @@ ha_innobase::update_row( } } -#ifdef EXTENDED_FOR_USERSTAT - if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { - rows_changed++; - } -#endif - innobase_srv_conc_exit_innodb(trx); func_exit: @@ -8362,12 +8350,6 @@ ha_innobase::delete_row( error = row_update_for_mysql((byte*) record, prebuilt); -#ifdef EXTENDED_FOR_USERSTAT - if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { - rows_changed++; - } -#endif - innobase_srv_conc_exit_innodb(trx); /* Tell the InnoDB server that there might be work for @@ -8717,11 +8699,6 @@ ha_innobase::index_read( error = 0; table->status = 0; srv_stats.n_rows_read.add((size_t) prebuilt->trx->id, 1); -#ifdef EXTENDED_FOR_USERSTAT - rows_read++; - if (active_index < MAX_KEY) - index_rows_read[active_index]++; -#endif break; case DB_RECORD_NOT_FOUND: error = HA_ERR_KEY_NOT_FOUND; @@ -8989,11 +8966,6 @@ ha_innobase::general_fetch( error = 0; table->status = 0; srv_stats.n_rows_read.add((size_t) prebuilt->trx->id, 1); -#ifdef EXTENDED_FOR_USERSTAT - rows_read++; - if (active_index < MAX_KEY) - index_rows_read[active_index]++; -#endif break; case DB_RECORD_NOT_FOUND: error = HA_ERR_END_OF_FILE; diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 296c04d9f621b..2a538aa4104a6 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -47,7 +47,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_BUGFIX MYSQL_VERSION_PATCH #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 79.0 +#define PERCONA_INNODB_VERSION 79.1 #endif /* Enable UNIV_LOG_ARCHIVE in XtraDB */ From d4f0686cd819f2cd4fb78822ba56fda8ddce19e4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 2 Dec 2016 10:24:00 +0100 Subject: [PATCH 222/295] 5.6.34-79.1 --- storage/tokudb/CMakeLists.txt | 2 +- storage/tokudb/ha_tokudb.cc | 3 + .../tokudb/r/table_index_statistics.result | 48 ++++++++++++ .../tokudb/t/table_index_statistics.test | 8 ++ .../r/rpl_tokudb_rfr_partition_table.result | 36 +++++++++ .../rpl_tokudb_rfr_partition_table-slave.opt | 1 + .../t/rpl_tokudb_rfr_partition_table.test | 74 +++++++++++++++++++ 7 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result create mode 100644 storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index ad30e6d40eb4f..7d723dcd9dddb 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(TOKUDB_VERSION 5.6.33-79.0) +SET(TOKUDB_VERSION 5.6.34-79.1) # PerconaFT only supports x86-64 and cmake-2.8.9+ IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT CMAKE_VERSION VERSION_LESS "2.8.9") diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index a77f46de9d0b1..c6935d7ead59e 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -3695,6 +3695,8 @@ int ha_tokudb::do_uniqueness_checks(uchar* record, DB_TXN* txn, THD* thd) { // first do uniqueness checks // if (share->has_unique_keys && do_unique_checks(thd, in_rpl_write_rows)) { + DBUG_EXECUTE_IF("tokudb_crash_if_rpl_does_uniqueness_check", + DBUG_ASSERT(0);); for (uint keynr = 0; keynr < table_share->keys; keynr++) { bool is_unique_key = (table->key_info[keynr].flags & HA_NOSAME) || (keynr == primary_key); bool is_unique = false; @@ -5915,6 +5917,7 @@ int ha_tokudb::rnd_pos(uchar * buf, uchar * pos) { // test rpl slave by inducing a delay before the point query THD *thd = ha_thd(); if (thd->slave_thread && (in_rpl_delete_rows || in_rpl_update_rows)) { + DBUG_EXECUTE_IF("tokudb_crash_if_rpl_looks_up_row", DBUG_ASSERT(0);); uint64_t delay_ms = tokudb::sysvars::rpl_lookup_rows_delay(thd); if (delay_ms) usleep(delay_ms * 1000); diff --git a/storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result b/storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result new file mode 100644 index 0000000000000..73b0759c466be --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result @@ -0,0 +1,48 @@ +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = TOKUDB; +FLUSH INDEX_STATISTICS; +FLUSH TABLE_STATISTICS; +SET @userstat_old= @@userstat; +SET GLOBAL userstat=ON; +CREATE TABLE t1 (id int(10), PRIMARY KEY (id)); +INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH TABLE_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH INDEX_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +DROP TABLE t1; +CREATE TABLE t2 (c1 INT UNSIGNED); +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +CREATE TABLE t2 (c1 INT UNSIGNED) ENGINE=InnoDB; +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +SET GLOBAL userstat= @userstat_old; +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test b/storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test new file mode 100644 index 0000000000000..6acacfb1a1862 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test @@ -0,0 +1,8 @@ +--source include/have_tokudb.inc + +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = TOKUDB; + +--source include/percona_table_index_statistics.inc + +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result new file mode 100644 index 0000000000000..4594959c6d0be --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result @@ -0,0 +1,36 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +call mtr.add_suppression(".*read free replication is disabled for TokuDB table.*continue with rows lookup"); +CREATE TABLE t1 (id int(11) NOT NULL, pid int(11), PRIMARY KEY (id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, +PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, +PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); +insert into t1 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); +CREATE TABLE t2 (id int(11) NOT NULL, pid int(11), key idx_1(id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, +PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, +PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); +insert into t2 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); +include/stop_slave.inc +set global debug= "+d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +include/start_slave.inc +insert into t1 values(21, 21); +delete from t1 where id = 11; +update t1 set pid = 2 where id = 1; +include/diff_tables.inc [master:test.t1, slave:test.t1] +insert into t2 values(21, 21); +delete from t2 where id = 11; +update t2 set pid = 2 where id = 1; +include/diff_tables.inc [master:test.t2, slave:test.t2] +drop table t1; +drop table t2; +include/stop_slave.inc +set global debug= "-d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +set global debug= @saved_debug; +include/start_slave.inc +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt new file mode 100644 index 0000000000000..2a7ec2590cc36 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt @@ -0,0 +1 @@ +--read-only=ON --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows=OFF diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test new file mode 100644 index 0000000000000..319e741e12c0f --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test @@ -0,0 +1,74 @@ +# test tokudb read free replication feature with partition table + +--source include/have_debug.inc +--source include/have_tokudb.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +call mtr.add_suppression(".*read free replication is disabled for TokuDB table.*continue with rows lookup"); + +connection master; + +# partition table with explicit PK +CREATE TABLE t1 (id int(11) NOT NULL, pid int(11), PRIMARY KEY (id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, + PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, + PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); + +insert into t1 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); + +# partition table without explicit PK +CREATE TABLE t2 (id int(11) NOT NULL, pid int(11), key idx_1(id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, + PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, + PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); + +insert into t2 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); + +--sync_slave_with_master + +# set tokudb rfr crash/assert conditions if we enter lookup code +# to make sure no unique checks or row lookups is invoked +connection slave; +--source include/stop_slave.inc +let $saved_debug = `select @@debug`; +set global debug= "+d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +--source include/start_slave.inc + +connection master; +insert into t1 values(21, 21); +delete from t1 where id = 11; +update t1 set pid = 2 where id = 1; + +sync_slave_with_master; + +connection master; + +--let $diff_tables= master:test.t1, slave:test.t1 +--source include/diff_tables.inc + +# print rfr disabled warning in errlog +connection master; +insert into t2 values(21, 21); +delete from t2 where id = 11; +update t2 set pid = 2 where id = 1; + +sync_slave_with_master; + +--let $diff_tables= master:test.t2, slave:test.t2 +--source include/diff_tables.inc + +connection master; +drop table t1; +drop table t2; +sync_slave_with_master; + +connection slave; +--source include/stop_slave.inc +set global debug= "-d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +set global debug= @saved_debug; +--source include/start_slave.inc + +--source include/rpl_end.inc From 2996f9aa883d28be3e3156f8cbd4d34ce6797e3c Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 30 Nov 2016 18:36:29 +0200 Subject: [PATCH 223/295] MDEV-11429 Increase number of max table_open_cache instances Increase max number of possible table_open_cache instances from 512K to 1024K. This only affects user who are trying to set the variable over the old limit. Delete not used test table_open_cache_instances_basic (Need to be added back and rewritten in 10.2) --- mysql-test/suite/sys_vars/disabled.def | 1 - mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result | 2 +- mysql-test/suite/sys_vars/r/table_open_cache_basic.result | 4 ++-- .../suite/sys_vars/r/table_open_cache_instances_basic.result | 3 --- .../suite/sys_vars/t/table_open_cache_instances_basic.test | 1 - sql/sys_vars.cc | 2 +- 6 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 mysql-test/suite/sys_vars/r/table_open_cache_instances_basic.result delete mode 100644 mysql-test/suite/sys_vars/t/table_open_cache_instances_basic.test diff --git a/mysql-test/suite/sys_vars/disabled.def b/mysql-test/suite/sys_vars/disabled.def index a009bc5c174e1..e4a2699f03135 100644 --- a/mysql-test/suite/sys_vars/disabled.def +++ b/mysql-test/suite/sys_vars/disabled.def @@ -11,5 +11,4 @@ ############################################################################## innodb_flush_checkpoint_debug_basic: removed from XtraDB-26.0 -table_open_cache_instances_basic: no such variable in MariaDB all_vars: obsolete, see sysvars_* tests diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 0797110926e33..33d1f9722df63 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -4540,7 +4540,7 @@ VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 1 -NUMERIC_MAX_VALUE 524288 +NUMERIC_MAX_VALUE 1048576 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/table_open_cache_basic.result b/mysql-test/suite/sys_vars/r/table_open_cache_basic.result index d940ed6990128..4c63578375390 100644 --- a/mysql-test/suite/sys_vars/r/table_open_cache_basic.result +++ b/mysql-test/suite/sys_vars/r/table_open_cache_basic.result @@ -27,7 +27,7 @@ Warnings: Warning 1292 Truncated incorrect table_open_cache value: '1073741824' SELECT @@global.table_open_cache ; @@global.table_open_cache -524288 +1048576 SET @@global.table_open_cache = 18000; SELECT @@global.table_open_cache ; @@global.table_open_cache @@ -48,7 +48,7 @@ Warnings: Warning 1292 Truncated incorrect table_open_cache value: '100000000000' SELECT @@global.table_open_cache ; @@global.table_open_cache -524288 +1048576 SET @@global.table_open_cache = -1024; Warnings: Warning 1292 Truncated incorrect table_open_cache value: '-1024' diff --git a/mysql-test/suite/sys_vars/r/table_open_cache_instances_basic.result b/mysql-test/suite/sys_vars/r/table_open_cache_instances_basic.result deleted file mode 100644 index e735c5ccbc658..0000000000000 --- a/mysql-test/suite/sys_vars/r/table_open_cache_instances_basic.result +++ /dev/null @@ -1,3 +0,0 @@ -select @@table_open_cache_instances; -@@table_open_cache_instances -1 diff --git a/mysql-test/suite/sys_vars/t/table_open_cache_instances_basic.test b/mysql-test/suite/sys_vars/t/table_open_cache_instances_basic.test deleted file mode 100644 index f7616c2c4e9a6..0000000000000 --- a/mysql-test/suite/sys_vars/t/table_open_cache_instances_basic.test +++ /dev/null @@ -1 +0,0 @@ -select @@table_open_cache_instances; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 306528c5c86d8..0518e434cbefe 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3219,7 +3219,7 @@ static bool fix_table_open_cache(sys_var *, THD *, enum_var_type) static Sys_var_ulong Sys_table_cache_size( "table_open_cache", "The number of cached open tables", GLOBAL_VAR(tc_size), CMD_LINE(REQUIRED_ARG), - VALID_RANGE(1, 512*1024), DEFAULT(TABLE_OPEN_CACHE_DEFAULT), + VALID_RANGE(1, 1024*1024), DEFAULT(TABLE_OPEN_CACHE_DEFAULT), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(fix_table_open_cache)); From 97b21a195354e1f00353a2810e18f0c38fc039cb Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 2 Dec 2016 14:02:30 +0200 Subject: [PATCH 224/295] MDEV-10759 Fix Aria to support 2-byte collation IDs - Used same fix as for MyISAM: High level collation byte stored in unused bit_end position. - Moved language from header to base_info - Removed unused bit_end part in HA_KEY_SEG --- include/maria.h | 2 +- include/my_compare.h | 2 +- mysql-test/suite/maria/collations.result | 25 ++++++++++++++++++++++++ mysql-test/suite/maria/collations.test | 14 +++++++++++++ storage/maria/ha_maria.cc | 3 +-- storage/maria/ma_check.c | 2 +- storage/maria/ma_create.c | 6 +++--- storage/maria/ma_open.c | 23 +++++++++++++++------- storage/maria/maria_chk.c | 6 +++--- storage/maria/maria_def.h | 6 ++++-- storage/myisam/ft_static.c | 4 ++-- storage/myisam/ha_myisam.cc | 3 +-- storage/myisam/mi_create.c | 1 - storage/myisam/mi_open.c | 1 - 14 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 mysql-test/suite/maria/collations.result create mode 100644 mysql-test/suite/maria/collations.test diff --git a/include/maria.h b/include/maria.h index bdd53f3d183d6..1305b6444c02e 100644 --- a/include/maria.h +++ b/include/maria.h @@ -149,7 +149,7 @@ typedef struct st_maria_create_info uint null_bytes; uint old_options; enum data_file_type org_data_file_type; - uint8 language; + uint16 language; my_bool with_auto_increment, transactional; } MARIA_CREATE_INFO; diff --git a/include/my_compare.h b/include/my_compare.h index 0db22b593f425..3440d9ef920c1 100644 --- a/include/my_compare.h +++ b/include/my_compare.h @@ -57,7 +57,7 @@ typedef struct st_HA_KEYSEG /* Key-portion */ uint16 language; uint8 type; /* Type of key (for sort) */ uint8 null_bit; /* bitmask to test for NULL */ - uint8 bit_start,bit_end; /* if bit field */ + uint8 bit_start; uint8 bit_length; /* Length of bit part */ } HA_KEYSEG; diff --git a/mysql-test/suite/maria/collations.result b/mysql-test/suite/maria/collations.result new file mode 100644 index 0000000000000..86f7117c378aa --- /dev/null +++ b/mysql-test/suite/maria/collations.result @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS t1; +Warnings: +Note 1051 Unknown table 'test.t1' +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_croatian_ci, KEY(a)) ENGINE=ARIA; +INSERT INTO t1 VALUES ('na'),('nj'),('nz'),('Z'); +explain SELECT a FROM t1 ORDER BY a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL a 33 NULL 4 Using index +SELECT a FROM t1 ORDER BY a; +a +na +nz +nj +Z +ALTER TABLE t1 engine=myisam; +explain SELECT a FROM t1 ORDER BY a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL a 33 NULL 4 Using index +SELECT a FROM t1 ORDER BY a; +a +na +nz +nj +Z +drop table t1; diff --git a/mysql-test/suite/maria/collations.test b/mysql-test/suite/maria/collations.test new file mode 100644 index 0000000000000..fe93e1913a7e6 --- /dev/null +++ b/mysql-test/suite/maria/collations.test @@ -0,0 +1,14 @@ +# +# Test 2-byte collations +# + +DROP TABLE IF EXISTS t1; + +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_croatian_ci, KEY(a)) ENGINE=ARIA; +INSERT INTO t1 VALUES ('na'),('nj'),('nz'),('Z'); +explain SELECT a FROM t1 ORDER BY a; +SELECT a FROM t1 ORDER BY a; +ALTER TABLE t1 engine=myisam; +explain SELECT a FROM t1 ORDER BY a; +SELECT a FROM t1 ORDER BY a; +drop table t1; diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index c6d457eeea86f..c1a99370d80a2 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -549,8 +549,7 @@ static int table2maria(TABLE *table_arg, data_file_type row_type, keydef[i].seg[j].type= (int) type; keydef[i].seg[j].start= pos->key_part[j].offset; keydef[i].seg[j].length= pos->key_part[j].length; - keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_end= - keydef[i].seg[j].bit_length= 0; + keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_length= 0; keydef[i].seg[j].bit_pos= 0; keydef[i].seg[j].language= field->charset()->number; diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index 5035df26b18f5..8424817bc6b58 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -6103,7 +6103,7 @@ int maria_recreate_table(HA_CHECK *param, MARIA_HA **org_info, char *filename) create_info.data_file_length=file_length; create_info.auto_increment=share.state.auto_increment; create_info.language = (param->language ? param->language : - share.state.header.language); + share.base.language); create_info.key_file_length= status_info.key_file_length; create_info.org_data_file_type= ((enum data_file_type) share.state.header.org_data_file_type); diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index f160499a94e70..0680b5d568e07 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -725,8 +725,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, mi_int2store(share.state.header.base_pos,base_pos); share.state.header.data_file_type= share.data_file_type= datafile_type; share.state.header.org_data_file_type= org_datafile_type; - share.state.header.language= (ci->language ? - ci->language : default_charset_info->number); + share.state.header.not_used= 0; share.state.dellink = HA_OFFSET_ERROR; share.state.first_bitmap_with_space= 0; @@ -739,6 +738,8 @@ int maria_create(const char *name, enum data_file_type datafile_type, share.options=options; share.base.rec_reflength=pointer; share.base.block_size= maria_block_size; + share.base.language= (ci->language ? ci->language : + default_charset_info->number); /* Get estimate for index file length (this may be wrong for FT keys) @@ -937,7 +938,6 @@ int maria_create(const char *name, enum data_file_type datafile_type, sseg.language= 7; /* Binary */ sseg.null_bit=0; sseg.bit_start=0; - sseg.bit_end=0; sseg.bit_length= 0; sseg.bit_pos= 0; sseg.length=SPLEN; diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index 42861e92ed417..4e97c6b43b9cb 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -276,6 +276,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) uint i,j,len,errpos,head_length,base_pos,keys, realpath_err, key_parts,unique_key_parts,fulltext_keys,uniques; uint internal_table= MY_TEST(open_flags & HA_OPEN_INTERNAL_TABLE); + uint file_version; size_t info_length; char name_buff[FN_REFLEN], org_name[FN_REFLEN], index_name[FN_REFLEN], data_name[FN_REFLEN]; @@ -335,8 +336,8 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) } share->mode=open_mode; errpos= 1; - if (mysql_file_pread(kfile,share->state.header.file_version, head_length, 0, - MYF(MY_NABP))) + if (mysql_file_pread(kfile,share->state.header.file_version, head_length, + 0, MYF(MY_NABP))) { my_errno= HA_ERR_NOT_A_TABLE; goto err; @@ -429,6 +430,14 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) len,MARIA_BASE_INFO_SIZE)); } disk_pos= _ma_base_info_read(disk_cache + base_pos, &share->base); + /* + Check if old version of Aria file. Version 0 has language + stored in header.not_used + */ + file_version= (share->state.header.not_used == 0); + if (file_version == 0) + share->base.language= share->state.header.not_used; + share->state.state_length=base_pos; /* For newly opened tables we reset the error-has-been-printed flag */ share->state.changed&= ~STATE_CRASHED_PRINTED; @@ -1581,7 +1590,7 @@ uint _ma_base_info_write(File file, MARIA_BASE_INFO *base) mi_int2store(ptr,base->null_bytes); ptr+= 2; mi_int2store(ptr,base->original_null_bytes); ptr+= 2; mi_int2store(ptr,base->field_offsets); ptr+= 2; - mi_int2store(ptr,0); ptr+= 2; /* reserved */ + mi_int2store(ptr,base->language); ptr+= 2; mi_int2store(ptr,base->block_size); ptr+= 2; *ptr++= base->rec_reflength; *ptr++= base->key_reflength; @@ -1624,7 +1633,7 @@ static uchar *_ma_base_info_read(uchar *ptr, MARIA_BASE_INFO *base) base->null_bytes= mi_uint2korr(ptr); ptr+= 2; base->original_null_bytes= mi_uint2korr(ptr); ptr+= 2; base->field_offsets= mi_uint2korr(ptr); ptr+= 2; - ptr+= 2; + base->language= mi_uint2korr(ptr); ptr+= 2; base->block_size= mi_uint2korr(ptr); ptr+= 2; base->rec_reflength= *ptr++; @@ -1689,10 +1698,10 @@ my_bool _ma_keyseg_write(File file, const HA_KEYSEG *keyseg) ulong pos; *ptr++= keyseg->type; - *ptr++= keyseg->language; + *ptr++= keyseg->language & 0xFF; /* Collation ID, low byte */ *ptr++= keyseg->null_bit; *ptr++= keyseg->bit_start; - *ptr++= keyseg->bit_end; + *ptr++= keyseg->language >> 8; /* Collation ID, high byte */ *ptr++= keyseg->bit_length; mi_int2store(ptr,keyseg->flag); ptr+= 2; mi_int2store(ptr,keyseg->length); ptr+= 2; @@ -1711,7 +1720,7 @@ uchar *_ma_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg) keyseg->language = *ptr++; keyseg->null_bit = *ptr++; keyseg->bit_start = *ptr++; - keyseg->bit_end = *ptr++; + keyseg->language += ((uint16) (*ptr++)) << 8; keyseg->bit_length = *ptr++; keyseg->flag = mi_uint2korr(ptr); ptr+= 2; keyseg->length = mi_uint2korr(ptr); ptr+= 2; diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c index c9d38400bc4f7..0c1c56dfa94b6 100644 --- a/storage/maria/maria_chk.c +++ b/storage/maria/maria_chk.c @@ -1120,7 +1120,7 @@ static int maria_chk(HA_CHECK *param, char *filename) maria_test_if_almost_full(info) || info->s->state.header.file_version[3] != maria_file_magic[3] || (set_collation && - set_collation->number != share->state.header.language))) + set_collation->number != share->base.language))) { if (set_collation) param->language= set_collation->number; @@ -1507,8 +1507,8 @@ static void descript(HA_CHECK *param, register MARIA_HA *info, char *name) printf("Crashsafe: %s\n", share->base.born_transactional ? "yes" : "no"); printf("Character set: %s (%d)\n", - get_charset_name(share->state.header.language), - share->state.header.language); + get_charset_name(share->base.language), + (int) share->base.language); if (param->testflag & T_VERBOSE) { diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index a4fac8c088a96..d216384a2aa7e 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -139,7 +139,7 @@ typedef struct st_maria_state_info uchar unique_key_parts[2]; /* Key parts + unique parts */ uchar keys; /* number of keys in file */ uchar uniques; /* number of UNIQUE definitions */ - uchar language; /* Language for indexes */ + uchar not_used; /* Language for indexes */ uchar fulltext_keys; uchar data_file_type; /* Used by mariapack to store the original data_file_type */ @@ -209,6 +209,7 @@ typedef struct st_maria_state_info } MARIA_STATE_INFO; +/* Number of bytes written be _ma_state_info_write_sub() */ #define MARIA_STATE_INFO_SIZE \ (24 + 2 + LSN_STORE_SIZE*3 + 4 + 11*8 + 4*4 + 8 + 3*4 + 5*8) #define MARIA_FILE_OPEN_COUNT_OFFSET 0 @@ -291,6 +292,8 @@ typedef struct st_ma_base_info uint extra_rec_buff_size; /* Tuning flags that can be ignored by older Maria versions */ uint extra_options; + /* default language, not really used but displayed by maria_chk */ + uint language; /* The following are from the header */ uint key_parts, all_key_parts; @@ -916,7 +919,6 @@ extern mysql_mutex_t THR_LOCK_maria; #define MARIA_SMALL_BLOB_BUFFER 1024 #define MARIA_MAX_CONTROL_FILE_LOCK_RETRY 30 /* Retry this many times */ - /* Some extern variables */ extern LIST *maria_open_list; extern uchar maria_file_magic[], maria_pack_file_magic[]; diff --git a/storage/myisam/ft_static.c b/storage/myisam/ft_static.c index aa8adba88a2f8..92a0621fd9f50 100644 --- a/storage/myisam/ft_static.c +++ b/storage/myisam/ft_static.c @@ -34,7 +34,7 @@ const HA_KEYSEG ft_keysegs[FT_SEGS]= { 63, /* language (will be overwritten) */ HA_KEYTYPE_VARTEXT2, /* type */ 0, /* null_bit */ - 2, 0, 0 /* bit_start, bit_end, bit_length */ + 2, 0 /* bit_start, bit_length */ }, { /* @@ -42,7 +42,7 @@ const HA_KEYSEG ft_keysegs[FT_SEGS]= { be packed in any way, otherwise w_search() won't be able to update key entry 'in vivo' */ - 0, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 63, HA_FT_WTYPE, 0, 0, 0, 0 + 0, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 63, HA_FT_WTYPE, 0, 0, 0 } }; diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 40a27de5cf945..7d518320d5035 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -279,8 +279,7 @@ int table2myisam(TABLE *table_arg, MI_KEYDEF **keydef_out, keydef[i].seg[j].type= (int) type; keydef[i].seg[j].start= pos->key_part[j].offset; keydef[i].seg[j].length= pos->key_part[j].length; - keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_end= - keydef[i].seg[j].bit_length= 0; + keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_length= 0; keydef[i].seg[j].bit_pos= 0; keydef[i].seg[j].language= field->charset_for_protocol()->number; diff --git a/storage/myisam/mi_create.c b/storage/myisam/mi_create.c index 88b9da6f8a91a..c781538dc0fdb 100644 --- a/storage/myisam/mi_create.c +++ b/storage/myisam/mi_create.c @@ -729,7 +729,6 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, sseg.language= 7; /* Binary */ sseg.null_bit=0; sseg.bit_start=0; - sseg.bit_end=0; sseg.bit_length= 0; sseg.bit_pos= 0; sseg.length=SPLEN; diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index b8cbefe6ac27b..2d794f611d76e 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -1183,7 +1183,6 @@ uchar *mi_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg) keyseg->length = mi_uint2korr(ptr); ptr +=2; keyseg->start = mi_uint4korr(ptr); ptr +=4; keyseg->null_pos = mi_uint4korr(ptr); ptr +=4; - keyseg->bit_end= 0; keyseg->charset=0; /* Will be filled in later */ if (keyseg->null_bit) /* We adjust bit_pos if null_bit is last in the byte */ From 1e7f961d008762946cde5f9473fd418d5950551f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 1 Dec 2016 14:56:09 +0200 Subject: [PATCH 225/295] MDEV-9451 innodb_buffer_pool_populate does not seem to work on 10.1.10 Instead of interpreting --innodb-buffer-pool-populate as --innodb-numa-interleave, display warning when the option is set, saying that the option will be removed in MariaDB 10.2.3. --- storage/xtradb/handler/ha_innodb.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index fd3ef64467380..c7056a005fa7a 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -202,6 +202,8 @@ static long innobase_buffer_pool_instances = 1; static ulong innobase_log_block_size; static long long innobase_buffer_pool_size, innobase_log_file_size; +/** Deprecated option that has no effect. */ +static my_bool innodb_buffer_pool_populate; /** Percentage of the buffer pool to reserve for 'old' blocks. Connected to buf_LRU_old_ratio. */ @@ -4282,6 +4284,15 @@ innobase_init( "allocator.\n"); } + if (innodb_buffer_pool_populate) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Warning: Setting " + "innodb_buffer_pool_populate is DEPRECATED" + " and has no effect. " + "This option will be removed in MariaDB 10.2.3.\n"); + } + srv_n_file_io_threads = (ulint) innobase_file_io_threads; srv_n_read_io_threads = (ulint) innobase_read_io_threads; srv_n_write_io_threads = (ulint) innobase_write_io_threads; @@ -20758,9 +20769,10 @@ static MYSQL_SYSVAR_LONGLONG(buffer_pool_size, innobase_buffer_pool_size, "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.", NULL, NULL, 128*1024*1024L, 5*1024*1024L, LONGLONG_MAX, 1024*1024L); -static MYSQL_SYSVAR_BOOL(buffer_pool_populate, srv_numa_interleave, +static MYSQL_SYSVAR_BOOL(buffer_pool_populate, innodb_buffer_pool_populate, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Depricated. This option is temporary alias of --innodb-numa-interleave.", + "Deprecated. This option has no effect and " + "will be removed in MariaDB 10.2.3.", NULL, NULL, FALSE); static MYSQL_SYSVAR_ENUM(foreground_preflush, srv_foreground_preflush, From f1b80d8ef11a8949fcc1c035f5ef0d5fb4382d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Dec 2016 16:25:47 +0200 Subject: [PATCH 226/295] MDEV-11236 Failing assertion: state == TRX_STATE_NOT_STARTED trx_state_eq(): Add the parameter bool relaxed=false, to allow trx->state==TRX_STATE_NOT_STARTED where a different state is expected, if an error has been reported. trx_release_savepoint_for_mysql(): Pass relaxed=true to trx_state_eq(). That is, allow the transaction to be idle when ROLLBACK TO SAVEPOINT is attempted after an error has been reported to the client. --- mysql-test/suite/innodb/r/trigger.result | 23 ++++++++++++ mysql-test/suite/innodb/t/trigger.test | 47 ++++++++++++++++++++++++ storage/innobase/include/trx0trx.h | 7 +++- storage/innobase/include/trx0trx.ic | 11 +++++- storage/innobase/trx/trx0roll.cc | 4 +- storage/xtradb/include/trx0trx.h | 9 ++++- storage/xtradb/include/trx0trx.ic | 11 +++++- storage/xtradb/trx/trx0roll.cc | 4 +- 8 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/innodb/r/trigger.result create mode 100644 mysql-test/suite/innodb/t/trigger.test diff --git a/mysql-test/suite/innodb/r/trigger.result b/mysql-test/suite/innodb/r/trigger.result new file mode 100644 index 0000000000000..c54c14adbfefb --- /dev/null +++ b/mysql-test/suite/innodb/r/trigger.result @@ -0,0 +1,23 @@ +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +CREATE TABLE t2 (b INT) ENGINE=InnoDB; +CREATE TABLE t3 (c INT) ENGINE=InnoDB; +CREATE TRIGGER tr BEFORE INSERT ON t3 FOR EACH ROW BEGIN SAVEPOINT sv; INSERT INTO t2 VALUES (0); END $$ +START TRANSACTION; +DELETE FROM t1; +START TRANSACTION; +INSERT INTO t2 VALUES (2); +UPDATE t2 SET b = b+1; +INSERT INTO t1 VALUES (1); +COMMIT; +COMMIT; +SELECT * FROM t1; +a +1 +SELECT * FROM t2; +b +3 +SELECT * FROM t3; +c +DROP TABLE t1, t2, t3; +DROP TRIGGER tr; +ERROR HY000: Trigger does not exist diff --git a/mysql-test/suite/innodb/t/trigger.test b/mysql-test/suite/innodb/t/trigger.test new file mode 100644 index 0000000000000..fe0bab2149765 --- /dev/null +++ b/mysql-test/suite/innodb/t/trigger.test @@ -0,0 +1,47 @@ +--source include/have_innodb.inc +--source include/count_sessions.inc + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +CREATE TABLE t2 (b INT) ENGINE=InnoDB; +CREATE TABLE t3 (c INT) ENGINE=InnoDB; + +--delimiter $$ +CREATE TRIGGER tr BEFORE INSERT ON t3 FOR EACH ROW BEGIN SAVEPOINT sv; INSERT INTO t2 VALUES (0); END $$ +--delimiter ; + +START TRANSACTION; +DELETE FROM t1; + +connect (con1,localhost,root,,test); +START TRANSACTION; +INSERT INTO t2 VALUES (2); +UPDATE t2 SET b = b+1; + +--send +INSERT INTO t1 VALUES (1); + +connection default; +let $wait_condition= + select count(*) = 1 from information_schema.processlist + where state = 'update' and info = 'INSERT INTO t1 VALUES (1)' +--source include/wait_condition.inc + +--error ER_LOCK_DEADLOCK +INSERT INTO t3 VALUES (2); +COMMIT; + +connection con1; +reap; +COMMIT; +disconnect con1; + +connection default; +SELECT * FROM t1; +SELECT * FROM t2; +SELECT * FROM t3; + +DROP TABLE t1, t2, t3; +--error ER_TRG_DOES_NOT_EXIST +DROP TRIGGER tr; + +--source include/wait_until_count_sessions.inc diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 3cc61de8e7650..11836183d5722 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -379,10 +380,14 @@ ibool trx_state_eq( /*=========*/ const trx_t* trx, /*!< in: transaction */ - trx_state_t state) /*!< in: state; + trx_state_t state, /*!< in: state; if state != TRX_STATE_NOT_STARTED asserts that trx->state != TRX_STATE_NOT_STARTED */ + bool relaxed = false) + /*!< in: whether to allow + trx->state == TRX_STATE_NOT_STARTED + after an error has been reported */ MY_ATTRIBUTE((nonnull, warn_unused_result)); # ifdef UNIV_DEBUG /**********************************************************************//** diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index 69ee17ea98b55..fa30e6229a33f 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,10 +37,14 @@ ibool trx_state_eq( /*=========*/ const trx_t* trx, /*!< in: transaction */ - trx_state_t state) /*!< in: state; + trx_state_t state, /*!< in: state; if state != TRX_STATE_NOT_STARTED asserts that trx->state != TRX_STATE_NOT_STARTED */ + bool relaxed) + /*!< in: whether to allow + trx->state == TRX_STATE_NOT_STARTED + after an error has been reported */ { #ifdef UNIV_DEBUG switch (trx->state) { @@ -57,7 +62,9 @@ trx_state_eq( case TRX_STATE_NOT_STARTED: /* This state is not allowed for running transactions. */ - ut_a(state == TRX_STATE_NOT_STARTED); + ut_a(state == TRX_STATE_NOT_STARTED + || (relaxed + && thd_get_error_number(trx->mysql_thd))); ut_ad(!trx->in_rw_trx_list); ut_ad(!trx->in_ro_trx_list); return(state == trx->state); diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index c65d95a9817f7..0c90a45c6c6ff 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -495,7 +496,8 @@ trx_release_savepoint_for_mysql( { trx_named_savept_t* savep; - ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || trx_state_eq(trx, TRX_STATE_PREPARED)); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE, true) + || trx_state_eq(trx, TRX_STATE_PREPARED, true)); ut_ad(trx->in_mysql_trx_list); savep = trx_savepoint_find(trx, savepoint_name); diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h index 59d6d961544fb..24bbff3098641 100644 --- a/storage/xtradb/include/trx0trx.h +++ b/storage/xtradb/include/trx0trx.h @@ -1,7 +1,8 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, MariaDB Corporation +Copyright (c) 2015, 2016, MariaDB Corporation. All Rights Reserved. + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -393,10 +394,14 @@ ibool trx_state_eq( /*=========*/ const trx_t* trx, /*!< in: transaction */ - trx_state_t state) /*!< in: state; + trx_state_t state, /*!< in: state; if state != TRX_STATE_NOT_STARTED asserts that trx->state != TRX_STATE_NOT_STARTED */ + bool relaxed = false) + /*!< in: whether to allow + trx->state == TRX_STATE_NOT_STARTED + after an error has been reported */ MY_ATTRIBUTE((nonnull, warn_unused_result)); # ifdef UNIV_DEBUG /**********************************************************************//** diff --git a/storage/xtradb/include/trx0trx.ic b/storage/xtradb/include/trx0trx.ic index 8582b63f806b4..eb7d62d9cad58 100644 --- a/storage/xtradb/include/trx0trx.ic +++ b/storage/xtradb/include/trx0trx.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,10 +37,14 @@ ibool trx_state_eq( /*=========*/ const trx_t* trx, /*!< in: transaction */ - trx_state_t state) /*!< in: state; + trx_state_t state, /*!< in: state; if state != TRX_STATE_NOT_STARTED asserts that trx->state != TRX_STATE_NOT_STARTED */ + bool relaxed) + /*!< in: whether to allow + trx->state == TRX_STATE_NOT_STARTED + after an error has been reported */ { #ifdef UNIV_DEBUG switch (trx->state) { @@ -57,7 +62,9 @@ trx_state_eq( case TRX_STATE_NOT_STARTED: /* This state is not allowed for running transactions. */ - ut_a(state == TRX_STATE_NOT_STARTED); + ut_a(state == TRX_STATE_NOT_STARTED + || (relaxed + && thd_get_error_number(trx->mysql_thd))); ut_ad(!trx->in_rw_trx_list); ut_ad(!trx->in_ro_trx_list); return(state == trx->state); diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index 4618cf653f04b..ee1497b209f11 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -507,7 +508,8 @@ trx_release_savepoint_for_mysql( { trx_named_savept_t* savep; - ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || trx_state_eq(trx, TRX_STATE_PREPARED)); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE, true) + || trx_state_eq(trx, TRX_STATE_PREPARED, true)); ut_ad(trx->in_mysql_trx_list); savep = trx_savepoint_find(trx, savepoint_name); From f640527e65cf83b5bfbc09df3c72813c1ca67d8d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 2 Dec 2016 15:22:11 +0100 Subject: [PATCH 227/295] typo fixed: s/MSYQL/MYSQL/ --- mysql-test/suite/binlog/r/binlog_index.result | 6 +++--- mysql-test/suite/binlog/t/binlog_index.test | 6 +++--- mysql-test/suite/rpl/r/rpl_binlog_errors.result | 4 ++-- mysql-test/suite/rpl/t/rpl_binlog_errors.test | 4 ++-- sql/log.cc | 14 +++++++------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/mysql-test/suite/binlog/r/binlog_index.result b/mysql-test/suite/binlog/r/binlog_index.result index ca92cc5398b51..35a86f479ccc2 100644 --- a/mysql-test/suite/binlog/r/binlog_index.result +++ b/mysql-test/suite/binlog/r/binlog_index.result @@ -1,9 +1,9 @@ call mtr.add_suppression('Attempting backtrace'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); -call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); +call mtr.add_suppression('MYSQL_BIN_LOG::open failed to sync the index file'); call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); call mtr.add_suppression('Could not open .*'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); flush tables; RESET MASTER; flush logs; diff --git a/mysql-test/suite/binlog/t/binlog_index.test b/mysql-test/suite/binlog/t/binlog_index.test index a264549ce17d7..09e817c0469a4 100644 --- a/mysql-test/suite/binlog/t/binlog_index.test +++ b/mysql-test/suite/binlog/t/binlog_index.test @@ -9,11 +9,11 @@ source include/have_debug.inc; # Avoid CrashReporter popup on Mac --source include/not_crashrep.inc call mtr.add_suppression('Attempting backtrace'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); -call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); +call mtr.add_suppression('MYSQL_BIN_LOG::open failed to sync the index file'); call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); call mtr.add_suppression('Could not open .*'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); flush tables; let $old=`select @@debug`; diff --git a/mysql-test/suite/rpl/r/rpl_binlog_errors.result b/mysql-test/suite/rpl/r/rpl_binlog_errors.result index ddafbca0672db..a71dcdfb1be74 100644 --- a/mysql-test/suite/rpl/r/rpl_binlog_errors.result +++ b/mysql-test/suite/rpl/r/rpl_binlog_errors.result @@ -169,7 +169,7 @@ count(*) SET SQL_LOG_BIN=1; SET GLOBAL debug_dbug="-d,error_unique_log_filename"; ###################### TEST #10 -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Could not open .*"); RESET MASTER; SHOW WARNINGS; @@ -226,7 +226,7 @@ include/rpl_reset.inc call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); call mtr.add_suppression("Error writing file .*"); call mtr.add_suppression("Could not open .*"); -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Can't generate a unique log-filename .*"); ###################### TEST #13 SET @old_debug=@@global.debug; diff --git a/mysql-test/suite/rpl/t/rpl_binlog_errors.test b/mysql-test/suite/rpl/t/rpl_binlog_errors.test index 9e4a106a5b5e7..fb3b39fac9d7c 100644 --- a/mysql-test/suite/rpl/t/rpl_binlog_errors.test +++ b/mysql-test/suite/rpl/t/rpl_binlog_errors.test @@ -256,7 +256,7 @@ SET GLOBAL debug_dbug="-d,error_unique_log_filename"; ### while registering the index file and the binary log ### file or failure to write the rotate event. -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Could not open .*"); RESET MASTER; @@ -362,7 +362,7 @@ RESET MASTER; call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); call mtr.add_suppression("Error writing file .*"); call mtr.add_suppression("Could not open .*"); -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Can't generate a unique log-filename .*"); -- echo ###################### TEST #13 diff --git a/sql/log.cc b/sql/log.cc index bb8f06c80f7fe..077bbf5e71186 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3059,7 +3059,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, if (init_and_set_log_file_name(log_name, new_name, log_type_arg, io_cache_type_arg)) { - sql_print_error("MSYQL_BIN_LOG::open failed to generate new file name."); + sql_print_error("MYSQL_BIN_LOG::open failed to generate new file name."); DBUG_RETURN(1); } @@ -3086,7 +3086,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, } }); - sql_print_error("MSYQL_BIN_LOG::open failed to sync the index file."); + sql_print_error("MYSQL_BIN_LOG::open failed to sync the index file."); DBUG_RETURN(1); } DBUG_EXECUTE_IF("crash_create_non_critical_before_update_index", DBUG_SUICIDE();); @@ -3849,14 +3849,14 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, if ((error= sync_purge_index_file())) { - sql_print_error("MSYQL_BIN_LOG::purge_logs failed to flush register file."); + sql_print_error("MYSQL_BIN_LOG::purge_logs failed to flush register file."); goto err; } /* We know how many files to delete. Update index file. */ if ((error=update_log_index(&log_info, need_update_threads))) { - sql_print_error("MSYQL_BIN_LOG::purge_logs failed to update the index file"); + sql_print_error("MYSQL_BIN_LOG::purge_logs failed to update the index file"); goto err; } @@ -3866,7 +3866,7 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, /* Read each entry from purge_index_file and delete the file. */ if (is_inited_purge_index_file() && (error= purge_index_entry(thd, reclaimed_space, FALSE))) - sql_print_error("MSYQL_BIN_LOG::purge_logs failed to process registered files" + sql_print_error("MYSQL_BIN_LOG::purge_logs failed to process registered files" " that would be purged."); close_purge_index_file(); @@ -3983,7 +3983,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *reclaimed_space, if ((error=reinit_io_cache(&purge_index_file, READ_CACHE, 0, 0, 0))) { - sql_print_error("MSYQL_BIN_LOG::purge_index_entry failed to reinit register file " + sql_print_error("MYSQL_BIN_LOG::purge_index_entry failed to reinit register file " "for read"); goto err; } @@ -3998,7 +3998,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *reclaimed_space, if (purge_index_file.error) { error= purge_index_file.error; - sql_print_error("MSYQL_BIN_LOG::purge_index_entry error %d reading from " + sql_print_error("MYSQL_BIN_LOG::purge_index_entry error %d reading from " "register file.", error); goto err; } From 0a4b508173a0cd32f329df3514bf34c2f2001317 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 1 Dec 2016 20:04:36 +0100 Subject: [PATCH 228/295] MDEV-11242 MariaDB Server releases contains promotion of MariaDB Corporation --- scripts/mysql_install_db.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 5d53b252fd72b..aefcc1a838415 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -512,10 +512,8 @@ then echo "The latest information about MariaDB is available at http://mariadb.org/." echo "You can find additional information about the MySQL part at:" echo "http://dev.mysql.com" - echo "Support MariaDB development by buying support/new features from MariaDB" - echo "Corporation Ab. You can contact us about this at sales@mariadb.com." - echo "Alternatively consider joining our community based development effort:" - echo "http://mariadb.com/kb/en/contributing-to-the-mariadb-project/" + echo "Consider joining MariaDB's strong and vibrant community:" + echo "https://mariadb.org/get-involved/" echo fi From 4a3acbcfd07291ecc141f37200eec79eb1713882 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 2 Dec 2016 00:19:49 +0100 Subject: [PATCH 229/295] MDEV-11241 Certain combining marks cause MariaDB to crash when doing Full-Text searches Don't assume that a word of n bytes can match a word of at most n * charset->mbmaxlen bytes, always go for the worst. --- mysql-test/r/fulltext_charsets.result | 7 +++++++ mysql-test/t/fulltext_charsets.test | 10 ++++++++++ storage/myisam/ft_boolean_search.c | 7 +------ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 mysql-test/r/fulltext_charsets.result create mode 100644 mysql-test/t/fulltext_charsets.test diff --git a/mysql-test/r/fulltext_charsets.result b/mysql-test/r/fulltext_charsets.result new file mode 100644 index 0000000000000..39ce02a3fce09 --- /dev/null +++ b/mysql-test/r/fulltext_charsets.result @@ -0,0 +1,7 @@ +set names utf8mb4; +create table t1 (a int, b text, fulltext (b)) charset=utf8mb4 collate=utf8mb4_unicode_ci; +insert t1 values (1000, 'C͓̙̯͔̩ͅͅi̩̘̜̲a̯̲̬̳̜̖̤o͕͓̜͓̺̖̗,̠̬͚ ̺T͇̲h͈̱e ̬̜DÌ–o̦̖͔̗͖̩̘c̣̼tÌ͉̫̮̗o͉̫̭r̙͎̗.͓̪̥'); +select a from t1 where match(b) against ('ciao' in boolean mode); +a +1000 +drop table t1; diff --git a/mysql-test/t/fulltext_charsets.test b/mysql-test/t/fulltext_charsets.test new file mode 100644 index 0000000000000..3ac9791bd1ac2 --- /dev/null +++ b/mysql-test/t/fulltext_charsets.test @@ -0,0 +1,10 @@ +# +# MDEV-11241 Certain combining marks cause MariaDB to crash when doing Full-Text searches +# +set names utf8mb4; + +create table t1 (a int, b text, fulltext (b)) charset=utf8mb4 collate=utf8mb4_unicode_ci; +insert t1 values (1000, 'C͓̙̯͔̩ͅͅi̩̘̜̲a̯̲̬̳̜̖̤o͕͓̜͓̺̖̗,̠̬͚ ̺T͇̲h͈̱e ̬̜DÌ–o̦̖͔̗͖̩̘c̣̼tÌ͉̫̮̗o͉̫̭r̙͎̗.͓̪̥'); +select a from t1 where match(b) against ('ciao' in boolean mode); +drop table t1; + diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c index b67f1ea6a25fe..16432d0161cda 100644 --- a/storage/myisam/ft_boolean_search.c +++ b/storage/myisam/ft_boolean_search.c @@ -195,12 +195,7 @@ static int ftb_query_add_word(MYSQL_FTPARSER_PARAM *param, switch (info->type) { case FT_TOKEN_WORD: ftbw= (FTB_WORD *)alloc_root(&ftb_param->ftb->mem_root, - sizeof(FTB_WORD) + - (info->trunc ? HA_MAX_KEY_BUFF : - (word_len + 1) * - ftb_param->ftb->charset->mbmaxlen + - HA_FT_WLEN + - ftb_param->ftb->info->s->rec_reflength)); + sizeof(FTB_WORD) + HA_MAX_KEY_BUFF); ftbw->len= word_len + 1; ftbw->flags= 0; ftbw->off= 0; From e4a0d75a0b72cf08ee98ab7f319c011019bd0c0e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 4 Dec 2016 01:35:57 +0100 Subject: [PATCH 230/295] import a test case from percona-server-5.6.34-79.1 --- mysql-test/extra/table_index_statistics.inc | 59 +++++++++++++++++++ .../innodb/r/table_index_statistics.result | 48 +++++++++++++++ .../innodb/t/table_index_statistics.test | 8 +++ 3 files changed, 115 insertions(+) create mode 100644 mysql-test/extra/table_index_statistics.inc create mode 100644 mysql-test/suite/innodb/r/table_index_statistics.result create mode 100644 mysql-test/suite/innodb/t/table_index_statistics.test diff --git a/mysql-test/extra/table_index_statistics.inc b/mysql-test/extra/table_index_statistics.inc new file mode 100644 index 0000000000000..67ee421303726 --- /dev/null +++ b/mysql-test/extra/table_index_statistics.inc @@ -0,0 +1,59 @@ +# include file to test index and table statstics for specific storage engine +# requires includer set the default strorage engine for the session + +# Bug 602047 (wrong rows_read value) + +FLUSH INDEX_STATISTICS; +FLUSH TABLE_STATISTICS; + +SET @userstat_old= @@userstat; +SET GLOBAL userstat=ON; + +CREATE TABLE t1 (id int(10), PRIMARY KEY (id)); +INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +SELECT COUNT(*) FROM t1; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +# Test that FLUSH clears one table but not another + +FLUSH TABLE_STATISTICS; + +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +# Test that FLUSH clears both tables now + +FLUSH INDEX_STATISTICS; + +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +# Test that stats are collected after the FLUSH again + +SELECT COUNT(*) FROM t1; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +DROP TABLE t1; + +# Bug 1183625 (handler::update_global_table_stats crash). + +CREATE TABLE t2 (c1 INT UNSIGNED); + +ALTER TABLE t2 MODIFY c1 FLOAT; + +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; + +DROP TABLE t2; + +# Bug 1183625 (handler::update_global_table_stats crash). + +CREATE TABLE t2 (c1 INT UNSIGNED) ENGINE=InnoDB; + +ALTER TABLE t2 MODIFY c1 FLOAT; + +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; + +DROP TABLE t2; + +SET GLOBAL userstat= @userstat_old; diff --git a/mysql-test/suite/innodb/r/table_index_statistics.result b/mysql-test/suite/innodb/r/table_index_statistics.result new file mode 100644 index 0000000000000..7389a11713ab2 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_index_statistics.result @@ -0,0 +1,48 @@ +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = INNODB; +FLUSH INDEX_STATISTICS; +FLUSH TABLE_STATISTICS; +SET @userstat_old= @@userstat; +SET GLOBAL userstat=ON; +CREATE TABLE t1 (id int(10), PRIMARY KEY (id)); +INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH TABLE_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH INDEX_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +DROP TABLE t1; +CREATE TABLE t2 (c1 INT UNSIGNED); +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +CREATE TABLE t2 (c1 INT UNSIGNED) ENGINE=InnoDB; +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +SET GLOBAL userstat= @userstat_old; +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/mysql-test/suite/innodb/t/table_index_statistics.test b/mysql-test/suite/innodb/t/table_index_statistics.test new file mode 100644 index 0000000000000..af6f194648608 --- /dev/null +++ b/mysql-test/suite/innodb/t/table_index_statistics.test @@ -0,0 +1,8 @@ +--source include/have_innodb.inc + +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = INNODB; + +--source extra/table_index_statistics.inc + +SET SESSION default_storage_engine = @default_storage_engine_old; From 7f2fd345004f9774cd1b6e00c8f67e719660cecc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 2 Dec 2016 14:34:45 +0100 Subject: [PATCH 231/295] MDEV-11231 Server crashes in check_duplicate_key on CREATE TABLE ... SELECT be consistent and don't include the table name into the error message, no other CREATE TABLE error does it. (the crash happened, because thd->lex->query_tables was NULL) --- mysql-test/r/alter_table.result | 26 +- mysql-test/r/check.result | 6 +- mysql-test/r/constraints.result | 4 +- mysql-test/r/create.result | 316 +++++++++--------- mysql-test/r/explain.result | 78 ++--- mysql-test/r/group_min_max.result | 2 +- mysql-test/r/index_merge_myisam.result | 2 +- mysql-test/r/information_schema.result | 4 +- mysql-test/r/innodb_icp.result | 4 +- mysql-test/r/innodb_mysql_sync.result | 4 +- mysql-test/r/key.result | 8 +- mysql-test/r/lowercase_table2.result | 2 +- mysql-test/r/mix2_myisam.result | 4 +- mysql-test/r/mrr_icp_extra.result | 2 +- mysql-test/r/myisam.result | 6 +- mysql-test/r/myisam_icp.result | 4 +- mysql-test/r/partition.result | 2 +- mysql-test/r/partition_innodb.result | 2 +- mysql-test/r/type_ranges.result | 6 +- mysql-test/suite/innodb/r/innodb-index.result | 2 +- mysql-test/suite/innodb/r/innodb.result | 6 +- .../suite/innodb/r/innodb_bug51378.result | 2 +- .../suite/innodb_fts/r/innodb_fts_misc.result | 2 +- .../r/innodb_index_large_prefix.result | 10 +- mysql-test/suite/maria/icp.result | 4 +- mysql-test/suite/maria/maria.result | 6 +- .../optimizer_unfixed_bugs/r/bug45221.result | 8 +- mysql-test/suite/rpl/r/rpl_sp.result | 2 +- mysql-test/t/create.test | 6 + sql/share/errmsg-utf8.txt | 2 +- sql/sql_table.cc | 8 +- .../mysql-test/tokudb/r/cluster_2968-1.result | 8 +- .../mysql-test/tokudb/r/cluster_2968-2.result | 4 +- .../mysql-test/tokudb/r/cluster_2968-3.result | 6 +- .../mysql-test/tokudb/r/type_ranges.result | 6 +- .../r/hcad_tmp_tables_56.result | 2 +- 36 files changed, 286 insertions(+), 280 deletions(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 2e371ac6ae6e3..67de5f3160aca 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -1535,17 +1535,17 @@ ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= DEFAULT; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= COPY; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= INPLACE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i5(b), ALGORITHM= INVALID; ERROR HY000: Unknown ALGORITHM 'INVALID' ALTER TABLE m1 ENABLE KEYS; @@ -1570,17 +1570,17 @@ ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= DEFAULT; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= COPY; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= INPLACE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. SET SESSION old_alter_table= 0; affected rows: 0 ALTER TABLE t1 DROP INDEX i1, DROP INDEX i2, DROP INDEX i3, DROP INDEX i4; @@ -1602,17 +1602,17 @@ ALTER TABLE t1 ADD INDEX i2(b), LOCK= NONE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), LOCK= SHARED; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), LOCK= EXCLUSIVE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i5(b), LOCK= INVALID; ERROR HY000: Unknown LOCK type 'INVALID' ALTER TABLE m1 ENABLE KEYS, LOCK= DEFAULT; @@ -1632,24 +1632,24 @@ ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= INPLACE, LOCK= SHARED; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= INPLACE, LOCK= EXCLUSIVE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= COPY, LOCK= NONE; ERROR 0A000: LOCK=NONE is not supported. Reason: COPY algorithm requires a lock. Try LOCK=SHARED. ALTER TABLE t1 ADD INDEX i5(b), ALGORITHM= COPY, LOCK= SHARED; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i5' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i5`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i6(b), ALGORITHM= COPY, LOCK= EXCLUSIVE; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i6' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i6`. This is deprecated and will be disallowed in a future release. ALTER TABLE m1 ENABLE KEYS, ALGORITHM= INPLACE, LOCK= NONE; ERROR 0A000: LOCK=NONE/SHARED is not supported for this operation. Try LOCK=EXCLUSIVE. ALTER TABLE m1 ENABLE KEYS, ALGORITHM= INPLACE, LOCK= SHARED; diff --git a/mysql-test/r/check.result b/mysql-test/r/check.result index 0ac42272c4f2b..715fe032b53ed 100644 --- a/mysql-test/r/check.result +++ b/mysql-test/r/check.result @@ -2,9 +2,9 @@ drop table if exists t1,t2; drop view if exists v1; create table t1(n int not null, key(n), key(n), key(n), key(n)); Warnings: -Note 1831 Duplicate index 'n_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'n_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'n_4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `n_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `n_3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `n_4`. This is deprecated and will be disallowed in a future release. check table t1 extended; insert into t1 values (200000); Table Op Msg_type Msg_text diff --git a/mysql-test/r/constraints.result b/mysql-test/r/constraints.result index 3bec2c3e16da3..891f12915cbf7 100644 --- a/mysql-test/r/constraints.result +++ b/mysql-test/r/constraints.result @@ -18,10 +18,10 @@ create table t1 (a int null); alter table t1 add constraint constraint_1 unique (a); alter table t1 add constraint unique key_1(a); Warnings: -Note 1831 Duplicate index 'key_1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_1`. This is deprecated and will be disallowed in a future release. alter table t1 add constraint constraint_2 unique key_2(a); Warnings: -Note 1831 Duplicate index 'key_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_2`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index 20120bf7663a0..5ef11e2482029 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -181,36 +181,36 @@ Warnings: Note 1051 Unknown table 'test.t2' create table t1 (a int not null, b int, primary key(a), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b)); Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_5' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_6' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_7' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_8' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_9' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_10' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_11' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_12' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_13' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_14' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_15' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_16' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_17' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_18' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_19' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_20' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_21' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_22' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_23' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_24' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_25' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_26' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_27' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_28' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_29' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_30' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_31' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_4`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_5`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_6`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_7`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_8`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_9`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_10`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_11`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_12`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_13`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_14`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_15`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_16`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_17`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_18`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_19`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_20`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_21`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_22`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_23`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_24`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_25`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_26`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_27`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_28`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_29`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_30`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_31`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -1045,69 +1045,69 @@ key a064_long_123456789_123456789_123456789_123456789_123456789_1234 ( c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16) ); Warnings: -Note 1831 Duplicate index 'a002_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a003_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a004_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a005_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a006_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a007_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a008_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a009_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a010_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a011_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a012_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a013_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a014_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a015_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a016_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a017_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a018_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a019_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a020_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a021_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a022_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a023_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a024_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a025_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a026_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a027_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a028_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a029_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a030_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a031_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a032_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a033_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a034_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a035_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a036_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a037_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a038_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a039_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a040_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a041_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a042_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a043_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a044_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a045_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a046_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a047_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a048_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a049_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a050_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a051_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a052_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a053_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a054_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a055_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a056_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a057_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a058_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a059_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a060_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a061_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a062_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a063_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a064_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a002_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a003_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a004_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a005_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a006_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a007_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a008_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a009_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a010_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a011_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a012_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a013_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a014_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a015_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a016_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a017_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a018_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a019_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a020_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a021_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a022_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a023_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a024_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a025_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a026_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a027_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a028_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a029_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a030_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a031_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a032_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a033_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a034_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a035_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a036_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a037_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a038_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a039_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a040_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a041_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a042_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a043_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a044_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a045_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a046_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a047_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a048_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a049_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a050_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a051_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a052_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a053_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a054_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a055_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a056_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a057_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a058_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a059_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a060_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a061_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a062_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a063_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a064_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -1410,69 +1410,69 @@ c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16), add key a064_long_123456789_123456789_123456789_123456789_123456789_1234 ( c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16); Warnings: -Note 1831 Duplicate index 'a002_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a003_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a004_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a005_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a006_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a007_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a008_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a009_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a010_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a011_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a012_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a013_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a014_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a015_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a016_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a017_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a018_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a019_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a020_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a021_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a022_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a023_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a024_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a025_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a026_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a027_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a028_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a029_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a030_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a031_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a032_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a033_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a034_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a035_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a036_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a037_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a038_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a039_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a040_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a041_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a042_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a043_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a044_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a045_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a046_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a047_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a048_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a049_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a050_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a051_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a052_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a053_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a054_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a055_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a056_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a057_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a058_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a059_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a060_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a061_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a062_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a063_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a064_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a002_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a003_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a004_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a005_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a006_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a007_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a008_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a009_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a010_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a011_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a012_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a013_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a014_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a015_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a016_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a017_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a018_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a019_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a020_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a021_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a022_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a023_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a024_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a025_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a026_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a027_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a028_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a029_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a030_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a031_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a032_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a033_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a034_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a035_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a036_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a037_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a038_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a039_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a040_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a041_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a042_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a043_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a044_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a045_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a046_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a047_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a048_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a049_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a050_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a051_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a052_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a053_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a054_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a055_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a056_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a057_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a058_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a059_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a060_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a061_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a062_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a063_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a064_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -2681,3 +2681,7 @@ drop function f1; End of 5.5 tests create table t1; ERROR 42000: A table must have at least 1 column +create table t1 (i int, j int, key(i), key(i)) as select 1 as i, 2 as j; +Warnings: +Note 1831 Duplicate index `i_2`. This is deprecated and will be disallowed in a future release. +drop table t1; diff --git a/mysql-test/r/explain.result b/mysql-test/r/explain.result index 5678f0f3968e1..b3c33e3559ade 100644 --- a/mysql-test/r/explain.result +++ b/mysql-test/r/explain.result @@ -99,45 +99,45 @@ KEY(b),KEY(b),KEY(b),KEY(b),KEY(b), KEY(b),KEY(b),KEY(b),KEY(b),KEY(b), KEY(b),KEY(b),KEY(b),KEY(b),KEY(b)); Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_3' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_4' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_5' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_6' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_7' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_8' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_9' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_10' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_11' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_12' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_13' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_14' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_15' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_16' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_17' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_18' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_19' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_20' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_21' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_22' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_23' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_24' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_25' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_26' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_27' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_28' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_29' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_30' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_31' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_32' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_33' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_34' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_35' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_36' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_37' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_38' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_39' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_40' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_4`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_5`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_6`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_7`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_8`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_9`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_10`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_11`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_12`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_13`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_14`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_15`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_16`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_17`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_18`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_19`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_20`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_21`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_22`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_23`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_24`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_25`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_26`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_27`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_28`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_29`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_30`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_31`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_32`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_33`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_34`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_35`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_36`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_37`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_38`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_39`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_40`. This is deprecated and will be disallowed in a future release. INSERT INTO t2 VALUES (),(),(); EXPLAIN SELECT 1 FROM (SELECT 1 FROM t2,t1 WHERE b < c GROUP BY 1 LIMIT 1) AS d2; diff --git a/mysql-test/r/group_min_max.result b/mysql-test/r/group_min_max.result index 3fd87425e2390..25cd4a252795e 100644 --- a/mysql-test/r/group_min_max.result +++ b/mysql-test/r/group_min_max.result @@ -2562,7 +2562,7 @@ a MIN(b) MAX(b) AVG(b) DROP TABLE t1; create table t1 (a int, b int, key (a,b), key `index` (a,b)) engine=MyISAM; Warnings: -Note 1831 Duplicate index 'index' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `index`. This is deprecated and will be disallowed in a future release. insert into t1 (a,b) values (0,0),(0,1),(0,2),(0,3),(0,4),(0,5),(0,6), (0,7),(0,8),(0,9),(0,10),(0,11),(0,12),(0,13), diff --git a/mysql-test/r/index_merge_myisam.result b/mysql-test/r/index_merge_myisam.result index c63ed13266255..b3beff5a96779 100644 --- a/mysql-test/r/index_merge_myisam.result +++ b/mysql-test/r/index_merge_myisam.result @@ -224,7 +224,7 @@ index i2_1(key2, key2_1), index i2_2(key2, key2_1) ); Warnings: -Note 1831 Duplicate index 'i2_2' defined on the table 'test.t4'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2_2`. This is deprecated and will be disallowed in a future release. insert into t4 select key1,key1,key1 div 10, key1 % 10, key1 % 10, key1 from t0; select * from t4 where key1a = 3 or key1b = 4; key1a key1b key2 key2_1 key2_2 key3 diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index ef503cef99e8c..21bb6abd7bda8 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -478,10 +478,10 @@ create table t1 (a int null, primary key(a)); alter table t1 add constraint constraint_1 unique (a); alter table t1 add constraint unique key_1(a); Warnings: -Note 1831 Duplicate index 'key_1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_1`. This is deprecated and will be disallowed in a future release. alter table t1 add constraint constraint_2 unique key_2(a); Warnings: -Note 1831 Duplicate index 'key_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_2`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/r/innodb_icp.result b/mysql-test/r/innodb_icp.result index fb467494525cd..33a8027adf1c8 100644 --- a/mysql-test/r/innodb_icp.result +++ b/mysql-test/r/innodb_icp.result @@ -712,8 +712,8 @@ b INT, c INT, d DATE NOT NULL, e VARCHAR(1), KEY (c), KEY (d), KEY k2(b), KEY k3(b), KEY k4(b) ); Warnings: -Note 1831 Duplicate index 'k3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'k4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k4`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 (b,c,d,e) VALUES (6,5,'2006-05-25','y'),(1,5,'2008-01-23','t'), (6,5,'2007-06-18','d'),(4,5,'1900-01-01','r'), diff --git a/mysql-test/r/innodb_mysql_sync.result b/mysql-test/r/innodb_mysql_sync.result index 478b010981636..cfe855fbb76a9 100644 --- a/mysql-test/r/innodb_mysql_sync.result +++ b/mysql-test/r/innodb_mysql_sync.result @@ -308,7 +308,7 @@ SET DEBUG_SYNC= 'now SIGNAL continue3'; # Connection default # Reaping ALTER TABLE ... Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. SET DEBUG_SYNC= 'RESET'; DELETE FROM t1 WHERE a= 3; # @@ -358,7 +358,7 @@ SET DEBUG_SYNC= 'now SIGNAL continue4'; # Connection default # Reaping ALTER TABLE ... Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. SET DEBUG_SYNC= 'RESET'; DROP TABLE t1; SET DEBUG_SYNC= 'RESET'; diff --git a/mysql-test/r/key.result b/mysql-test/r/key.result index a89a79dadbbff..0616a1b0d0a37 100644 --- a/mysql-test/r/key.result +++ b/mysql-test/r/key.result @@ -428,7 +428,7 @@ index i5 (c1, c2, c3, c4), primary key (c2, c3), index (c2, c4)); Warnings: -Note 1831 Duplicate index 'i1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i1`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -448,17 +448,17 @@ t1 CREATE TABLE `t1` ( alter table t1 drop index c1; alter table t1 add index (c1); Warnings: -Note 1831 Duplicate index 'c1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `c1`. This is deprecated and will be disallowed in a future release. alter table t1 add index (c1); Warnings: -Note 1831 Duplicate index 'c1_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `c1_2`. This is deprecated and will be disallowed in a future release. alter table t1 drop index i3; alter table t1 add index i3 (c3); alter table t1 drop index i2, drop index i4; alter table t1 add index i2 (c2), add index i4 (c4); alter table t1 drop index i2, drop index i4, add index i6 (c2, c4); Warnings: -Note 1831 Duplicate index 'i6' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i6`. This is deprecated and will be disallowed in a future release. alter table t1 add index i2 (c2), add index i4 (c4), drop index i6; alter table t1 drop index i2, drop index i4, add unique i4 (c4); alter table t1 add index i2 (c2), drop index i4, add index i4 (c4); diff --git a/mysql-test/r/lowercase_table2.result b/mysql-test/r/lowercase_table2.result index 7491334907e16..1130e782291ed 100644 --- a/mysql-test/r/lowercase_table2.result +++ b/mysql-test/r/lowercase_table2.result @@ -138,7 +138,7 @@ Tables_in_test (T1%) T1 alter table t1 add index (A); Warnings: -Note 1831 Duplicate index 'A_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `A_2`. This is deprecated and will be disallowed in a future release. show tables like 't1%'; Tables_in_test (t1%) t1 diff --git a/mysql-test/r/mix2_myisam.result b/mysql-test/r/mix2_myisam.result index e824db8928a0a..8aacbf060bd60 100644 --- a/mysql-test/r/mix2_myisam.result +++ b/mysql-test/r/mix2_myisam.result @@ -257,7 +257,7 @@ drop table t1; CREATE TABLE t1 (a int not null, b int not null,c int not null, key(a),primary key(a,b), unique(c),key(a),unique(b)) ENGINE = MyISAM; Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. show index from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t1 0 PRIMARY 1 a A # NULL NULL BTREE @@ -1550,7 +1550,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* diff --git a/mysql-test/r/mrr_icp_extra.result b/mysql-test/r/mrr_icp_extra.result index 855ac4beab573..eaabd3d81195c 100644 --- a/mysql-test/r/mrr_icp_extra.result +++ b/mysql-test/r/mrr_icp_extra.result @@ -351,7 +351,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* diff --git a/mysql-test/r/myisam.result b/mysql-test/r/myisam.result index 5506e4e419c87..2a52b25b97fb0 100644 --- a/mysql-test/r/myisam.result +++ b/mysql-test/r/myisam.result @@ -1243,7 +1243,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* @@ -2442,8 +2442,8 @@ SET myisam_repair_threads=2; SET myisam_sort_buffer_size=4096; CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a)); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_3`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3); REPAIR TABLE t1; Table Op Msg_type Msg_text diff --git a/mysql-test/r/myisam_icp.result b/mysql-test/r/myisam_icp.result index 49cba60320d62..95befeb5a42c0 100644 --- a/mysql-test/r/myisam_icp.result +++ b/mysql-test/r/myisam_icp.result @@ -716,8 +716,8 @@ b INT, c INT, d DATE NOT NULL, e VARCHAR(1), KEY (c), KEY (d), KEY k2(b), KEY k3(b), KEY k4(b) ); Warnings: -Note 1831 Duplicate index 'k3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'k4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k4`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 (b,c,d,e) VALUES (6,5,'2006-05-25','y'),(1,5,'2008-01-23','t'), (6,5,'2007-06-18','d'),(4,5,'1900-01-01','r'), diff --git a/mysql-test/r/partition.result b/mysql-test/r/partition.result index 09b22761bcb7b..d4b719f3a7580 100644 --- a/mysql-test/r/partition.result +++ b/mysql-test/r/partition.result @@ -300,7 +300,7 @@ create index i on t1 (a); ERROR 42000: Duplicate key name 'i' create index i2 on t1 (a); Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. drop table t1; CREATE TABLE t1 (a INT, FOREIGN KEY (a) REFERENCES t0 (a)) ENGINE=MyISAM diff --git a/mysql-test/r/partition_innodb.result b/mysql-test/r/partition_innodb.result index f1c3795791ac5..e1bc075f14a5c 100644 --- a/mysql-test/r/partition_innodb.result +++ b/mysql-test/r/partition_innodb.result @@ -609,7 +609,7 @@ c INT, PRIMARY KEY (c,a), KEY (a),KEY (a) ) ENGINE=INNODB PARTITION BY KEY () PARTITIONS 2; Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (1,5,1),(2,4,1),(3,3,1),(4,2,1),(5,1,1); UPDATE t1 SET b = 0, c=1 WHERE a <=>0; SELECT * FROM t1; diff --git a/mysql-test/r/type_ranges.result b/mysql-test/r/type_ranges.result index 53fc31046661a..ad52ab5da7d99 100644 --- a/mysql-test/r/type_ranges.result +++ b/mysql-test/r/type_ranges.result @@ -86,10 +86,10 @@ t1 1 options 2 flags A NULL NULL NULL BTREE CREATE UNIQUE INDEX test on t1 ( auto ) ; CREATE INDEX test2 on t1 ( ulonglong,ulong) ; Warnings: -Note 1831 Duplicate index 'test2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test2`. This is deprecated and will be disallowed in a future release. CREATE INDEX test3 on t1 ( medium ) ; Warnings: -Note 1831 Duplicate index 'test3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test3`. This is deprecated and will be disallowed in a future release. DROP INDEX test ON t1; insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one'); insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one'); @@ -308,7 +308,7 @@ const int(1) NULL NO 0 # drop table t1,t2,t3; create table t1 ( myfield INT NOT NULL, UNIQUE INDEX (myfield), unique (myfield), index(myfield)); Warnings: -Note 1831 Duplicate index 'myfield_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `myfield_2`. This is deprecated and will be disallowed in a future release. drop table t1; create table t1 ( id integer unsigned not null primary key ); create table t2 ( id integer unsigned not null primary key ); diff --git a/mysql-test/suite/innodb/r/innodb-index.result b/mysql-test/suite/innodb/r/innodb-index.result index 996bc45f7e25d..3d5a0f840c1ac 100644 --- a/mysql-test/suite/innodb/r/innodb-index.result +++ b/mysql-test/suite/innodb/r/innodb-index.result @@ -64,7 +64,7 @@ alter table t1 add unique index (c), add index (d); affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'd' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `d`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/suite/innodb/r/innodb.result b/mysql-test/suite/innodb/r/innodb.result index f8020e080e678..cded1f98fb380 100644 --- a/mysql-test/suite/innodb/r/innodb.result +++ b/mysql-test/suite/innodb/r/innodb.result @@ -405,7 +405,7 @@ drop table t1; CREATE TABLE t1 (a int not null, b int not null,c int not null, key(a),primary key(a,b), unique(c),key(a),unique(b)); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. show index from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t1 0 PRIMARY 1 a A # NULL NULL BTREE @@ -1445,7 +1445,7 @@ t2 CREATE TABLE `t2` ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1 create index id2 on t2 (id); Warnings: -Note 1831 Duplicate index 'id2' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `id2`. This is deprecated and will be disallowed in a future release. show create table t2; Table Create Table t2 CREATE TABLE `t2` ( @@ -1851,7 +1851,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* diff --git a/mysql-test/suite/innodb/r/innodb_bug51378.result b/mysql-test/suite/innodb/r/innodb_bug51378.result index a2a9bcad5648e..08b6b656085d2 100644 --- a/mysql-test/suite/innodb/r/innodb_bug51378.result +++ b/mysql-test/suite/innodb/r/innodb_bug51378.result @@ -5,7 +5,7 @@ col3 time not null) engine = innodb; create unique index idx on bug51378(col1, col2(31)); alter table bug51378 add unique index idx2(col1, col2(31)); Warnings: -Note 1831 Duplicate index 'idx2' defined on the table 'test.bug51378'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx2`. This is deprecated and will be disallowed in a future release. create unique index idx3 on bug51378(col1, col3); SHOW CREATE TABLE bug51378; Table Create Table diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result b/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result index f6be36a24ef94..628fe8519ed64 100644 --- a/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result +++ b/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result @@ -1018,7 +1018,7 @@ CREATE TABLE `t21` (`a` text, `b` int not null, fulltext key (`a`), fulltext key (`a`) ) ENGINE=INNODB DEFAULT CHARSET=LATIN1; Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t21'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. ALTER TABLE `t21` ADD UNIQUE INDEX (`b`), ALGORITHM=INPLACE; ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: InnoDB presently supports one FULLTEXT index creation at a time. Try ALGORITHM=COPY. ALTER TABLE `t21` ADD UNIQUE INDEX (`b`); diff --git a/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result b/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result index b610d1ed7b857..a02a4acb7408f 100644 --- a/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result +++ b/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result @@ -193,10 +193,10 @@ Level Code Message Warning 1071 Specified key was too long; max key length is 3072 bytes create index idx3 on worklog5743_8(a2(3072)); Warnings: -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_8'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. show warnings; Level Code Message -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_8'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. create index idx4 on worklog5743_8(a1, a2(3069)); ERROR 42000: Specified key was too long; max key length is 3072 bytes show warnings; @@ -229,10 +229,10 @@ Level Code Message Warning 1071 Specified key was too long; max key length is 3072 bytes create index idx3 on worklog5743_16(a2(3072)); Warnings: -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_16'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. show warnings; Level Code Message -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_16'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. create index idx4 on worklog5743_16(a1, a2(3069)); ERROR 42000: Specified key was too long; max key length is 3072 bytes show warnings; @@ -462,7 +462,7 @@ Warnings: Warning 1071 Specified key was too long; max key length is 3072 bytes create index idx2 on worklog5743(a(3072)); Warnings: -Note 1831 Duplicate index 'idx2' defined on the table 'test.worklog5743'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx2`. This is deprecated and will be disallowed in a future release. show create table worklog5743; Table Create Table worklog5743 CREATE TABLE `worklog5743` ( diff --git a/mysql-test/suite/maria/icp.result b/mysql-test/suite/maria/icp.result index 85a21ada47dd3..1c11072cbde23 100644 --- a/mysql-test/suite/maria/icp.result +++ b/mysql-test/suite/maria/icp.result @@ -718,8 +718,8 @@ b INT, c INT, d DATE NOT NULL, e VARCHAR(1), KEY (c), KEY (d), KEY k2(b), KEY k3(b), KEY k4(b) ); Warnings: -Note 1831 Duplicate index 'k3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'k4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k4`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 (b,c,d,e) VALUES (6,5,'2006-05-25','y'),(1,5,'2008-01-23','t'), (6,5,'2007-06-18','d'),(4,5,'1900-01-01','r'), diff --git a/mysql-test/suite/maria/maria.result b/mysql-test/suite/maria/maria.result index 952cdc09c903f..d410944ddb291 100644 --- a/mysql-test/suite/maria/maria.result +++ b/mysql-test/suite/maria/maria.result @@ -1139,7 +1139,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* @@ -2696,8 +2696,8 @@ SET aria_repair_threads=2; SET aria_sort_buffer_size=8192; CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a)); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_3`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3); REPAIR TABLE t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result b/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result index 470d06c341d02..92918959cda05 100644 --- a/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result +++ b/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result @@ -8,7 +8,7 @@ KEY `int_key` (`int_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=30 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO `t2` VALUES (10,8,NULL,'2002-02-26 06:14:37'),(11,9,'2006-06-14','1900-01-01 00:00:00'),(12,9,'2002-09-12','2006-12-03 09:37:26'),(13,186,'2005-02-15','2008-05-26 12:27:10'),(14,NULL,NULL,'2004-12-14 16:37:30'),(15,2,'2008-11-04','2003-02-11 21:19:41'),(16,3,'2004-09-04','2009-10-18 02:27:49'),(17,0,'2006-06-05','2000-09-26 07:45:57'),(18,133,'1900-01-01',NULL),(19,1,'1900-01-01','2005-11-10 12:40:29'),(20,8,'1900-01-01','2009-04-25 00:00:00'),(21,5,'2005-01-13','2002-11-27 00:00:00'),(22,5,'2006-05-21','2004-01-26 20:32:32'),(23,8,'2003-09-08','2007-10-26 11:41:40'),(24,6,'2006-12-23','2005-10-07 00:00:00'),(25,51,'2006-10-15','2000-07-15 05:00:34'),(26,4,'2005-04-06','2000-04-03 16:33:32'),(27,7,'2008-04-07',NULL),(28,6,'2006-10-10','2001-04-25 01:26:12'),(29,4,'1900-01-01','2000-12-27 00:00:00'); CREATE TABLE t1 ( `pk` int(11) NOT NULL AUTO_INCREMENT, @@ -20,7 +20,7 @@ KEY `int_key` (`int_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=21 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (1,2,NULL,'2004-10-11 18:13:16'),(2,9,'2001-09-19',NULL),(3,3,'2004-09-12','1900-01-01 00:00:00'),(4,9,NULL,'2009-07-25 00:00:00'),(5,NULL,'2002-07-19',NULL),(6,9,'2002-12-16','2008-07-27 00:00:00'),(7,3,'2006-02-08','2002-11-13 16:37:31'),(8,8,'2006-08-28','1900-01-01 00:00:00'),(9,8,'2001-04-14','2003-12-10 00:00:00'),(10,53,'2000-01-05','2001-12-21 22:38:22'),(11,0,'2003-12-06','2008-12-13 23:16:44'),(12,5,'1900-01-01','2005-08-15 12:39:41'),(13,166,'2002-11-27',NULL),(14,3,NULL,'2006-09-11 12:06:14'),(15,0,'2003-05-27','2007-12-15 12:39:34'),(16,1,'2005-05-03','2005-08-09 00:00:00'),(17,9,'2001-04-18','2001-09-02 22:50:02'),(18,5,'2005-12-27','2005-12-16 22:58:11'),(19,6,'2004-08-20','2007-04-19 00:19:53'),(20,2,'1900-01-01','1900-01-01 00:00:00'); SELECT `pk` FROM t1 OUTR @@ -77,7 +77,7 @@ KEY `datetime_key` (`datetime_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=30 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO `t2` VALUES (10,7,8,NULL,'2002-02-26 06:14:37','2002-02-26 06:14:37'),(11,1,9,'2006-06-14','1900-01-01 00:00:00','1900-01-01 00:00:00'),(12,5,9,'2002-09-12','2006-12-03 09:37:26','2006-12-03 09:37:26'),(13,3,186,'2005-02-15','2008-05-26 12:27:10','2008-05-26 12:27:10'),(14,6,NULL,NULL,'2004-12-14 16:37:30','2004-12-14 16:37:30'),(15,92,2,'2008-11-04','2003-02-11 21:19:41','2003-02-11 21:19:41'),(16,7,3,'2004-09-04','2009-10-18 02:27:49','2009-10-18 02:27:49'),(17,NULL,0,'2006-06-05','2000-09-26 07:45:57','2000-09-26 07:45:57'),(18,3,133,'1900-01-01',NULL,NULL),(19,5,1,'1900-01-01','2005-11-10 12:40:29','2005-11-10 12:40:29'),(20,1,8,'1900-01-01','2009-04-25 00:00:00','2009-04-25 00:00:00'),(21,2,5,'2005-01-13','2002-11-27 00:00:00','2002-11-27 00:00:00'),(22,NULL,5,'2006-05-21','2004-01-26 20:32:32','2004-01-26 20:32:32'),(23,1,8,'2003-09-08','2007-10-26 11:41:40','2007-10-26 11:41:40'),(24,0,6,'2006-12-23','2005-10-07 00:00:00','2005-10-07 00:00:00'),(25,210,51,'2006-10-15','2000-07-15 05:00:34','2000-07-15 05:00:34'),(26,8,4,'2005-04-06','2000-04-03 16:33:32','2000-04-03 16:33:32'),(27,7,7,'2008-04-07',NULL,NULL),(28,5,6,'2006-10-10','2001-04-25 01:26:12','2001-04-25 01:26:12'),(29,NULL,4,'1900-01-01','2000-12-27 00:00:00','2000-12-27 00:00:00'); CREATE TABLE t1 ( `pk` int(11) NOT NULL AUTO_INCREMENT, @@ -92,7 +92,7 @@ KEY `datetime_key` (`datetime_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=21 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (1,NULL,2,NULL,'2004-10-11 18:13:16','2004-10-11 18:13:16'),(2,7,9,'2001-09-19',NULL,NULL),(3,9,3,'2004-09-12','1900-01-01 00:00:00','1900-01-01 00:00:00'),(4,7,9,NULL,'2009-07-25 00:00:00','2009-07-25 00:00:00'),(5,4,NULL,'2002-07-19',NULL,NULL),(6,2,9,'2002-12-16','2008-07-27 00:00:00','2008-07-27 00:00:00'),(7,6,3,'2006-02-08','2002-11-13 16:37:31','2002-11-13 16:37:31'),(8,8,8,'2006-08-28','1900-01-01 00:00:00','1900-01-01 00:00:00'),(9,NULL,8,'2001-04-14','2003-12-10 00:00:00','2003-12-10 00:00:00'),(10,5,53,'2000-01-05','2001-12-21 22:38:22','2001-12-21 22:38:22'),(11,NULL,0,'2003-12-06','2008-12-13 23:16:44','2008-12-13 23:16:44'),(12,6,5,'1900-01-01','2005-08-15 12:39:41','2005-08-15 12:39:41'),(13,188,166,'2002-11-27',NULL,NULL),(14,2,3,NULL,'2006-09-11 12:06:14','2006-09-11 12:06:14'),(15,1,0,'2003-05-27','2007-12-15 12:39:34','2007-12-15 12:39:34'),(16,1,1,'2005-05-03','2005-08-09 00:00:00','2005-08-09 00:00:00'),(17,0,9,'2001-04-18','2001-09-02 22:50:02','2001-09-02 22:50:02'),(18,9,5,'2005-12-27','2005-12-16 22:58:11','2005-12-16 22:58:11'),(19,NULL,6,'2004-08-20','2007-04-19 00:19:53','2007-04-19 00:19:53'),(20,4,2,'1900-01-01','1900-01-01 00:00:00','1900-01-01 00:00:00'); SELECT OUTR . `pk` AS X FROM t1 AS OUTR diff --git a/mysql-test/suite/rpl/r/rpl_sp.result b/mysql-test/suite/rpl/r/rpl_sp.result index 8ca7267207362..b7b90cd159450 100644 --- a/mysql-test/suite/rpl/r/rpl_sp.result +++ b/mysql-test/suite/rpl/r/rpl_sp.result @@ -253,7 +253,7 @@ end delete from t2; alter table t2 add unique (a); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'mysqltest1.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. drop function fn1; create function fn1(x int) returns int diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test index ded9096900cd6..999a50f3acb3f 100644 --- a/mysql-test/t/create.test +++ b/mysql-test/t/create.test @@ -2088,3 +2088,9 @@ drop function f1; # --error ER_TABLE_MUST_HAVE_COLUMNS create table t1; + +# +# MDEV-11231 Server crashes in check_duplicate_key on CREATE TABLE ... SELECT +# +create table t1 (i int, j int, key(i), key(i)) as select 1 as i, 2 as j; +drop table t1; diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 627f54631a3fe..d8c64a9d534f3 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -6783,7 +6783,7 @@ ER_FK_COLUMN_NOT_NULL ger "Spalte '%-.192s' kann nicht NOT NULL sein: wird für eine Fremdschlüsselbeschränkung '%-.192s' SET NULL benötigt" ER_DUP_INDEX - eng "Duplicate index '%-.64s' defined on the table '%-.64s.%-.64s'. This is deprecated and will be disallowed in a future release." + eng "Duplicate index %`s. This is deprecated and will be disallowed in a future release." ER_FK_COLUMN_CANNOT_CHANGE eng "Cannot change column '%-.192s': used in a foreign key constraint '%-.192s'" diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 5d4c551d7306f..3b104b55de2c2 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3120,8 +3120,7 @@ void promote_first_timestamp_column(List *column_definitions) @param key_info Key meta-data info. @param key_list List of existing keys. */ -static void check_duplicate_key(THD *thd, - Key *key, KEY *key_info, +static void check_duplicate_key(THD *thd, Key *key, KEY *key_info, List *key_list) { /* @@ -3183,14 +3182,11 @@ static void check_duplicate_key(THD *thd, // Report a warning if we have two identical keys. - DBUG_ASSERT(thd->lex->query_tables->alias); if (all_columns_are_identical) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_DUP_INDEX, ER(ER_DUP_INDEX), - key_info->name, - thd->lex->query_tables->db, - thd->lex->query_tables->alias); + key_info->name); break; } } diff --git a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result index f79b7aa05aa76..0108e33c71050 100644 --- a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result +++ b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result @@ -1045,10 +1045,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t ref b b 5 test.s.b 1 alter table s add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. alter table t add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( @@ -1095,10 +1095,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t ref b_2 b_2 5 test.s.b 1 Using index alter table s add key(b); Warnings: -Note 1831 Duplicate index 'b' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b`. This is deprecated and will be disallowed in a future release. alter table t add key(b); Warnings: -Note 1831 Duplicate index 'b' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( diff --git a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result index a935574ba7f29..b14fb676ed20b 100644 --- a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result +++ b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result @@ -1069,10 +1069,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t ref b,b_2 b_2 5 test.s.b 1 Using index alter table s add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_3' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. alter table t add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_3' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( diff --git a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result index 85a318b71631f..89641d25239ed 100644 --- a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result +++ b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result @@ -1066,13 +1066,13 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE u ref c c 5 test.s.c 1 alter table s add key (b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. alter table t add key (b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. alter table u add key (c) clustering=yes; Warnings: -Note 1831 Duplicate index 'c_2' defined on the table 'test.u'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `c_2`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( diff --git a/storage/tokudb/mysql-test/tokudb/r/type_ranges.result b/storage/tokudb/mysql-test/tokudb/r/type_ranges.result index c1656a663e115..51b9acb9f7afb 100644 --- a/storage/tokudb/mysql-test/tokudb/r/type_ranges.result +++ b/storage/tokudb/mysql-test/tokudb/r/type_ranges.result @@ -87,10 +87,10 @@ t1 1 options 2 flags A NA NULL NULL BTREE CREATE UNIQUE INDEX test on t1 ( auto ) ; CREATE INDEX test2 on t1 ( ulonglong,ulong) ; Warnings: -Note 1831 Duplicate index 'test2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test2`. This is deprecated and will be disallowed in a future release. CREATE INDEX test3 on t1 ( medium ) ; Warnings: -Note 1831 Duplicate index 'test3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test3`. This is deprecated and will be disallowed in a future release. DROP INDEX test ON t1; insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one'); insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one'); @@ -309,7 +309,7 @@ const int(1) NULL NO 0 # drop table t1,t2,t3; create table t1 ( myfield INT NOT NULL, UNIQUE INDEX (myfield), unique (myfield), index(myfield)); Warnings: -Note 1831 Duplicate index 'myfield_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `myfield_2`. This is deprecated and will be disallowed in a future release. drop table t1; create table t1 ( id integer unsigned not null primary key ); create table t2 ( id integer unsigned not null primary key ); diff --git a/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result b/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result index 0ebc7b3d8748b..ff03e9c81abe5 100644 --- a/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result +++ b/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result @@ -5,7 +5,7 @@ create temporary table bar (a int, key(a))engine=TOkuDB; alter table bar add column c int default 0; create index blah on bar(a); Warnings: -Note 1831 Duplicate index 'blah' defined on the table 'test.bar'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `blah`. This is deprecated and will be disallowed in a future release. drop index a on bar; set session tokudb_disable_slow_alter=OFF; insert into bar (a) values (1),(2),(3); From b0754ad45e6a7e23b352099ebfe01e4f35cfad79 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 5 Dec 2016 03:11:42 +0200 Subject: [PATCH 232/295] Follow-up for MDEV-11429 - fix result files for embedded and 32-bit tests --- mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff | 2 +- mysql-test/suite/sys_vars/r/sysvars_server_embedded.result | 2 +- .../suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff index 6d6dcda7bae5f..13fa9b8e2b404 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff @@ -1151,7 +1151,7 @@ +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 1 - NUMERIC_MAX_VALUE 524288 + NUMERIC_MAX_VALUE 1048576 @@ -3851,7 +3851,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index ae61832eb0a15..dfac296655857 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -3840,7 +3840,7 @@ VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 1 -NUMERIC_MAX_VALUE 524288 +NUMERIC_MAX_VALUE 1048576 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff index 6661ea288fda8..2246560d45a22 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff @@ -1187,7 +1187,7 @@ +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 1 - NUMERIC_MAX_VALUE 524288 + NUMERIC_MAX_VALUE 1048576 @@ -4551,7 +4551,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 From ead6d0de0280b2e2f5b3f4daedc82d3750a41740 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 5 Dec 2016 03:13:28 +0200 Subject: [PATCH 233/295] Follow-up for MDEV-9451 - fix XtraDB rdiff files --- mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff | 2 +- mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff index ad4d44086ef24..597ea055bbbda 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff @@ -107,7 +107,7 @@ +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN -+VARIABLE_COMMENT Depricated. This option is temporary alias of --innodb-numa-interleave. ++VARIABLE_COMMENT Deprecated. This option has no effect and will be removed in MariaDB 10.2.3. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff index 5bcb144d9a758..6f931bdb45e03 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,xtradb.rdiff @@ -32,7 +32,7 @@ +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN -+VARIABLE_COMMENT Depricated. This option is temporary alias of --innodb-numa-interleave. ++VARIABLE_COMMENT Deprecated. This option has no effect and will be removed in MariaDB 10.2.3. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL From 02d153c7b9739ce4c2445805aeaf4b185c6ac6f0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 26 Jun 2016 13:37:27 +0200 Subject: [PATCH 234/295] str2decimal: don't return a negative zero --- mysql-test/r/type_decimal.result | 5 +++-- mysql-test/t/type_decimal.test | 5 +++++ storage/tokudb/mysql-test/tokudb/r/type_decimal.result | 2 -- strings/decimal.c | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/type_decimal.result b/mysql-test/r/type_decimal.result index cde8816dee4b1..7ff74c8270a36 100644 --- a/mysql-test/r/type_decimal.result +++ b/mysql-test/r/type_decimal.result @@ -221,7 +221,6 @@ drop table t1; create table t1 (a decimal(10,2) unsigned); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: @@ -280,7 +279,6 @@ drop table t1; create table t1 (a decimal(10,2) zerofill); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: @@ -1012,6 +1010,9 @@ SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE COLUMN_NAME DATA_TYPE COLUMN_TYPE a decimal decimal(10,2)/*old*/ DROP TABLE t1dec102; +select cast('-0.0' as decimal(5,1)) < 0; +cast('-0.0' as decimal(5,1)) < 0 +0 # # End of 5.5 tests # diff --git a/mysql-test/t/type_decimal.test b/mysql-test/t/type_decimal.test index 659e75270caa8..834fd0c53276b 100644 --- a/mysql-test/t/type_decimal.test +++ b/mysql-test/t/type_decimal.test @@ -604,6 +604,11 @@ SHOW COLUMNS FROM t1dec102; SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='t1dec102'; DROP TABLE t1dec102; +# +# MDEV-10552 equality operation on cast of the value "-0.0" to decimal not working +# +select cast('-0.0' as decimal(5,1)) < 0; + --echo # --echo # End of 5.5 tests --echo # diff --git a/storage/tokudb/mysql-test/tokudb/r/type_decimal.result b/storage/tokudb/mysql-test/tokudb/r/type_decimal.result index 5d271bc73fa3d..4540c24238b60 100644 --- a/storage/tokudb/mysql-test/tokudb/r/type_decimal.result +++ b/storage/tokudb/mysql-test/tokudb/r/type_decimal.result @@ -222,7 +222,6 @@ drop table t1; create table t1 (a decimal(10,2) unsigned); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: @@ -281,7 +280,6 @@ drop table t1; create table t1 (a decimal(10,2) zerofill); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: diff --git a/strings/decimal.c b/strings/decimal.c index b0c57d3db0c9f..3d90a58ea8a49 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -928,6 +928,8 @@ internal_str2dec(const char *from, decimal_t *to, char **end, my_bool fixed) error= decimal_shift(to, (int) exponent); } } + if (to->sign && decimal_is_zero(to)) + to->sign= 0; return error; fatal_error: From 9199d727598d60e2e56cebaadb74f4fb042cbcd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 5 Dec 2016 15:25:59 +0200 Subject: [PATCH 235/295] MDEV-11233 CREATE FULLTEXT INDEX with a token longer than 127 bytes crashes server This bug is the result of merging the Oracle MySQL follow-up fix BUG#22963169 MYSQL CRASHES ON CREATE FULLTEXT INDEX without merging the base bug fix: Bug#79475 Insert a token of 84 4-bytes chars into fts index causes server crash. Unlike the above mentioned fixes in MySQL, our fix will not change the storage format of fulltext indexes in InnoDB or XtraDB when a character encoding with mbmaxlen=2 or mbmaxlen=3 and the length of a word is between 128 and 84*mbmaxlen bytes. The Oracle fix would allocate 2 length bytes for these cases. Compatibility with other MySQL and MariaDB releases is ensured by persisting the used maximum length in the SYS_COLUMNS table in the InnoDB data dictionary. This fix also removes some unnecessary strcmp() calls when checking for the legacy default collation my_charset_latin1 (my_charset_latin1.name=="latin1_swedish_ci"). fts_create_one_index_table(): Store the actual length in bytes. This metadata will be written to the SYS_COLUMNS table. fts_zip_initialize(): Initialize only the first byte of the buffer. Actually the code should not even care about this first byte, because the length is set as 0. FTX_MAX_WORD_LEN: Define as HA_FT_MAXCHARLEN * 4 aka 336 bytes, not as 254 bytes. row_merge_create_fts_sort_index(): Set the actual maximum length of the column in bytes, similar to fts_create_one_index_table(). row_merge_fts_doc_tokenize(): Remove the redundant parameter word_dtype. Use the actual maximum length of the column. Calculate the extra_size in the same way as row_merge_buf_encode() does. --- mysql-test/suite/innodb_fts/r/create.result | 168 ++++++++++++++++++++ mysql-test/suite/innodb_fts/t/create.opt | 1 + mysql-test/suite/innodb_fts/t/create.test | 92 +++++++++++ storage/innobase/fts/fts0fts.cc | 16 +- storage/innobase/fts/fts0opt.cc | 3 +- storage/innobase/handler/ha_innodb.cc | 11 +- storage/innobase/include/fts0fts.h | 5 +- storage/innobase/row/row0ftsort.cc | 71 ++++----- storage/xtradb/fts/fts0fts.cc | 16 +- storage/xtradb/fts/fts0opt.cc | 3 +- storage/xtradb/handler/ha_innodb.cc | 11 +- storage/xtradb/include/fts0fts.h | 5 +- storage/xtradb/row/row0ftsort.cc | 71 ++++----- 13 files changed, 365 insertions(+), 108 deletions(-) create mode 100644 mysql-test/suite/innodb_fts/r/create.result create mode 100644 mysql-test/suite/innodb_fts/t/create.opt create mode 100644 mysql-test/suite/innodb_fts/t/create.test diff --git a/mysql-test/suite/innodb_fts/r/create.result b/mysql-test/suite/innodb_fts/r/create.result new file mode 100644 index 0000000000000..c537aa81efd26 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/create.result @@ -0,0 +1,168 @@ +SET NAMES utf8mb4; +# +# MDEV-11233 CREATE FULLTEXT INDEX with a token +# longer than 127 bytes crashes server +# +CREATE TABLE t(t TEXT CHARACTER SET utf8mb3) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +Warnings: +Warning 124 InnoDB rebuilding table to add column FTS_DOC_ID +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +# The data below is not 3-byte UTF-8, but 4-byte chars. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +Warnings: +Warning 1366 Incorrect string value: '\xF0\x9F\x96\x95\xF0\x9F...' for column 't' at row 1 +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +Warnings: +Warning 1366 Incorrect string value: '\xF0\x9F\x96\x96\xF0\x9F...' for column 't' at row 1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST +(REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +COUNT(*) +0 +SELECT * FROM t; +t +็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้ +ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ +beforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbefor +BeforeTheIndexCreation +this was inserted after creating the index +１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１ +afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文 +???????????????????????????????????????????????????????????????????????????????????? +????????????????????????????????????????????????????????????????????????????????????? +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +len COUNT(*) +252 6 +DROP TABLE t; +CREATE TABLE t(t TEXT CHARACTER SET utf8mb4) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +Warnings: +Warning 124 InnoDB rebuilding table to add column FTS_DOC_ID +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); +INSERT INTO t SET t=REPEAT(concat(repeat(_utf8mb3 0xE0B987, 4), repeat(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +# The token below exceeds the 84-character limit. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +COUNT(*) +0 +SELECT * FROM t; +t +甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文 +ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ +beforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbefor +BeforeTheIndexCreation +this was inserted after creating the index +１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１ +afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้ +🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕 +🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖 +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +len COUNT(*) +336 6 +DROP TABLE t; +CREATE TABLE t(t TEXT CHARACTER SET latin1, FULLTEXT INDEX(t)) +ENGINE=InnoDB; +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +len COUNT(*) +84 6 +DROP TABLE t; diff --git a/mysql-test/suite/innodb_fts/t/create.opt b/mysql-test/suite/innodb_fts/t/create.opt new file mode 100644 index 0000000000000..3ad568c816ef1 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/create.opt @@ -0,0 +1 @@ +--loose-innodb-sys-columns diff --git a/mysql-test/suite/innodb_fts/t/create.test b/mysql-test/suite/innodb_fts/t/create.test new file mode 100644 index 0000000000000..f0329602ed185 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/create.test @@ -0,0 +1,92 @@ +--source include/have_innodb.inc +SET NAMES utf8mb4; + +--echo # +--echo # MDEV-11233 CREATE FULLTEXT INDEX with a token +--echo # longer than 127 bytes crashes server +--echo # + +# This bug is the result of merging the Oracle MySQL follow-up fix +# BUG#22963169 MYSQL CRASHES ON CREATE FULLTEXT INDEX +# without merging a fix of Bug#79475 Insert a token of 84 4-bytes +# chars into fts index causes server crash. + +# Oracle did not publish tests for either of the above MySQL bugs. +# The tests below were developed for MariaDB Server. +# The maximum length of a fulltext-indexed word is 84 characters. + +CREATE TABLE t(t TEXT CHARACTER SET utf8mb3) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); # too long, will not be indexed +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); # too long, will not be indexed +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +--echo # The data below is not 3-byte UTF-8, but 4-byte chars. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST +(REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +SELECT * FROM t; + +# The column length should be 252 bytes (84 characters * 3 bytes/character). +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +DROP TABLE t; + +CREATE TABLE t(t TEXT CHARACTER SET utf8mb4) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); # too long, will not be indexed +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); # too long, will not be indexed +INSERT INTO t SET t=REPEAT(concat(repeat(_utf8mb3 0xE0B987, 4), repeat(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +--echo # The token below exceeds the 84-character limit. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +SELECT * FROM t; + +# The column length should be 336 bytes (84 characters * 4 bytes/character). +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +DROP TABLE t; + +CREATE TABLE t(t TEXT CHARACTER SET latin1, FULLTEXT INDEX(t)) +ENGINE=InnoDB; + +# The column length should be 84 bytes (84 characters * 1 byte/character). +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +DROP TABLE t; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 61b6f5408cf4f..124837e5151d2 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1934,6 +1935,8 @@ fts_create_common_tables( /*************************************************************//** Wrapper function of fts_create_index_tables_low(), create auxiliary tables for an FTS index + +@see row_merge_create_fts_sort_index() @return: DB_SUCCESS or error code */ static dict_table_t* @@ -1965,13 +1968,12 @@ fts_create_one_index_table( (int)(field->col->prtype & DATA_MYSQL_TYPE_MASK), (uint) dtype_get_charset_coll(field->col->prtype)); - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARCHAR, - field->col->prtype, FTS_MAX_WORD_LEN); - } else { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARMYSQL, - field->col->prtype, FTS_MAX_WORD_LEN); - } + dict_mem_table_add_col(new_table, heap, "word", + charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL, + field->col->prtype, + FTS_MAX_WORD_LEN_IN_CHAR + * DATA_MBMAXLEN(field->col->mbminmaxlen)); dict_mem_table_add_col(new_table, heap, "first_doc_id", DATA_INT, DATA_NOT_NULL | DATA_UNSIGNED, diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ea937c2075217..ed882d3354858 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -281,7 +282,7 @@ fts_zip_initialize( zip->last_big_block = 0; zip->word.f_len = 0; - memset(zip->word.f_str, 0, FTS_MAX_WORD_LEN); + *zip->word.f_str = 0; ib_vector_reset(zip->blocks); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 553d48aa48f3f..31baf3d7b99e2 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3887,7 +3887,6 @@ innobase_init( and consequently we do not need to know the ordering internally in InnoDB. */ - ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); srv_latin1_ordering = my_charset_latin1.sort_order; innobase_commit_concurrency_init_default(); @@ -6719,18 +6718,16 @@ get_innobase_type_from_mysql_type( case MYSQL_TYPE_VARCHAR: /* new >= 5.0.3 true VARCHAR */ if (field->binary()) { return(DATA_BINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_VARCHAR); } else { return(DATA_VARMYSQL); } case MYSQL_TYPE_BIT: - case MYSQL_TYPE_STRING: if (field->binary()) { - + case MYSQL_TYPE_STRING: + if (field->binary()) { return(DATA_FIXBINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_CHAR); } else { return(DATA_MYSQL); diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 3e2f359bbebef..7aa7055640c67 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -366,8 +367,8 @@ extern ulong fts_min_token_size; need a sync to free some memory */ extern bool fts_need_sync; -/** Maximum possible Fulltext word length */ -#define FTS_MAX_WORD_LEN HA_FT_MAXBYTELEN +/** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */ +#define FTS_MAX_WORD_LEN (HA_FT_MAXCHARLEN * 4) /** Maximum possible Fulltext word length (in characters) */ #define FTS_MAX_WORD_LEN_IN_CHAR HA_FT_MAXCHARLEN diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index bdd0e63c0a6f8..b941105e502c4 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -59,6 +59,8 @@ tokenized doc string. The index has three "fields": integer value) 3) Word's position in original doc. +@see fts_create_one_index_table() + @return dict_index_t structure for the fts sort index */ UNIV_INTERN dict_index_t* @@ -98,16 +100,12 @@ row_merge_create_fts_sort_index( field->prefix_len = 0; field->col = static_cast( mem_heap_alloc(new_index->heap, sizeof(dict_col_t))); - field->col->len = FTS_MAX_WORD_LEN; - - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - field->col->mtype = DATA_VARCHAR; - } else { - field->col->mtype = DATA_VARMYSQL; - } - field->col->prtype = idx_field->col->prtype | DATA_NOT_NULL; + field->col->mtype = charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL; field->col->mbminmaxlen = idx_field->col->mbminmaxlen; + field->col->len = HA_FT_MAXCHARLEN * DATA_MBMAXLEN(field->col->mbminmaxlen); + field->fixed_len = 0; /* Doc ID */ @@ -404,6 +402,7 @@ row_fts_free_pll_merge_buf( /*********************************************************************//** Tokenize incoming text data and add to the sort buffer. +@see row_merge_buf_encode() @return TRUE if the record passed, FALSE if out of space */ static ibool @@ -412,8 +411,6 @@ row_merge_fts_doc_tokenize( row_merge_buf_t** sort_buf, /*!< in/out: sort buffer */ doc_id_t doc_id, /*!< in: Doc ID */ fts_doc_t* doc, /*!< in: Doc to be tokenized */ - dtype_t* word_dtype, /*!< in: data structure for - word col */ merge_file_t** merge_file, /*!< in/out: merge file */ ibool opt_doc_id_size,/*!< in: whether to use 4 bytes instead of 8 bytes integer to @@ -445,7 +442,7 @@ row_merge_fts_doc_tokenize( ulint idx = 0; ib_uint32_t position; ulint offset = 0; - ulint cur_len = 0; + ulint cur_len; doc_id_t write_doc_id; inc = innobase_mysql_fts_get_token( @@ -499,14 +496,34 @@ row_merge_fts_doc_tokenize( dfield_set_data(field, t_str.f_str, t_str.f_len); len = dfield_get_len(field); - field->type.mtype = word_dtype->mtype; - field->type.prtype = word_dtype->prtype | DATA_NOT_NULL; + dict_col_copy_type(dict_index_get_nth_col(buf->index, 0), &field->type); + field->type.prtype |= DATA_NOT_NULL; + ut_ad(len <= field->type.len); - /* Variable length field, set to max size. */ - field->type.len = FTS_MAX_WORD_LEN; - field->type.mbminmaxlen = word_dtype->mbminmaxlen; + /* For the temporary file, row_merge_buf_encode() uses + 1 byte for representing the number of extra_size bytes. + This number will always be 1, because for this 3-field index + consisting of one variable-size column, extra_size will always + be 1 or 2, which can be encoded in one byte. + + The extra_size is 1 byte if the length of the + variable-length column is less than 128 bytes or the + maximum length is less than 256 bytes. */ + + /* One variable length column, word with its lenght less than + fts_max_token_size, add one extra size and one extra byte. + + Since the max length for FTS token now is larger than 255, + so we will need to signify length byte itself, so only 1 to 128 + bytes can be used for 1 bytes, larger than that 2 bytes. */ + if (len < 128 || field->type.len < 256) { + /* Extra size is one byte. */ + cur_len = 2 + len; + } else { + /* Extra size is two bytes. */ + cur_len = 3 + len; + } - cur_len += len; dfield_dup(field, buf->heap); field++; @@ -556,20 +573,6 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* One variable length column, word with its lenght less than - fts_max_token_size, add one extra size and one extra byte. - - Since the max length for FTS token now is larger than 255, - so we will need to signify length byte itself, so only 1 to 128 - bytes can be used for 1 bytes, larger than that 2 bytes. */ - if (t_str.f_len < 128) { - /* Extra size is one byte. */ - cur_len += 2; - } else { - /* Extra size is two bytes. */ - cur_len += 3; - } - /* Reserve one byte for the end marker of row_merge_block_t and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for encryption key_version in the beginning of the buffer. */ @@ -665,7 +668,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dtype_t word_dtype; dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; @@ -691,10 +693,6 @@ fts_parallel_tokenization( idx_field = dict_index_get_nth_field( psort_info->psort_common->dup->index, 0); - word_dtype.prtype = idx_field->col->prtype; - word_dtype.mbminmaxlen = idx_field->col->mbminmaxlen; - word_dtype.mtype = (strcmp(doc.charset->name, "latin1_swedish_ci") == 0) - ? DATA_VARCHAR : DATA_VARMYSQL; block = psort_info->merge_block; crypt_block = psort_info->crypt_block; @@ -747,7 +745,6 @@ fts_parallel_tokenization( processed = row_merge_fts_doc_tokenize( buf, doc_item->doc_id, &doc, - &word_dtype, merge_file, psort_info->psort_common->opt_doc_id_size, &t_ctx); diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index d1cde3edf7573..a9c4d17571532 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1934,6 +1935,8 @@ fts_create_common_tables( /*************************************************************//** Wrapper function of fts_create_index_tables_low(), create auxiliary tables for an FTS index + +@see row_merge_create_fts_sort_index() @return: DB_SUCCESS or error code */ static dict_table_t* @@ -1965,13 +1968,12 @@ fts_create_one_index_table( (int)(field->col->prtype & DATA_MYSQL_TYPE_MASK), (uint) dtype_get_charset_coll(field->col->prtype)); - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARCHAR, - field->col->prtype, FTS_MAX_WORD_LEN); - } else { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARMYSQL, - field->col->prtype, FTS_MAX_WORD_LEN); - } + dict_mem_table_add_col(new_table, heap, "word", + charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL, + field->col->prtype, + FTS_MAX_WORD_LEN_IN_CHAR + * DATA_MBMAXLEN(field->col->mbminmaxlen)); dict_mem_table_add_col(new_table, heap, "first_doc_id", DATA_INT, DATA_NOT_NULL | DATA_UNSIGNED, diff --git a/storage/xtradb/fts/fts0opt.cc b/storage/xtradb/fts/fts0opt.cc index ea937c2075217..ed882d3354858 100644 --- a/storage/xtradb/fts/fts0opt.cc +++ b/storage/xtradb/fts/fts0opt.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -281,7 +282,7 @@ fts_zip_initialize( zip->last_big_block = 0; zip->word.f_len = 0; - memset(zip->word.f_str, 0, FTS_MAX_WORD_LEN); + *zip->word.f_str = 0; ib_vector_reset(zip->blocks); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index c7056a005fa7a..40667f498f4f7 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4376,7 +4376,6 @@ innobase_init( and consequently we do not need to know the ordering internally in InnoDB. */ - ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); srv_latin1_ordering = my_charset_latin1.sort_order; innobase_commit_concurrency_init_default(); @@ -7498,18 +7497,16 @@ get_innobase_type_from_mysql_type( case MYSQL_TYPE_VARCHAR: /* new >= 5.0.3 true VARCHAR */ if (field->binary()) { return(DATA_BINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_VARCHAR); } else { return(DATA_VARMYSQL); } case MYSQL_TYPE_BIT: - case MYSQL_TYPE_STRING: if (field->binary()) { - + case MYSQL_TYPE_STRING: + if (field->binary()) { return(DATA_FIXBINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_CHAR); } else { return(DATA_MYSQL); diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h index 3e2f359bbebef..7aa7055640c67 100644 --- a/storage/xtradb/include/fts0fts.h +++ b/storage/xtradb/include/fts0fts.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -366,8 +367,8 @@ extern ulong fts_min_token_size; need a sync to free some memory */ extern bool fts_need_sync; -/** Maximum possible Fulltext word length */ -#define FTS_MAX_WORD_LEN HA_FT_MAXBYTELEN +/** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */ +#define FTS_MAX_WORD_LEN (HA_FT_MAXCHARLEN * 4) /** Maximum possible Fulltext word length (in characters) */ #define FTS_MAX_WORD_LEN_IN_CHAR HA_FT_MAXCHARLEN diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 43cbdecb1c1e2..8534b30d85483 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -60,6 +60,8 @@ tokenized doc string. The index has three "fields": integer value) 3) Word's position in original doc. +@see fts_create_one_index_table() + @return dict_index_t structure for the fts sort index */ UNIV_INTERN dict_index_t* @@ -101,16 +103,12 @@ row_merge_create_fts_sort_index( field->prefix_len = 0; field->col = static_cast( mem_heap_alloc(new_index->heap, sizeof(dict_col_t))); - field->col->len = FTS_MAX_WORD_LEN; - - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - field->col->mtype = DATA_VARCHAR; - } else { - field->col->mtype = DATA_VARMYSQL; - } - field->col->prtype = idx_field->col->prtype | DATA_NOT_NULL; + field->col->mtype = charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL; field->col->mbminmaxlen = idx_field->col->mbminmaxlen; + field->col->len = HA_FT_MAXCHARLEN * DATA_MBMAXLEN(field->col->mbminmaxlen); + field->fixed_len = 0; /* Doc ID */ @@ -407,6 +405,7 @@ row_fts_free_pll_merge_buf( /*********************************************************************//** Tokenize incoming text data and add to the sort buffer. +@see row_merge_buf_encode() @return TRUE if the record passed, FALSE if out of space */ static ibool @@ -415,8 +414,6 @@ row_merge_fts_doc_tokenize( row_merge_buf_t** sort_buf, /*!< in/out: sort buffer */ doc_id_t doc_id, /*!< in: Doc ID */ fts_doc_t* doc, /*!< in: Doc to be tokenized */ - dtype_t* word_dtype, /*!< in: data structure for - word col */ merge_file_t** merge_file, /*!< in/out: merge file */ ibool opt_doc_id_size,/*!< in: whether to use 4 bytes instead of 8 bytes integer to @@ -448,7 +445,7 @@ row_merge_fts_doc_tokenize( ulint idx = 0; ib_uint32_t position; ulint offset = 0; - ulint cur_len = 0; + ulint cur_len; doc_id_t write_doc_id; inc = innobase_mysql_fts_get_token( @@ -502,14 +499,34 @@ row_merge_fts_doc_tokenize( dfield_set_data(field, t_str.f_str, t_str.f_len); len = dfield_get_len(field); - field->type.mtype = word_dtype->mtype; - field->type.prtype = word_dtype->prtype | DATA_NOT_NULL; + dict_col_copy_type(dict_index_get_nth_col(buf->index, 0), &field->type); + field->type.prtype |= DATA_NOT_NULL; + ut_ad(len <= field->type.len); - /* Variable length field, set to max size. */ - field->type.len = FTS_MAX_WORD_LEN; - field->type.mbminmaxlen = word_dtype->mbminmaxlen; + /* For the temporary file, row_merge_buf_encode() uses + 1 byte for representing the number of extra_size bytes. + This number will always be 1, because for this 3-field index + consisting of one variable-size column, extra_size will always + be 1 or 2, which can be encoded in one byte. + + The extra_size is 1 byte if the length of the + variable-length column is less than 128 bytes or the + maximum length is less than 256 bytes. */ + + /* One variable length column, word with its lenght less than + fts_max_token_size, add one extra size and one extra byte. + + Since the max length for FTS token now is larger than 255, + so we will need to signify length byte itself, so only 1 to 128 + bytes can be used for 1 bytes, larger than that 2 bytes. */ + if (len < 128 || field->type.len < 256) { + /* Extra size is one byte. */ + cur_len = 2 + len; + } else { + /* Extra size is two bytes. */ + cur_len = 3 + len; + } - cur_len += len; dfield_dup(field, buf->heap); field++; @@ -559,20 +576,6 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* One variable length column, word with its lenght less than - fts_max_token_size, add one extra size and one extra byte. - - Since the max length for FTS token now is larger than 255, - so we will need to signify length byte itself, so only 1 to 128 - bytes can be used for 1 bytes, larger than that 2 bytes. */ - if (t_str.f_len < 128) { - /* Extra size is one byte. */ - cur_len += 2; - } else { - /* Extra size is two bytes. */ - cur_len += 3; - } - /* Reserve one byte for the end marker of row_merge_block_t and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for encryption key_version in the beginning of the buffer. */ @@ -668,7 +671,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dtype_t word_dtype; dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; @@ -694,10 +696,6 @@ fts_parallel_tokenization( idx_field = dict_index_get_nth_field( psort_info->psort_common->dup->index, 0); - word_dtype.prtype = idx_field->col->prtype; - word_dtype.mbminmaxlen = idx_field->col->mbminmaxlen; - word_dtype.mtype = (strcmp(doc.charset->name, "latin1_swedish_ci") == 0) - ? DATA_VARCHAR : DATA_VARMYSQL; block = psort_info->merge_block; crypt_block = psort_info->crypt_block; @@ -750,7 +748,6 @@ fts_parallel_tokenization( processed = row_merge_fts_doc_tokenize( buf, doc_item->doc_id, &doc, - &word_dtype, merge_file, psort_info->psort_common->opt_doc_id_size, &t_ctx); From 18cdff6765b954431934f6e6d0b1e281c8f0e0b8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 4 Dec 2016 12:37:01 +0100 Subject: [PATCH 236/295] MDEV-10293 'setupterm' was not declared in this scope Check for readline before checking for curses headers, because MYSQL_CHECK_READLINE fails when curses is not found, but CHECK_INCLUDE_FILES simply remembers the fact and continues. So if there's no curses, MYSQL_CHECK_READLINE will abort, the user will then installs curses and continue the build. Thus, CHECK_INCLUDE_HEADERS will remember that there is no curses, but other checks from MYSQL_CHECK_READLINE will remember that curses are there. It will result in inconsistent HAVE_xxx defines. --- cmake/readline.cmake | 1 + configure.cmake | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/readline.cmake b/cmake/readline.cmake index c72f8d0da5c6d..64e04ef8663e0 100644 --- a/cmake/readline.cmake +++ b/cmake/readline.cmake @@ -230,5 +230,6 @@ MACRO (MYSQL_CHECK_READLINE) SET(CMAKE_REQUIRED_LIBRARIES) SET(CMAKE_REQUIRED_INCLUDES) ENDIF(NOT WIN32) + CHECK_INCLUDE_FILES ("curses.h;term.h" HAVE_TERM_H) ENDMACRO() diff --git a/configure.cmake b/configure.cmake index f0517bfa41eed..4051b55374762 100644 --- a/configure.cmake +++ b/configure.cmake @@ -237,7 +237,6 @@ CHECK_INCLUDE_FILES (sys/socket.h HAVE_SYS_SOCKET_H) CHECK_INCLUDE_FILES (sys/stat.h HAVE_SYS_STAT_H) CHECK_INCLUDE_FILES (sys/stream.h HAVE_SYS_STREAM_H) CHECK_INCLUDE_FILES (sys/termcap.h HAVE_SYS_TERMCAP_H) -CHECK_INCLUDE_FILES ("curses.h;term.h" HAVE_TERM_H) CHECK_INCLUDE_FILES (asm/termbits.h HAVE_ASM_TERMBITS_H) CHECK_INCLUDE_FILES (termbits.h HAVE_TERMBITS_H) CHECK_INCLUDE_FILES (termios.h HAVE_TERMIOS_H) From 611f91605adce17df87acf96b5aede0b73d0fc12 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 5 Dec 2016 20:19:01 +0200 Subject: [PATCH 237/295] MDEV-9038 Binlog encryption tests - created binlog_encryption test suite and added it to the default list - moved some tests from rpl, binlog and multisource suites to extra so that they could be re-used in different suites - made minor changes in include files --- .../extra/binlog_tests/binlog_incident.inc | 68 + .../extra/binlog_tests/binlog_index.inc | 278 ++ .../extra/binlog_tests/binlog_ioerr.inc | 36 + .../binlog_tests/binlog_mysqlbinlog-cp932.inc | 50 + .../binlog_tests/binlog_row_annotate.inc | 211 ++ .../extra/binlog_tests/binlog_write_error.inc | 108 + .../extra/binlog_tests/binlog_xa_recover.inc | 281 ++ mysql-test/extra/rpl_tests/multisource.inc | 313 +++ .../extra/rpl_tests/rpl_binlog_errors.inc | 422 +++ .../rpl_cant_read_event_incident.inc | 83 + mysql-test/extra/rpl_tests/rpl_checksum.inc | 333 +++ .../extra/rpl_tests/rpl_checksum_cache.inc | 261 ++ mysql-test/extra/rpl_tests/rpl_corruption.inc | 176 ++ mysql-test/extra/rpl_tests/rpl_gtid_basic.inc | 568 ++++ mysql-test/extra/rpl_tests/rpl_incident.inc | 54 + .../extra/rpl_tests/rpl_init_slave_errors.inc | 95 + .../extra/rpl_tests/rpl_loaddata_local.inc | 246 ++ mysql-test/extra/rpl_tests/rpl_loadfile.inc | 120 + mysql-test/extra/rpl_tests/rpl_packet.inc | 183 ++ mysql-test/extra/rpl_tests/rpl_parallel.inc | 2479 +++++++++++++++++ ...parallel_show_binlog_events_purge_logs.inc | 40 + .../extra/rpl_tests/rpl_relayrotate.inc | 18 + mysql-test/extra/rpl_tests/rpl_semi_sync.inc | 580 ++++ .../extra/rpl_tests/rpl_skip_replication.inc | 402 +++ .../extra/rpl_tests/rpl_special_charset.inc | 32 + .../extra/rpl_tests/rpl_sporadic_master.inc | 32 + mysql-test/extra/rpl_tests/rpl_ssl.inc | 117 + .../rpl_tests/rpl_stm_relay_ign_space.inc | 107 + .../rpl_tests/rpl_switch_stm_row_mixed.inc | 631 +++++ mysql-test/extra/rpl_tests/rpl_sync.inc | 159 ++ ...rpl_temporal_format_default_to_default.inc | 82 + mysql-test/extra/rpl_tests/rpl_typeconv.inc | 78 + .../reset_master_slave.inc | 0 mysql-test/include/search_pattern_in_file.inc | 35 +- .../wait_for_sql_thread_read_all.inc | 0 mysql-test/mysql-test-run.pl | 1 + .../suite/binlog/t/binlog_incident.test | 30 +- mysql-test/suite/binlog/t/binlog_index.test | 273 +- mysql-test/suite/binlog/t/binlog_ioerr.test | 31 +- .../binlog/t/binlog_mysqlbinlog-cp932.test | 27 +- .../suite/binlog/t/binlog_row_annotate.test | 192 +- .../suite/binlog/t/binlog_write_error.test | 103 +- .../suite/binlog/t/binlog_xa_recover.test | 276 +- .../binlog_incident.combinations | 8 + .../binlog_encryption/binlog_incident.result | 13 + .../binlog_encryption/binlog_incident.test | 2 + .../binlog_encryption/binlog_index.result | 187 ++ .../suite/binlog_encryption/binlog_index.test | 1 + .../binlog_encryption/binlog_ioerr.result | 32 + .../suite/binlog_encryption/binlog_ioerr.test | 1 + .../binlog_mysqlbinlog-cp932-master.opt | 1 + .../binlog_mysqlbinlog-cp932.result | 19 + .../binlog_mysqlbinlog-cp932.test | 2 + .../binlog_row_annotate-master.opt | 1 + .../binlog_row_annotate.result | 724 +++++ .../binlog_row_annotate.test | 2 + .../binlog_write_error.result | 108 + .../binlog_encryption/binlog_write_error.test | 1 + .../binlog_xa_recover-master.opt | 1 + .../binlog_xa_recover.result | 216 ++ .../binlog_encryption/binlog_xa_recover.test | 1 + .../suite/binlog_encryption/disabled.def | 4 + .../binlog_encryption/encrypted_master.result | 622 +++++ .../binlog_encryption/encrypted_master.test | 183 ++ .../encrypted_master_lost_key.result | 111 + .../encrypted_master_lost_key.test | 205 ++ ...encrypted_master_switch_to_unencrypted.cnf | 5 + ...ncrypted_master_switch_to_unencrypted.test | 135 + .../binlog_encryption/encrypted_slave.cnf | 12 + .../binlog_encryption/encrypted_slave.result | 175 ++ .../binlog_encryption/encrypted_slave.test | 117 + .../encryption_algorithms.combinations | 5 + .../encryption_algorithms.inc | 2 + .../binlog_encryption/encryption_combo.cnf | 5 + .../binlog_encryption/encryption_combo.result | 78 + .../binlog_encryption/encryption_combo.test | 136 + .../suite/binlog_encryption/multisource.cnf | 17 + .../binlog_encryption/multisource.result | 205 ++ .../suite/binlog_encryption/multisource.test | 2 + mysql-test/suite/binlog_encryption/my.cnf | 27 + .../binlog_encryption/restart_server.inc | 35 + .../binlog_encryption/rpl_binlog_errors.cnf | 7 + .../rpl_binlog_errors.result | 275 ++ .../binlog_encryption/rpl_binlog_errors.test | 2 + .../rpl_cant_read_event_incident.result | 21 + .../rpl_cant_read_event_incident.test | 1 + .../suite/binlog_encryption/rpl_checksum.cnf | 10 + .../binlog_encryption/rpl_checksum.result | 161 ++ .../suite/binlog_encryption/rpl_checksum.test | 1 + .../rpl_checksum_cache.result | 119 + .../binlog_encryption/rpl_checksum_cache.test | 1 + .../binlog_encryption/rpl_corruption.cnf | 9 + .../binlog_encryption/rpl_corruption.result | 51 + .../binlog_encryption/rpl_corruption.test | 1 + .../binlog_encryption/rpl_gtid_basic.cnf | 24 + .../binlog_encryption/rpl_gtid_basic.result | 481 ++++ .../binlog_encryption/rpl_gtid_basic.test | 1 + .../suite/binlog_encryption/rpl_incident.cnf | 7 + .../binlog_encryption/rpl_incident.result | 38 + .../suite/binlog_encryption/rpl_incident.test | 1 + .../rpl_init_slave_errors.result | 19 + .../rpl_init_slave_errors.test | 1 + .../rpl_loaddata_local.result | 125 + .../binlog_encryption/rpl_loaddata_local.test | 1 + .../binlog_encryption/rpl_loadfile.result | 242 ++ .../suite/binlog_encryption/rpl_loadfile.test | 11 + .../rpl_mixed_binlog_max_cache_size.result | 186 ++ .../rpl_mixed_binlog_max_cache_size.test | 7 + .../suite/binlog_encryption/rpl_packet.cnf | 10 + .../suite/binlog_encryption/rpl_packet.result | 58 + .../suite/binlog_encryption/rpl_packet.test | 1 + .../binlog_encryption/rpl_parallel.result | 1774 ++++++++++++ .../suite/binlog_encryption/rpl_parallel.test | 1 + ...parallel_show_binlog_events_purge_logs.cnf | 6 + ...allel_show_binlog_events_purge_logs.result | 12 + ...arallel_show_binlog_events_purge_logs.test | 1 + .../rpl_relayrotate-slave.opt | 5 + .../binlog_encryption/rpl_relayrotate.result | 13 + .../binlog_encryption/rpl_relayrotate.test | 1 + .../binlog_encryption/rpl_semi_sync.result | 465 ++++ .../binlog_encryption/rpl_semi_sync.test | 1 + .../rpl_skip_replication.cnf | 6 + .../rpl_skip_replication.result | 252 ++ .../rpl_skip_replication.test | 2 + .../binlog_encryption/rpl_special_charset.opt | 1 + .../rpl_special_charset.result | 8 + .../rpl_special_charset.test | 1 + .../rpl_sporadic_master-master.opt | 1 + .../rpl_sporadic_master.result | 23 + .../rpl_sporadic_master.test | 1 + .../suite/binlog_encryption/rpl_ssl.result | 47 + .../suite/binlog_encryption/rpl_ssl.test | 1 + .../rpl_stm_relay_ign_space-slave.opt | 1 + .../rpl_stm_relay_ign_space.result | 5 + .../rpl_stm_relay_ign_space.test | 1 + .../rpl_switch_stm_row_mixed.result | 428 +++ .../rpl_switch_stm_row_mixed.test | 1 + .../binlog_encryption/rpl_sync-master.opt | 2 + .../binlog_encryption/rpl_sync-slave.opt | 2 + .../suite/binlog_encryption/rpl_sync.result | 41 + .../suite/binlog_encryption/rpl_sync.test | 1 + ...rpl_temporal_format_default_to_default.cnf | 6 + ..._temporal_format_default_to_default.result | 83 + ...pl_temporal_format_default_to_default.test | 1 + ...l_temporal_format_mariadb53_to_mysql56.cnf | 6 + ...emporal_format_mariadb53_to_mysql56.result | 85 + ..._temporal_format_mariadb53_to_mysql56.test | 6 + ...l_temporal_format_mysql56_to_mariadb53.cnf | 6 + ...emporal_format_mysql56_to_mariadb53.result | 85 + ..._temporal_format_mysql56_to_mariadb53.test | 4 + .../binlog_encryption/rpl_typeconv.result | 540 ++++ .../suite/binlog_encryption/rpl_typeconv.test | 1 + mysql-test/suite/binlog_encryption/suite.pm | 18 + .../suite/binlog_encryption/testdata.inc | 207 ++ .../suite/binlog_encryption/testdata.opt | 1 + mysql-test/suite/multi_source/gtid.test | 8 +- .../multi_source/gtid_ignore_duplicates.test | 8 +- mysql-test/suite/multi_source/info_logs.test | 6 +- mysql-test/suite/multi_source/load_data.test | 6 +- .../suite/multi_source/multisource.test | 292 +- .../suite/multi_source/relaylog_events.test | 4 +- .../suite/multi_source/reset_slave.test | 4 +- mysql-test/suite/multi_source/simple.test | 6 +- .../suite/multi_source/skip_counter.test | 6 +- .../suite/multi_source/status_vars.test | 6 +- mysql-test/suite/rpl/t/rpl_binlog_errors.test | 404 +-- .../rpl/t/rpl_cant_read_event_incident.test | 78 +- mysql-test/suite/rpl/t/rpl_checksum.test | 328 +-- .../suite/rpl/t/rpl_checksum_cache.test | 256 +- mysql-test/suite/rpl/t/rpl_corruption.test | 171 +- mysql-test/suite/rpl/t/rpl_gtid_basic.test | 563 +--- mysql-test/suite/rpl/t/rpl_incident.test | 49 +- .../suite/rpl/t/rpl_init_slave_errors.test | 90 +- mysql-test/suite/rpl/t/rpl_loaddatalocal.test | 241 +- mysql-test/suite/rpl/t/rpl_loadfile.test | 115 +- mysql-test/suite/rpl/t/rpl_packet.test | 178 +- mysql-test/suite/rpl/t/rpl_parallel.test | 2474 +--------------- ...arallel_show_binlog_events_purge_logs.test | 35 +- mysql-test/suite/rpl/t/rpl_relayrotate.test | 13 +- mysql-test/suite/rpl/t/rpl_semi_sync.test | 575 +--- .../suite/rpl/t/rpl_skip_replication.test | 378 +-- .../suite/rpl/t/rpl_special_charset.test | 27 +- .../suite/rpl/t/rpl_sporadic_master.test | 27 +- mysql-test/suite/rpl/t/rpl_ssl.test | 112 +- .../suite/rpl/t/rpl_stm_relay_ign_space.test | 102 +- .../suite/rpl/t/rpl_switch_stm_row_mixed.test | 626 +---- mysql-test/suite/rpl/t/rpl_sync.test | 153 +- ...pl_temporal_format_default_to_default.test | 77 +- mysql-test/suite/rpl/t/rpl_typeconv.test | 73 +- mysql-test/unstable-tests | 4 + 190 files changed, 18153 insertions(+), 8379 deletions(-) create mode 100644 mysql-test/extra/binlog_tests/binlog_incident.inc create mode 100644 mysql-test/extra/binlog_tests/binlog_index.inc create mode 100644 mysql-test/extra/binlog_tests/binlog_ioerr.inc create mode 100644 mysql-test/extra/binlog_tests/binlog_mysqlbinlog-cp932.inc create mode 100644 mysql-test/extra/binlog_tests/binlog_row_annotate.inc create mode 100644 mysql-test/extra/binlog_tests/binlog_write_error.inc create mode 100644 mysql-test/extra/binlog_tests/binlog_xa_recover.inc create mode 100644 mysql-test/extra/rpl_tests/multisource.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_binlog_errors.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_cant_read_event_incident.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_checksum.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_checksum_cache.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_corruption.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_gtid_basic.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_incident.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_init_slave_errors.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_loaddata_local.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_loadfile.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_packet.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_parallel.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_parallel_show_binlog_events_purge_logs.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_relayrotate.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_semi_sync.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_skip_replication.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_special_charset.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_sporadic_master.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_ssl.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_stm_relay_ign_space.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_switch_stm_row_mixed.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_sync.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_temporal_format_default_to_default.inc create mode 100644 mysql-test/extra/rpl_tests/rpl_typeconv.inc rename mysql-test/{suite/multi_source => include}/reset_master_slave.inc (100%) rename mysql-test/{suite/multi_source => include}/wait_for_sql_thread_read_all.inc (100%) create mode 100644 mysql-test/suite/binlog_encryption/binlog_incident.combinations create mode 100644 mysql-test/suite/binlog_encryption/binlog_incident.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_incident.test create mode 100644 mysql-test/suite/binlog_encryption/binlog_index.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_index.test create mode 100644 mysql-test/suite/binlog_encryption/binlog_ioerr.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_ioerr.test create mode 100644 mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932-master.opt create mode 100644 mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.test create mode 100644 mysql-test/suite/binlog_encryption/binlog_row_annotate-master.opt create mode 100644 mysql-test/suite/binlog_encryption/binlog_row_annotate.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_row_annotate.test create mode 100644 mysql-test/suite/binlog_encryption/binlog_write_error.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_write_error.test create mode 100644 mysql-test/suite/binlog_encryption/binlog_xa_recover-master.opt create mode 100644 mysql-test/suite/binlog_encryption/binlog_xa_recover.result create mode 100644 mysql-test/suite/binlog_encryption/binlog_xa_recover.test create mode 100644 mysql-test/suite/binlog_encryption/disabled.def create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master.result create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master.test create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master_lost_key.result create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master_lost_key.test create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.cnf create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test create mode 100644 mysql-test/suite/binlog_encryption/encrypted_slave.cnf create mode 100644 mysql-test/suite/binlog_encryption/encrypted_slave.result create mode 100644 mysql-test/suite/binlog_encryption/encrypted_slave.test create mode 100644 mysql-test/suite/binlog_encryption/encryption_algorithms.combinations create mode 100644 mysql-test/suite/binlog_encryption/encryption_algorithms.inc create mode 100644 mysql-test/suite/binlog_encryption/encryption_combo.cnf create mode 100644 mysql-test/suite/binlog_encryption/encryption_combo.result create mode 100644 mysql-test/suite/binlog_encryption/encryption_combo.test create mode 100644 mysql-test/suite/binlog_encryption/multisource.cnf create mode 100644 mysql-test/suite/binlog_encryption/multisource.result create mode 100644 mysql-test/suite/binlog_encryption/multisource.test create mode 100644 mysql-test/suite/binlog_encryption/my.cnf create mode 100644 mysql-test/suite/binlog_encryption/restart_server.inc create mode 100644 mysql-test/suite/binlog_encryption/rpl_binlog_errors.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_binlog_errors.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_binlog_errors.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_checksum.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_checksum.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_checksum.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_checksum_cache.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_checksum_cache.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_corruption.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_corruption.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_corruption.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_gtid_basic.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_gtid_basic.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_gtid_basic.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_incident.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_incident.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_incident.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_init_slave_errors.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_init_slave_errors.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_loaddata_local.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_loaddata_local.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_loadfile.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_loadfile.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_packet.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_packet.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_packet.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_parallel.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_parallel.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_relayrotate-slave.opt create mode 100644 mysql-test/suite/binlog_encryption/rpl_relayrotate.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_relayrotate.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_semi_sync.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_semi_sync.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_skip_replication.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_skip_replication.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_skip_replication.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_special_charset.opt create mode 100644 mysql-test/suite/binlog_encryption/rpl_special_charset.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_special_charset.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_sporadic_master-master.opt create mode 100644 mysql-test/suite/binlog_encryption/rpl_sporadic_master.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_sporadic_master.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_ssl.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_ssl.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space-slave.opt create mode 100644 mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_sync-master.opt create mode 100644 mysql-test/suite/binlog_encryption/rpl_sync-slave.opt create mode 100644 mysql-test/suite/binlog_encryption/rpl_sync.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_sync.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.cnf create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.test create mode 100644 mysql-test/suite/binlog_encryption/rpl_typeconv.result create mode 100644 mysql-test/suite/binlog_encryption/rpl_typeconv.test create mode 100644 mysql-test/suite/binlog_encryption/suite.pm create mode 100644 mysql-test/suite/binlog_encryption/testdata.inc create mode 100644 mysql-test/suite/binlog_encryption/testdata.opt diff --git a/mysql-test/extra/binlog_tests/binlog_incident.inc b/mysql-test/extra/binlog_tests/binlog_incident.inc new file mode 100644 index 0000000000000..8ec67746f261f --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_incident.inc @@ -0,0 +1,68 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# +# Usage: +# +# --let $use_remote_mysqlbinlog= 1 # optional +# --let $binlog_start_pos= # optional +# --let $binlog_file= # optional +# +# --source extra/binlog_tests/binlog_incident.inc +# +# The script uses MYSQLBINLOG to verify certain results. +# By default, it uses binary logs directly. If it is undesirable, +# this behavior can be overridden by setting $use_remote_binlog +# as shown above. +# +# All values will be unset after every execution of the script, +# so if they are needed, they should be set explicitly before each call. +# + +# The purpose of this test is to provide a reference for how the +# incident log event is represented in the output from the mysqlbinlog +# program. + +source include/have_log_bin.inc; +source include/have_debug.inc; +source include/binlog_start_pos.inc; + +let $MYSQLD_DATADIR= `select @@datadir`; +RESET MASTER; + +CREATE TABLE t1 (a INT); + +INSERT INTO t1 VALUES (1),(2),(3); +SELECT * FROM t1; + +# This will generate an incident log event and store it in the binary +# log before the replace statement. +REPLACE INTO t1 VALUES (4); + +DROP TABLE t1; +FLUSH LOGS; + +if ($binlog_start_pos) +{ + --let $startpos= --start-position=$binlog_start_pos + --let $binlog_start_pos= +} +--let $filename= master-bin.000001 +if ($binlog_file) +{ + --let $filename= $binlog_file + --let $binlog_file= +} +--let $mysqlbinlog_args= $MYSQLD_DATADIR/$filename +if ($use_remote_mysqlbinlog) +{ + --let $mysqlbinlog_args= --read-from-remote-server --protocol=tcp --host=127.0.0.1 --port=$MASTER_MYPORT -uroot $filename + --let $use_remote_mysqlbinlog= 0 +} +exec $MYSQL_BINLOG $startpos $mysqlbinlog_args >$MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql; +--disable_query_log +eval SELECT cont LIKE '%RELOAD DATABASE; # Shall generate syntax error%' AS `Contain RELOAD DATABASE` FROM (SELECT load_file('$MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql') AS cont) AS tbl; +--enable_query_log + +remove_file $MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql; diff --git a/mysql-test/extra/binlog_tests/binlog_index.inc b/mysql-test/extra/binlog_tests/binlog_index.inc new file mode 100644 index 0000000000000..56c621a74c5ee --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_index.inc @@ -0,0 +1,278 @@ +# +# This include file is used by more than one test suite +# (currently binlog and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# +# testing of purging of binary log files bug#18199/Bug#18453 +# +source include/have_log_bin.inc; +source include/not_embedded.inc; +# Don't test this under valgrind, memory leaks will occur +--source include/not_valgrind.inc +source include/have_debug.inc; +# Avoid CrashReporter popup on Mac +--source include/not_crashrep.inc +call mtr.add_suppression('Attempting backtrace'); +call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); +call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); +call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); +call mtr.add_suppression('Could not open .*'); +call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); +flush tables; + +let $old=`select @@debug`; + +RESET MASTER; + +let $MYSQLD_DATADIR= `select @@datadir`; +let $INDEX=$MYSQLD_DATADIR/master-bin.index; + +# +# testing purge binary logs TO +# + +flush logs; +flush logs; +flush logs; + +source include/show_binary_logs.inc; +remove_file $MYSQLD_DATADIR/master-bin.000001; +flush tables; + +# there must be a warning with file names +replace_regex /\.[\\\/]master/master/; +--source include/wait_for_binlog_checkpoint.inc +purge binary logs TO 'master-bin.000004'; + +--echo *** must show a list starting from the 'TO' argument of PURGE *** +source include/show_binary_logs.inc; + +# +# testing purge binary logs BEFORE +# + +reset master; + +flush logs; +flush logs; +flush logs; +remove_file $MYSQLD_DATADIR/master-bin.000001; + +--echo *** must be a warning master-bin.000001 was not found *** +let $date=`select NOW() + INTERVAL 1 MINUTE`; +--disable_query_log +replace_regex /\.[\\\/]master/master/; +--source include/wait_for_binlog_checkpoint.inc +eval purge binary logs BEFORE '$date'; +--enable_query_log + +--echo *** must show one record, of the active binlog, left in the index file after PURGE *** +source include/show_binary_logs.inc; + +# +# testing a fatal error +# Turning a binlog file into a directory must be a portable setup +# + +reset master; + +flush logs; +flush logs; +flush logs; + +remove_file $MYSQLD_DATADIR/master-bin.000001; +mkdir $MYSQLD_DATADIR/master-bin.000001; + +--source include/wait_for_binlog_checkpoint.inc +--error ER_BINLOG_PURGE_FATAL_ERR +purge binary logs TO 'master-bin.000002'; +replace_regex /\.[\\\/]master/master/; +show warnings; +rmdir $MYSQLD_DATADIR/master-bin.000001; +--disable_warnings +reset master; +--enable_warnings + +--echo # crash_purge_before_update_index +flush logs; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug_dbug="+d,crash_purge_before_update_index"; +--source include/wait_for_binlog_checkpoint.inc +--error 2013 +purge binary logs TO 'master-bin.000002'; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +file_exists $MYSQLD_DATADIR/master-bin.000001; +file_exists $MYSQLD_DATADIR/master-bin.000002; +file_exists $MYSQLD_DATADIR/master-bin.000003; +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # crash_purge_non_critical_after_update_index +flush logs; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug_dbug="+d,crash_purge_non_critical_after_update_index"; +--source include/wait_for_binlog_checkpoint.inc +--error 2013 +purge binary logs TO 'master-bin.000004'; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000001; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000002; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000003; +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # crash_purge_critical_after_update_index +flush logs; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug_dbug="+d,crash_purge_critical_after_update_index"; +--source include/wait_for_binlog_checkpoint.inc +--error 2013 +purge binary logs TO 'master-bin.000006'; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000004; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000005; +file_exists $MYSQLD_DATADIR/master-bin.000006; +file_exists $MYSQLD_DATADIR/master-bin.000007; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000008; +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # crash_create_non_critical_before_update_index +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug_dbug="+d,crash_create_non_critical_before_update_index"; +--error 2013 +flush logs; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +file_exists $MYSQLD_DATADIR/master-bin.000008; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000009; +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # crash_create_critical_before_update_index +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug_dbug="+d,crash_create_critical_before_update_index"; +--error 2013 +flush logs; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +file_exists $MYSQLD_DATADIR/master-bin.000009; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000010; +--error 1 +file_exists $MYSQLD_DATADIR/master-bin.000011; +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # crash_create_after_update_index +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug_dbug="+d,crash_create_after_update_index"; +--error 2013 +flush logs; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +file_exists $MYSQLD_DATADIR/master-bin.000010; +file_exists $MYSQLD_DATADIR/master-bin.000011; +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # +--echo # This should put the server in unsafe state and stop +--echo # accepting any command. If we inject a fault at this +--echo # point and continue the execution the server crashes. +--echo # + +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # fault_injection_registering_index +SET SESSION debug_dbug="+d,fault_injection_registering_index"; +-- replace_regex /\.[\\\/]master/master/ +-- error ER_CANT_OPEN_FILE +flush logs; + +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--source include/restart_mysqld.inc + +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--echo # fault_injection_updating_index +SET SESSION debug_dbug="+d,fault_injection_updating_index"; +-- replace_regex /\.[\\\/]master/master/ +-- error ER_CANT_OPEN_FILE +flush logs; + +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +--source include/restart_mysqld.inc + +--chmod 0644 $INDEX +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SET @index=LOAD_FILE('$index') +-- replace_regex /\.[\\\/]master/master/ +SELECT @index; + +eval SET SESSION debug_dbug="$old"; + +--echo End of tests diff --git a/mysql-test/extra/binlog_tests/binlog_ioerr.inc b/mysql-test/extra/binlog_tests/binlog_ioerr.inc new file mode 100644 index 0000000000000..8d1069bacb01a --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_ioerr.inc @@ -0,0 +1,36 @@ +# +# This include file is used by more than one test suite +# (currently binlog and binlog_encryption). +# Please check all dependent tests after modifying it +# + +source include/have_debug.inc; +source include/have_innodb.inc; +source include/have_log_bin.inc; +source include/have_binlog_format_mixed_or_statement.inc; + +CALL mtr.add_suppression("Error writing file 'master-bin'"); + +RESET MASTER; + +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb; +INSERT INTO t1 VALUES(0); +SET SESSION debug_dbug='+d,fail_binlog_write_1'; +--error ER_ERROR_ON_WRITE +INSERT INTO t1 VALUES(1); +--error ER_ERROR_ON_WRITE +INSERT INTO t1 VALUES(2); +SET SESSION debug_dbug=''; +INSERT INTO t1 VALUES(3); +SELECT * FROM t1; + +# Actually the output from this currently shows a bug. +# The injected IO error leaves partially written transactions in the binlog in +# the form of stray "BEGIN" events. +# These should disappear from the output if binlog error handling is improved +# (see MySQL Bug#37148 and WL#1790). +--replace_regex /\/\* xid=.* \*\//\/* XID *\// /Server ver: .*, Binlog ver: .*/Server ver: #, Binlog ver: #/ /table_id: [0-9]+/table_id: #/ +--replace_column 1 BINLOG 2 POS 5 ENDPOS +SHOW BINLOG EVENTS; + +DROP TABLE t1; diff --git a/mysql-test/extra/binlog_tests/binlog_mysqlbinlog-cp932.inc b/mysql-test/extra/binlog_tests/binlog_mysqlbinlog-cp932.inc new file mode 100644 index 0000000000000..ac637b2da03bd --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_mysqlbinlog-cp932.inc @@ -0,0 +1,50 @@ +# +# This include file is used by more than one test suite +# (currently binlog and binlog_encryption). +# Please check all dependent tests after modifying it +# +# Usage: +# --let $use_remote_mysqlbinlog= 1 # optional +# --source extra/binlog_tests/binlog_mysqlbinlog-cp932.inc +# +# By default, the script calls mysqlbinlog to read binary logs directly. +# If it is undesirable, this behavior can be overridden by setting +# $use_remote_binlog as shown above. +# The value will be unset after every execution of the script, +# so if it is needed, it should be set explicitly before each call. + + +# disabled in embedded until tools running is fixed with embedded +--source include/not_embedded.inc + +-- source include/have_binlog_format_mixed_or_statement.inc +-- source include/have_cp932.inc +-- source include/have_log_bin.inc + +RESET MASTER; + +# Bug#16217 (mysql client did not know how not switch its internal charset) +create table t3 (f text character set utf8); +create table t4 (f text character set cp932); +--exec $MYSQL --default-character-set=utf8 test -e "insert into t3 values(_utf8'ソ')" +--exec $MYSQL --default-character-set=cp932 test -e "insert into t4 values(_cp932'ƒ\');" +flush logs; +rename table t3 to t03, t4 to t04; +let $MYSQLD_DATADIR= `select @@datadir`; + +--let $mysqlbinlog_args= $MYSQLD_DATADIR/master-bin.000001 +if ($use_remote_mysqlbinlog) +{ + --let $mysqlbinlog_args= --read-from-remote-server --protocol=tcp --host=127.0.0.1 --port=$MASTER_MYPORT -uroot master-bin.000001 + --let $use_remote_mysqlbinlog= 0 +} + +--exec $MYSQL_BINLOG --short-form $mysqlbinlog_args | $MYSQL --default-character-set=utf8 +# original and recovered data must be equal +select HEX(f) from t03; +select HEX(f) from t3; +select HEX(f) from t04; +select HEX(f) from t4; + +drop table t3, t4, t03, t04; +--echo End of 5.0 tests diff --git a/mysql-test/extra/binlog_tests/binlog_row_annotate.inc b/mysql-test/extra/binlog_tests/binlog_row_annotate.inc new file mode 100644 index 0000000000000..97d641fca808a --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_row_annotate.inc @@ -0,0 +1,211 @@ +# +# This include file is used by more than one test suite +# (currently binlog and binlog_encryption). +# Please check all dependent tests after modifying it +# +# Usage: +# --let $use_remote_mysqlbinlog= 1 # optional +# --source extra/binlog_tests/binlog_row_annotate.inc +# +# By default, the script uses mysqlbinlog both with direct access to files +# and via connection to the server. In some cases, direct access to files +# might be impossible (e.g. with encryption). If use_remote_mysqlbinlog +# flag is set, this part of the logic will be omitted. +# + +############################################################################### +# WL47: Store in binlog text of statements that caused RBR events +# new event: ANNOTATE_ROWS_EVENT +# new master option: --binlog-annotate-row-events +# new mysqlbinlog option: --skip-annotate-row-events +# +# Intended to test that: +# *** If the --binlog-annotate-row-events option is switched on on master +# then Annotate_rows events: +# - are generated; +# - are generated only once for "multi-table-maps" rbr queries; +# - are not generated when the corresponding queries are filtered away; +# - are generated when the corresponding queries are filtered away partialy +# (e.g. in case of multi-delete). +# *** Annotate_rows events are printed by mysqlbinlog started without +# --skip-annotate-row-events options both in remote and local cases. +# *** Annotate_rows events are not printed by mysqlbinlog started with +# --skip-annotate-row-events options both in remote and local cases. +############################################################################### + +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/binlog_start_pos.inc + +--disable_query_log + +set sql_mode=""; + +# Fix timestamp to avoid varying results +SET timestamp=1000000000; + +# Delete all existing binary logs +RESET MASTER; + +--disable_warnings +DROP DATABASE IF EXISTS test1; +DROP DATABASE IF EXISTS test2; +DROP DATABASE IF EXISTS test3; +DROP DATABASE IF EXISTS xtest1; +DROP DATABASE IF EXISTS xtest2; +--enable_warnings + +CREATE DATABASE test1; +CREATE TABLE test1.t1(a int); + +CREATE DATABASE test2; +CREATE TABLE test2.t2(a int); +CREATE VIEW test2.v2 AS SELECT * FROM test2.t2; + +CREATE DATABASE test3; +CREATE TABLE test3.t3(a int); + +CREATE DATABASE xtest1; +CREATE TABLE xtest1.xt1(a int); + +CREATE DATABASE xtest2; +CREATE TABLE xtest2.xt2(a int); + +# By default SESSION binlog_annotate_row_events = OFF + +INSERT INTO test1.t1 VALUES (1), (2), (3); + +SET SESSION binlog_annotate_row_events = ON; + +INSERT INTO test2.t2 VALUES (1), (2), (3); +INSERT INTO test3.t3 VALUES (1), (2), (3); + +# This query generates two Table maps but the Annotate +# event should appear only once before the first Table map +DELETE test1.t1, test2.t2 + FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 + WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a; + +# This event should be filtered out together with Annotate event +INSERT INTO xtest1.xt1 VALUES (1), (2), (3); + +# This event should pass the filter +INSERT INTO test2.v2 VALUES (1), (2), (3); + +# This event should pass the filter only for test2.t2 part +DELETE xtest1.xt1, test2.t2 + FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 + WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a; + +# These events should be filtered out together with Annotate events +INSERT INTO xtest1.xt1 VALUES (1), (2), (3); +INSERT INTO xtest2.xt2 VALUES (1), (2), (3); +DELETE xtest1.xt1, xtest2.xt2 + FROM xtest1.xt1 INNER JOIN xtest2.xt2 INNER JOIN test3.t3 + WHERE xtest1.xt1.a=xtest2.xt2.a AND xtest2.xt2.a=test3.t3.a; + +FLUSH LOGS; +--enable_query_log + +--echo ##################################################################################### +--echo # The following Annotate_rows events should appear below: +--echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) +--echo # - INSERT INTO test3.t3 VALUES (1), (2), (3) +--echo # - DELETE test1.t1, test2.t2 FROM <...> +--echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) +--echo # - DELETE xtest1.xt1, test2.t2 FROM <...> +--echo ##################################################################################### + +let $start_pos= `select @binlog_start_pos`; +--replace_column 2 # 5 # +--replace_result $start_pos +--replace_regex /table_id: [0-9]+/table_id: #/ /\/\* xid=.* \*\//\/* xid= *\// +--eval show binlog events in 'master-bin.000001' from $start_pos + +if (!$use_remote_mysqlbinlog) +{ + --echo # + --echo ##################################################################################### + --echo # mysqlbinlog + --echo # The following Annotates should appear in this output: + --echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) + --echo # - INSERT INTO test3.t3 VALUES (1), (2), (3) + --echo # - DELETE test1.t1, test2.t2 FROM <...> (with two subsequent Table maps) + --echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) + --echo # - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) + --echo ##################################################################################### + + let $MYSQLD_DATADIR= `select @@datadir`; + --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ + --exec $MYSQL_BINLOG --base64-output=decode-rows -v -v $MYSQLD_DATADIR/master-bin.000001 + + --echo # + --echo ##################################################################################### + --echo # mysqlbinlog --database=test1 + --echo # The following Annotate should appear in this output: + --echo # - DELETE test1.t1, test2.t2 FROM <...> + --echo ##################################################################################### + + let $MYSQLD_DATADIR= `select @@datadir`; + --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ + --exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v $MYSQLD_DATADIR/master-bin.000001 + + --echo # + --echo ##################################################################################### + --echo # mysqlbinlog --skip-annotate-row-events + --echo # No Annotates should appear in this output + --echo ##################################################################################### + + let $MYSQLD_DATADIR= `select @@datadir`; + --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ + --exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v $MYSQLD_DATADIR/master-bin.000001 + + --let $use_remote_mysqlbinlog= 0 +} + +--echo # +--echo ##################################################################################### +--echo # mysqlbinlog --read-from-remote-server +--echo # The following Annotates should appear in this output: +--echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) +--echo # - INSERT INTO test3.t3 VALUES (1), (2), (3) +--echo # - DELETE test1.t1, test2.t2 FROM <...> (with two subsequent Table maps) +--echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) +--echo # - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) +--echo ##################################################################################### + +let $MYSQLD_DATADIR= `select @@datadir`; +--replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ +--exec $MYSQL_BINLOG --base64-output=decode-rows -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 + +--echo # +--echo ##################################################################################### +--echo # mysqlbinlog --read-from-remote-server --database=test1 +--echo # The following Annotate should appear in this output: +--echo # - DELETE test1.t1, test2.t2 FROM <...> +--echo ##################################################################################### + +let $MYSQLD_DATADIR= `select @@datadir`; +--replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ +--exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 + +--echo # +--echo ##################################################################################### +--echo # mysqlbinlog --read-from-remote-server --skip-annotate-row-events +--echo # No Annotates should appear in this output +--echo ##################################################################################### + +let $MYSQLD_DATADIR= `select @@datadir`; +--replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ +--exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 + +# Clean-up + +--disable_query_log +DROP DATABASE test1; +DROP DATABASE test2; +DROP DATABASE test3; +DROP DATABASE xtest1; +DROP DATABASE xtest2; +--enable_query_log + diff --git a/mysql-test/extra/binlog_tests/binlog_write_error.inc b/mysql-test/extra/binlog_tests/binlog_write_error.inc new file mode 100644 index 0000000000000..da66dbf5a954e --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_write_error.inc @@ -0,0 +1,108 @@ +# +# This include file is used by more than one test suite +# (currently binlog and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# +# === Name === +# +# binlog_write_error.test +# +# === Description === +# +# This test case check if the error of writing binlog file is properly +# reported and handled when executing statements. +# +# === Related Bugs === +# +# BUG#37148 +# + +source include/have_log_bin.inc; +source include/have_debug.inc; +source include/have_binlog_format_mixed_or_statement.inc; + +--echo # +--echo # Initialization +--echo # + +disable_warnings; +DROP TABLE IF EXISTS t1, t2; +DROP FUNCTION IF EXISTS f1; +DROP FUNCTION IF EXISTS f2; +DROP PROCEDURE IF EXISTS p1; +DROP PROCEDURE IF EXISTS p2; +DROP TRIGGER IF EXISTS tr1; +DROP TRIGGER IF EXISTS tr2; +DROP VIEW IF EXISTS v1, v2; +enable_warnings; + +--echo # +--echo # Test injecting binlog write error when executing queries +--echo # + +let $query= CREATE TABLE t1 (a INT); +source include/binlog_inject_error.inc; + +INSERT INTO t1 VALUES (1),(2),(3); + +let $query= INSERT INTO t1 VALUES (4),(5),(6); +source include/binlog_inject_error.inc; + +let $query= UPDATE t1 set a=a+1; +source include/binlog_inject_error.inc; + +let $query= DELETE FROM t1; +source include/binlog_inject_error.inc; + +let $query= CREATE TRIGGER tr1 AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t1 VALUES (new.a + 100); +source include/binlog_inject_error.inc; + +let $query= DROP TRIGGER tr1; +source include/binlog_inject_error.inc; + +let $query= ALTER TABLE t1 ADD (b INT); +source include/binlog_inject_error.inc; + +let $query= CREATE VIEW v1 AS SELECT a FROM t1; +source include/binlog_inject_error.inc; + +let $query= DROP VIEW v1; +source include/binlog_inject_error.inc; + +let $query= CREATE PROCEDURE p1(OUT rows INT) SELECT count(*) INTO rows FROM t1; +source include/binlog_inject_error.inc; + +let $query= DROP PROCEDURE p1; +source include/binlog_inject_error.inc; + +let $query= DROP TABLE t1; +source include/binlog_inject_error.inc; + +let $query= CREATE FUNCTION f1() RETURNS INT return 1; +source include/binlog_inject_error.inc; + +let $query= DROP FUNCTION f1; +source include/binlog_inject_error.inc; + +let $query= CREATE USER user1; +source include/binlog_inject_error.inc; + +let $query= REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1; +source include/binlog_inject_error.inc; + +let $query= DROP USER user1; +source include/binlog_inject_error.inc; + +--echo # +--echo # Cleanup +--echo # + +disable_warnings; +DROP TABLE IF EXISTS t1, t2; +DROP FUNCTION IF EXISTS f1; +DROP PROCEDURE IF EXISTS p1; +DROP TRIGGER IF EXISTS tr1; +DROP VIEW IF EXISTS v1, v2; +enable_warnings; diff --git a/mysql-test/extra/binlog_tests/binlog_xa_recover.inc b/mysql-test/extra/binlog_tests/binlog_xa_recover.inc new file mode 100644 index 0000000000000..de2703377cc6a --- /dev/null +++ b/mysql-test/extra/binlog_tests/binlog_xa_recover.inc @@ -0,0 +1,281 @@ +# +# This include file is used by more than one test suite +# (currently binlog and binlog_encryption). +# Please check all dependent tests after modifying it +# + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_binlog_format_row.inc +# Valgrind does not work well with test that crashes the server +--source include/not_valgrind.inc + +# (We do not need to restore these settings, as we crash the server). +SET GLOBAL max_binlog_size= 4096; +SET GLOBAL innodb_flush_log_at_trx_commit= 1; +RESET MASTER; + +CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb; +# Insert some data to force a couple binlog rotations (3), so we get some +# normal binlog checkpoints before starting the test. +INSERT INTO t1 VALUES (100, REPEAT("x", 4100)); +# Wait for the master-bin.000002 binlog checkpoint to appear. +--let $wait_for_all= 0 +--let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000002" +--let $field= Info +--let $condition= = "master-bin.000002" +--source include/wait_show_condition.inc +INSERT INTO t1 VALUES (101, REPEAT("x", 4100)); +--let $wait_for_all= 0 +--let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000003" +--let $field= Info +--let $condition= = "master-bin.000003" +--source include/wait_show_condition.inc +INSERT INTO t1 VALUES (102, REPEAT("x", 4100)); +--let $wait_for_all= 0 +--let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000004" +--let $field= Info +--let $condition= = "master-bin.000004" +--source include/wait_show_condition.inc + +# Now start a bunch of transactions that span multiple binlog +# files. Leave then in the state prepared-but-not-committed in the engine +# and crash the server. Check that crash recovery is able to recover all +# of them. +# +# We use debug_sync to get all the transactions into the prepared state before +# we commit any of them. This is because the prepare step flushes the InnoDB +# redo log - including any commits made before, so recovery would become +# unnecessary, decreasing the value of this test. +# +# We arrange to have con1 with a prepared transaction in master-bin.000004, +# con2 and con3 with a prepared transaction in master-bin.000005, and a new +# empty master-bin.000006. So the latest binlog checkpoint should be +# master-bin.000006. + +connect(con1,localhost,root,,); +# First wait after prepare and before write to binlog. +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con1_wait WAIT_FOR con1_cont"; +# Then complete InnoDB commit in memory (but not commit checkpoint / write to +# disk), and hang until crash, leaving a transaction to be XA recovered. +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con1_ready WAIT_FOR _ever"; +send INSERT INTO t1 VALUES (1, REPEAT("x", 4100)); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con1_wait"; + +connect(con2,localhost,root,,); +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con2_wait WAIT_FOR con2_cont"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con2_ready WAIT_FOR _ever"; +send INSERT INTO t1 VALUES (2, NULL); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con2_wait"; + +connect(con3,localhost,root,,); +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con3_wait WAIT_FOR con3_cont"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con3_ready WAIT_FOR _ever"; +send INSERT INTO t1 VALUES (3, REPEAT("x", 4100)); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con3_wait"; + +connect(con4,localhost,root,,); +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con4_wait WAIT_FOR con4_cont"; +SET SESSION debug_dbug="+d,crash_commit_after_log"; +send INSERT INTO t1 VALUES (4, NULL); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con4_wait"; + +SET DEBUG_SYNC= "now SIGNAL con1_cont"; +SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; +SET DEBUG_SYNC= "now SIGNAL con2_cont"; +SET DEBUG_SYNC= "now WAIT_FOR con2_ready"; +SET DEBUG_SYNC= "now SIGNAL con3_cont"; +SET DEBUG_SYNC= "now WAIT_FOR con3_ready"; + +# Check that everything is committed in binary log. +--source include/show_binary_logs.inc +--let $binlog_file= master-bin.000003 +--let $binlog_start= 4 +--source include/show_binlog_events.inc +--let $binlog_file= master-bin.000004 +--source include/show_binlog_events.inc +--let $binlog_file= master-bin.000005 +--source include/show_binlog_events.inc +--let $binlog_file= master-bin.000006 +--source include/show_binlog_events.inc + + +# Check that server will not purge too much. +PURGE BINARY LOGS TO "master-bin.000006"; +--source include/show_binary_logs.inc + +# Now crash the server with one more transaction in prepared state. +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait-binlog_xa_recover.test +EOF +--error 0,2006,2013 +SET DEBUG_SYNC= "now SIGNAL con4_cont"; +connection con4; +--error 2006,2013 +reap; + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart-group_commit_binlog_pos.test +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Check that all transactions are recovered. +SELECT a FROM t1 ORDER BY a; + +--echo Test that with multiple binlog checkpoints, recovery starts from the last one. +SET GLOBAL max_binlog_size= 4096; +SET GLOBAL innodb_flush_log_at_trx_commit= 1; +RESET MASTER; + +# Rotate to binlog master-bin.000003 while delaying binlog checkpoints. +# So we get multiple binlog checkpoints in master-bin.000003. +# Then complete the checkpoints, crash, and check that we only scan +# the necessary binlog file (ie. that we use the _last_ checkpoint). + +connect(con10,localhost,root,,); +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con10_ready WAIT_FOR con10_cont"; +send INSERT INTO t1 VALUES (10, REPEAT("x", 4100)); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con10_ready"; + +connect(con11,localhost,root,,); +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con11_ready WAIT_FOR con11_cont"; +send INSERT INTO t1 VALUES (11, REPEAT("x", 4100)); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con11_ready"; + +connect(con12,localhost,root,,); +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con12_ready WAIT_FOR con12_cont"; +send INSERT INTO t1 VALUES (12, REPEAT("x", 4100)); + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con12_ready"; +INSERT INTO t1 VALUES (13, NULL); + +--source include/show_binary_logs.inc +--let $binlog_file= master-bin.000004 +--let $binlog_start= 4 +--source include/show_binlog_events.inc + +SET DEBUG_SYNC= "now SIGNAL con10_cont"; +connection con10; +reap; +connection default; + +# We need to sync the test case with the background processing of the +# commit checkpoint, otherwise we get nondeterministic results. +SET @old_dbug= @@global.DEBUG_DBUG; +SET GLOBAL debug_dbug="+d,binlog_background_checkpoint_processed"; + +SET DEBUG_SYNC= "now SIGNAL con12_cont"; +connection con12; +reap; +connection default; +SET DEBUG_SYNC= "now WAIT_FOR binlog_background_checkpoint_processed"; +SET GLOBAL debug_dbug= @old_dbug; + +SET DEBUG_SYNC= "now SIGNAL con11_cont"; +connection con11; +reap; + +connection default; +# Wait for the last (master-bin.000004) binlog checkpoint to appear. +--let $wait_for_all= 0 +--let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000004" +--let $field= Info +--let $condition= = "master-bin.000004" +--source include/wait_show_condition.inc + +--echo Checking that master-bin.000004 is the last binlog checkpoint +--source include/show_binlog_events.inc + +--echo Now crash the server +# It is not too easy to test XA recovery, as it runs early during server +# startup, before any connections can be made. +# What we do is set a DBUG error insert which will crash if XA recovery +# starts from any other binlog than master-bin.000004 (check the file +# binlog_xa_recover-master.opt). Then we will fail here if XA recovery +# would start from the wrong place. +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait-binlog_xa_recover.test +EOF +SET SESSION debug_dbug="+d,crash_commit_after_log"; +--error 2006,2013 +INSERT INTO t1 VALUES (14, NULL); + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart-group_commit_binlog_pos.test +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Check that all transactions are recovered. +SELECT a FROM t1 ORDER BY a; + + +--echo *** Check that recovery works if we crashed early during rotate, before +--echo *** binlog checkpoint event could be written. + +SET GLOBAL max_binlog_size= 4096; +SET GLOBAL innodb_flush_log_at_trx_commit= 1; +RESET MASTER; + +# We need some initial data to reach binlog master-bin.000004. Otherwise +# crash recovery fails due to the error insert used for previous test. +INSERT INTO t1 VALUES (21, REPEAT("x", 4100)); +INSERT INTO t1 VALUES (22, REPEAT("x", 4100)); +# Wait for the master-bin.000003 binlog checkpoint to appear. +--let $wait_for_all= 0 +--let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000003" +--let $field= Info +--let $condition= = "master-bin.000003" +--source include/wait_show_condition.inc +INSERT INTO t1 VALUES (23, REPEAT("x", 4100)); +# Wait for the last (master-bin.000004) binlog checkpoint to appear. +--let $wait_for_all= 0 +--let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000004" +--let $field= Info +--let $condition= = "master-bin.000004" +--source include/wait_show_condition.inc + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait-binlog_xa_recover.test +EOF +SET SESSION debug_dbug="+d,crash_before_write_checkpoint_event"; +--error 2006,2013 +INSERT INTO t1 VALUES (24, REPEAT("x", 4100)); + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart-group_commit_binlog_pos.test +EOF + +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Check that all transactions are recovered. +SELECT a FROM t1 ORDER BY a; + +--source include/show_binary_logs.inc +--let $binlog_file= master-bin.000004 +--let $binlog_start= 4 +--source include/show_binlog_events.inc + +# Cleanup +connection default; +DROP TABLE t1; diff --git a/mysql-test/extra/rpl_tests/multisource.inc b/mysql-test/extra/rpl_tests/multisource.inc new file mode 100644 index 0000000000000..83a7e6ad259d6 --- /dev/null +++ b/mysql-test/extra/rpl_tests/multisource.inc @@ -0,0 +1,313 @@ +# +# This include file is used by more than one test suite +# (currently multisource and binlog_encryption). +# Please check all dependent tests after modifying it +# +# Usage: +# --let $binlog_extra_length= X # optional, default 0 +# --source extra/rpl_tests/multisource.inc +# +# By default, the script expects the length of the 2nd binary log to be +# $binlog_start_pos + length(Gtid_list event) + 2 x length(Binlog_checkpoint event) +# Some tests can have specific configuration which would change it, +# e.g. for encrypted binlogs there will be additional event +# Start_encryption of the length of 36. +# binlog_extra_length should compensate for the difference. + +if (!$binlog_extra_length) +{ + --let $binlog_extra_length= 0 +} + +# +# Test basic replication functionality +# in multi-source setup +# + +--source include/not_embedded.inc +--source include/have_innodb.inc +--source include/binlog_start_pos.inc +--let $rpl_server_count= 0 + +--connect (slave,127.0.0.1,root,,,$SERVER_MYPORT_3) + +# MDEV-3984: crash/read of freed memory when changing master with named connection +# This fails after adding the new master 'abc', check we do not free twice. +--error ER_RELAY_LOG_INIT +change master 'abc' to relay_log_file=''; +# This fails before adding the new master, check that we do free it. +--error ER_WRONG_ARGUMENTS +change master 'abc2' to master_host=''; + + +# Start replication from the first master + +--replace_result $SERVER_MYPORT_1 MYPORT_1 +eval change master 'master1' to +master_port=$SERVER_MYPORT_1, +master_host='127.0.0.1', +master_user='root'; + +start slave 'master1'; +set default_master_connection = 'master1'; +--source include/wait_for_slave_to_start.inc + +--connect (master1,127.0.0.1,root,,,$SERVER_MYPORT_1) +--save_master_pos + +--connection slave +--sync_with_master 0,'master1' + +# Here and further: add an extra check on SQL thread status +# as the normal sync is not always enough +--source include/wait_for_sql_thread_read_all.inc + +# each of the 3 commands should produce +# 'master1' status + +let $wait_for_all= 1; +let $show_statement= SHOW ALL SLAVES STATUS; +let $field= Slave_IO_State; +let $condition= = 'Waiting for master to send event'; +--source include/wait_show_condition.inc + +--echo # +--echo # Checking SHOW SLAVE 'master1' STATUS +--echo # +--let $status_items= Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno +--let $slave_field_result_replace= /$SERVER_MYPORT_1/MYPORT_1/ +--let $slave_name= 'master1' +--source include/show_slave_status.inc +--let $slave_name= + +--echo # +--echo # Checking SHOW SLAVE STATUS +--echo # +--source include/show_slave_status.inc + +--echo # +--echo # Checking SHOW ALL SLAVES STATUS +--echo # +--let $all_slaves_status= 1 +--let $status_items= Connection_name, Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno, Slave_heartbeat_period +--source include/show_slave_status.inc +--let $all_slaves_status= +--echo # + + +# Check that replication actually works + +--connection master1 + +--disable_warnings +drop database if exists db1; +--enable_warnings +create database db1; +use db1; +create table t1 (i int auto_increment, f1 varchar(16), primary key pk (i,f1)) engine=MyISAM; +insert into t1 (f1) values ('one'),('two'); +--save_master_pos + +--connection slave +--sync_with_master 0,'master1' + +--sorted_result +select * from db1.t1; + +--let $datadir = `SELECT @@datadir` + +--echo # List of relay log files in the datadir +--list_files $datadir mysqld-relay-bin-master1.* + +# Check that relay logs are recognizable + +let binlog_start=4; +let binlog_file=; +source include/show_relaylog_events.inc; +let binlog_file= mysqld-relay-bin-master1.000002; +source include/show_relaylog_events.inc; + +# Try to configure connection with the same name again, +# should get an error because the slave is running + +--replace_result $SERVER_MYPORT_2 MYPORT_2 +--error ER_SLAVE_MUST_STOP +eval change master 'master1' to +master_port=$SERVER_MYPORT_2, +master_host='127.0.0.1', +master_user='root'; + +# Try to configure using the default connection name +# (which is 'master1' at the moment), +# again, should get an error + +--replace_result $SERVER_MYPORT_2 MYPORT_2 +--error ER_SLAVE_MUST_STOP +eval change master to +master_port=$SERVER_MYPORT_2, +master_host='127.0.0.1', +master_user='root'; + +# Try to configure a connection with the same master +# using a different name, should get a conflict + +--replace_result $SERVER_MYPORT_1 MYPORT_1 +--error ER_CONNECTION_ALREADY_EXISTS +eval change master 'master2' to +master_port=$SERVER_MYPORT_1, +master_host='127.0.0.1', +master_user='root'; + + +# Set up a proper 'default' connection to master2 + +set default_master_connection = ''; + +--replace_result $SERVER_MYPORT_2 MYPORT_2 +eval change master to +master_port=$SERVER_MYPORT_2, +master_host='127.0.0.1', +master_user='root'; + +start slave; +--source include/wait_for_slave_to_start.inc + +--source include/wait_for_sql_thread_read_all.inc + +# See both connections in the same status output + +let $wait_for_all= 1; +let $show_statement= SHOW ALL SLAVES STATUS; +let $field= Slave_IO_State; +let $condition= = 'Waiting for master to send event'; +--source include/wait_show_condition.inc + +--echo # +--echo # Checking SHOW ALL SLAVES STATUS +--echo # +--let $all_slaves_status= 1 +--let $status_items= Connection_name, Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno, Slave_heartbeat_period +--let $slave_field_result_replace= /$SERVER_MYPORT_1/MYPORT_1/ /$SERVER_MYPORT_2/MYPORT_2/ +--source include/show_slave_status.inc +--let $all_slaves_status= +--echo # + +# Check that replication from two servers actually works + +--connection master1 + +insert into t1 (f1) values ('three'); +--save_master_pos + +--connect (master2,127.0.0.1,root,,,$SERVER_MYPORT_2) + +--disable_warnings +drop database if exists db2; +--enable_warnings +create database db2; +use db2; +create table t1 (pk int auto_increment primary key, f1 int) engine=InnoDB; +begin; +insert into t1 (f1) values (1),(2); + +--connection slave +--sync_with_master 0,'master1' + +--connection master2 +--save_master_pos + +--connection slave +--sync_with_master 0 +--sorted_result +select * from db1.t1; +select * from db2.t1; + +--connection master2 +commit; +--save_master_pos + +--connection slave +--sync_with_master 0 +--sorted_result +select * from db2.t1; + +# Flush and purge logs on one master, +# make sure slaves don't get confused + +--connection master1 +flush logs; +--source include/wait_for_binlog_checkpoint.inc +--save_master_pos +--connection slave +--sync_with_master 0, 'master1' + +--connection master1 +purge binary logs to 'master-bin.000002'; +# Additional events: 39 (Gtid_list) + 2 x 40 (Binlog_checkpoint) = 119 +let filesize=`select $binlog_start_pos+119+$binlog_extra_length`; +--replace_result $filesize filesize +show binary logs; +insert into t1 (f1) values ('four'); +create table db1.t3 (f1 int) engine=InnoDB; +--save_master_pos + +--connection slave +--sync_with_master 0,'master1' + +--source include/wait_for_sql_thread_read_all.inc + +let $wait_for_all= 1; +let $show_statement= SHOW ALL SLAVES STATUS; +let $field= Slave_IO_State; +let $condition= = 'Waiting for master to send event'; +--source include/wait_show_condition.inc + +--echo # +--echo # Checking SHOW ALL SLAVES STATUS +--echo # +--let $all_slaves_status= 1 +--let $status_items= Connection_name, Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno, Slave_heartbeat_period +--let $slave_field_result_replace= /$SERVER_MYPORT_1/MYPORT_1/ /$SERVER_MYPORT_2/MYPORT_2/ +--source include/show_slave_status.inc +--let $all_slaves_status= +--echo # + +--sorted_result +select * from db1.t1; + +# This should show relay log events for the default master +# (the one with the empty name) +let binlog_file=; +source include/show_relaylog_events.inc; +let binlog_file= mysqld-relay-bin.000002; +source include/show_relaylog_events.inc; + +# Make sure we don't lose control over replication connections +# after reconnecting to the slave + +--disconnect slave +--connect (slave,127.0.0.1,root,,,$SERVER_MYPORT_3) + +stop slave io_thread; +show status like 'Slave_running'; +set default_master_connection = 'master1'; +show status like 'Slave_running'; + +# Cleanup + +drop database db1; +drop database db2; + +--source include/reset_master_slave.inc +--disconnect slave + +--connection master1 +drop database db1; +--source include/reset_master_slave.inc +--disconnect master1 + +--connection master2 +drop database db2; +--source include/reset_master_slave.inc +--disconnect master2 + diff --git a/mysql-test/extra/rpl_tests/rpl_binlog_errors.inc b/mysql-test/extra/rpl_tests/rpl_binlog_errors.inc new file mode 100644 index 0000000000000..98a9558ce0b1d --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_binlog_errors.inc @@ -0,0 +1,422 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# +# Usage: +# --let $binlog_limit= X[,Y] # optional +# +# Semantics of the value is the same as in include/show_binlog_events.inc +# which the script calls as a part of the test flow. +# The goal is to print the event demonstrating the triggered error, +# so normally Y should be 1 (print the exact event only); +# however, depending on test-specific server options, the offset X +# can be different. +# + +# BUG#46166: MYSQL_BIN_LOG::new_file_impl is not propagating error +# when generating new name. +# +# WHY +# === +# +# We want to check whether error is reported or not when +# new_file_impl fails (this may happen when rotation is not +# possible because there is some problem finding an +# unique filename). +# +# HOW +# === +# +# Test cases are documented inline. + +-- source include/have_innodb.inc +-- source include/have_debug.inc +-- source include/master-slave.inc + +-- echo ####################################################################### +-- echo ####################### PART 1: MASTER TESTS ########################## +-- echo ####################################################################### + + +### ACTION: stopping slave as it is not needed for the first part of +### the test + +-- connection slave +-- source include/stop_slave.inc +-- connection master + +call mtr.add_suppression("Can't generate a unique log-filename"); +call mtr.add_suppression("Writing one row to the row-based binary log failed.*"); +call mtr.add_suppression("Error writing file .*"); + +SET @old_debug= @@global.debug; + +### ACTION: create a large file (> 4096 bytes) that will be later used +### in LOAD DATA INFILE to check binlog errors in its vacinity +-- let $load_file= $MYSQLTEST_VARDIR/tmp/bug_46166.data +-- let $MYSQLD_DATADIR= `select @@datadir` +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SELECT repeat('x',8192) INTO OUTFILE '$load_file' + +### ACTION: create a small file (< 4096 bytes) that will be later used +### in LOAD DATA INFILE to check for absence of binlog errors +### when file loading this file does not force flushing and +### rotating the binary log +-- let $load_file2= $MYSQLTEST_VARDIR/tmp/bug_46166-2.data +-- let $MYSQLD_DATADIR= `select @@datadir` +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval SELECT repeat('x',10) INTO OUTFILE '$load_file2' + +RESET MASTER; + +-- echo ###################### TEST #1 + +### ASSERTION: no problem flushing logs (should show two binlogs) +FLUSH LOGS; +-- echo # assert: must show two binlogs +-- source include/show_binary_logs.inc + +-- echo ###################### TEST #2 + +### ASSERTION: check that FLUSH LOGS actually fails and reports +### failure back to the user if find_uniq_filename fails +### (should show just one binlog) + +RESET MASTER; +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +-- error ER_NO_UNIQUE_LOGFILE +FLUSH LOGS; +-- echo # assert: must show one binlog +-- source include/show_binary_logs.inc + +### ACTION: clean up and move to next test +SET GLOBAL debug_dbug=@old_debug; +RESET MASTER; + +-- echo ###################### TEST #3 + +### ACTION: create some tables (t1, t2, t4) and insert some values in +### table t1 +CREATE TABLE t1 (a INT); +CREATE TABLE t2 (a VARCHAR(16384)) Engine=InnoDB; +CREATE TABLE t4 (a VARCHAR(16384)); +INSERT INTO t1 VALUES (1); +RESET MASTER; + +### ASSERTION: we force rotation of the binary log because it exceeds +### the max_binlog_size option (should show two binary +### logs) + +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 + +# shows two binary logs +-- echo # assert: must show two binlog +-- source include/show_binary_logs.inc + +# clean up the table and the binlog to be used in next part of test +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; + +-- echo ###################### TEST #4 + +### ASSERTION: load the big file into a transactional table and check +### that it reports error. The table will contain the +### changes performed despite the fact that it reported an +### error. + +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- error ER_NO_UNIQUE_LOGFILE +-- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 + +# show table +-- echo # assert: must show one entry +SELECT count(*) FROM t2; + +# clean up the table and the binlog to be used in next part of test +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; + +-- echo ###################### TEST #5 + +### ASSERTION: load the small file into a transactional table and +### check that it succeeds + +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval LOAD DATA INFILE '$load_file2' INTO TABLE t2 + +# show table +-- echo # assert: must show one entry +SELECT count(*) FROM t2; + +# clean up the table and the binlog to be used in next part of test +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; + +-- echo ###################### TEST #6 + +### ASSERTION: check that even if one is using a transactional table +### and explicit transactions (no autocommit) if rotation +### fails we get the error. Transaction is not rolledback +### because rotation happens after the commit. + +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +SET AUTOCOMMIT=0; +INSERT INTO t2 VALUES ('muse'); +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 +INSERT INTO t2 VALUES ('muse'); +-- error ER_NO_UNIQUE_LOGFILE +COMMIT; + +### ACTION: Show the contents of the table after the test +-- echo # assert: must show three entries +SELECT count(*) FROM t2; + +### ACTION: clean up and move to the next test +SET AUTOCOMMIT= 1; +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; + +-- echo ###################### TEST #7 + +### ASSERTION: check that on a non-transactional table, if rotation +### fails then an error is reported and an incident event +### is written to the current binary log. + +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +SELECT count(*) FROM t4; +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- error ER_NO_UNIQUE_LOGFILE +-- eval LOAD DATA INFILE '$load_file' INTO TABLE t4 + +-- echo # assert: must show 1 entry +SELECT count(*) FROM t4; + +-- echo ### check that the incident event is written to the current log +SET GLOBAL debug_dbug=@old_debug; +if (!$binlog_limit) +{ + -- let $binlog_limit= 4,1 +} +-- source include/show_binlog_events.inc + +# clean up and move to next test +DELETE FROM t4; +RESET MASTER; + +-- echo ###################### TEST #8 + +### ASSERTION: check that statements end up in error but they succeed +### on changing the data. + +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +-- echo # must show 0 entries +SELECT count(*) FROM t4; +SELECT count(*) FROM t2; + +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- error ER_NO_UNIQUE_LOGFILE +-- eval LOAD DATA INFILE '$load_file' INTO TABLE t4 +-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +-- error ER_NO_UNIQUE_LOGFILE +-- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 +-- error ER_NO_UNIQUE_LOGFILE +INSERT INTO t2 VALUES ('aaa'), ('bbb'), ('ccc'); + +-- echo # INFO: Count(*) Before Offending DELETEs +-- echo # assert: must show 1 entry +SELECT count(*) FROM t4; +-- echo # assert: must show 4 entries +SELECT count(*) FROM t2; + +-- error ER_NO_UNIQUE_LOGFILE +DELETE FROM t4; +-- error ER_NO_UNIQUE_LOGFILE +DELETE FROM t2; + +-- echo # INFO: Count(*) After Offending DELETEs +-- echo # assert: must show zero entries +SELECT count(*) FROM t4; +SELECT count(*) FROM t2; + +# remove fault injection +SET GLOBAL debug_dbug=@old_debug; + +-- echo ###################### TEST #9 + +### ASSERTION: check that if we disable binlogging, then statements +### succeed. +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +SET SQL_LOG_BIN=0; +INSERT INTO t2 VALUES ('aaa'), ('bbb'), ('ccc'), ('ddd'); +INSERT INTO t4 VALUES ('eee'), ('fff'), ('ggg'), ('hhh'); +-- echo # assert: must show four entries +SELECT count(*) FROM t2; +SELECT count(*) FROM t4; +DELETE FROM t2; +DELETE FROM t4; +-- echo # assert: must show zero entries +SELECT count(*) FROM t2; +SELECT count(*) FROM t4; +SET SQL_LOG_BIN=1; +SET GLOBAL debug_dbug=@old_debug; + +-- echo ###################### TEST #10 + +### ASSERTION: check that error is reported if there is a failure +### while registering the index file and the binary log +### file or failure to write the rotate event. + +call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("Could not open .*"); + +RESET MASTER; +SHOW WARNINGS; + +# +d,fault_injection_registering_index => injects fault on MYSQL_BIN_LOG::open +SET GLOBAL debug_dbug="+d,fault_injection_registering_index"; +-- replace_regex /\.[\\\/]master/master/ +-- error ER_CANT_OPEN_FILE +FLUSH LOGS; +SET GLOBAL debug_dbug="-d,fault_injection_registering_index"; + +-- error ER_NO_BINARY_LOGGING +SHOW BINARY LOGS; + +# issue some statements and check that they don't fail +CREATE TABLE t5 (a INT); +INSERT INTO t4 VALUES ('bbbbb'); +INSERT INTO t2 VALUES ('aaaaa'); +DELETE FROM t4; +DELETE FROM t2; +DROP TABLE t5; + +-- echo ###################### TEST #11 + +### ASSERTION: check that error is reported if there is a failure +### while opening the index file and the binary log file or +### failure to write the rotate event. + +# restart the server so that we have binlog again +--let $rpl_server_number= 1 +--source include/rpl_restart_server.inc + +# +d,fault_injection_openning_index => injects fault on MYSQL_BIN_LOG::open_index_file +SET GLOBAL debug_dbug="+d,fault_injection_openning_index"; +-- replace_regex /\.[\\\/]master/master/ +-- error ER_CANT_OPEN_FILE +FLUSH LOGS; +SET GLOBAL debug_dbug="-d,fault_injection_openning_index"; + +-- error ER_FLUSH_MASTER_BINLOG_CLOSED +RESET MASTER; + +# issue some statements and check that they don't fail +CREATE TABLE t5 (a INT); +INSERT INTO t4 VALUES ('bbbbb'); +INSERT INTO t2 VALUES ('aaaaa'); +DELETE FROM t4; +DELETE FROM t2; +DROP TABLE t5; + +# restart the server so that we have binlog again +--let $rpl_server_number= 1 +--source include/rpl_restart_server.inc + +-- echo ###################### TEST #12 + +### ASSERTION: check that error is reported if there is a failure +### while writing the rotate event when creating a new log +### file. + +# +d,fault_injection_new_file_rotate_event => injects fault on MYSQL_BIN_LOG::MYSQL_BIN_LOG::new_file_impl +SET GLOBAL debug_dbug="+d,fault_injection_new_file_rotate_event"; +-- error ER_ERROR_ON_WRITE +FLUSH LOGS; +SET GLOBAL debug_dbug="-d,fault_injection_new_file_rotate_event"; + +-- error ER_FLUSH_MASTER_BINLOG_CLOSED +RESET MASTER; + +# issue some statements and check that they don't fail +CREATE TABLE t5 (a INT); +INSERT INTO t4 VALUES ('bbbbb'); +INSERT INTO t2 VALUES ('aaaaa'); +DELETE FROM t4; +DELETE FROM t2; +DROP TABLE t5; + +# restart the server so that we have binlog again +--let $rpl_server_number= 1 +--source include/rpl_restart_server.inc + +## clean up +DROP TABLE t1, t2, t4; +RESET MASTER; + +# restart slave again +-- connection slave +-- source include/start_slave.inc +-- connection master + +-- echo ####################################################################### +-- echo ####################### PART 2: SLAVE TESTS ########################### +-- echo ####################################################################### + +### setup +--source include/rpl_reset.inc +-- connection slave + +# slave suppressions + +call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); +call mtr.add_suppression("Error writing file .*"); +call mtr.add_suppression("Could not open .*"); +call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("Can't generate a unique log-filename .*"); +-- echo ###################### TEST #13 + +#### ASSERTION: check against unique log filename error +-- let $io_thd_injection_fault_flag= error_unique_log_filename +-- let $slave_io_errno= 1595 +-- let $show_slave_io_error= 1 +-- source include/io_thd_fault_injection.inc + +-- echo ###################### TEST #14 + +#### ASSERTION: check against rotate failing +-- let $io_thd_injection_fault_flag= fault_injection_new_file_rotate_event +-- let $slave_io_errno= 1595 +-- let $show_slave_io_error= 1 +-- source include/io_thd_fault_injection.inc + +-- echo ###################### TEST #15 + +#### ASSERTION: check against relay log open failure +-- let $io_thd_injection_fault_flag= fault_injection_registering_index +-- let $slave_io_errno= 1595 +-- let $show_slave_io_error= 1 +-- source include/io_thd_fault_injection.inc + +-- echo ###################### TEST #16 + +#### ASSERTION: check against relay log index open failure +-- let $io_thd_injection_fault_flag= fault_injection_openning_index +-- let $slave_io_errno= 1595 +-- let $show_slave_io_error= 1 +-- source include/io_thd_fault_injection.inc + +### clean up +-- source include/stop_slave_sql.inc +RESET SLAVE; +RESET MASTER; +--let $rpl_only_running_threads= 1 +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_cant_read_event_incident.inc b/mysql-test/extra/rpl_tests/rpl_cant_read_event_incident.inc new file mode 100644 index 0000000000000..a9534a999e2d1 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_cant_read_event_incident.inc @@ -0,0 +1,83 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# +# Bug#11747416 : 32228 A disk full makes binary log corrupt. +# +# +# The test demonstrates reading from binlog error propagation to slave +# and reporting there. +# Conditions for the bug include a crash at time of the last event to +# the binlog was written partly. With the fixes the event is not sent out +# any longer, but rather the dump thread sends out a sound error message. +# +# Crash is not simulated. A binlog with partly written event in its end is installed +# and replication is started from it. +# + +--source include/master-slave.inc +--source include/have_binlog_format_mixed.inc + +--connection slave +# Make sure the slave is stopped while we are messing with master. +# Otherwise we get occasional failures as the slave manages to re-connect +# to the newly started master and we get extra events applied, causing +# conflicts. +--source include/stop_slave.inc + +--connection master +call mtr.add_suppression("Error in Log_event::read_log_event()"); +--let $datadir= `SELECT @@datadir` + +--let $rpl_server_number= 1 +--source include/rpl_stop_server.inc + +--remove_file $datadir/master-bin.000001 +--copy_file $MYSQL_TEST_DIR/std_data/bug11747416_32228_binlog.000001 $datadir/master-bin.000001 + +--let $rpl_server_number= 1 +--source include/rpl_start_server.inc + +--source include/wait_until_connected_again.inc + +# evidence of the partial binlog +--error ER_ERROR_WHEN_EXECUTING_COMMAND +show binlog events; + +--connection slave +call mtr.add_suppression("Slave I/O: Got fatal error 1236 from master when reading data from binary log"); +reset slave; +start slave; + +# ER_MASTER_FATAL_ERROR_READING_BINLOG 1236 +--let $slave_param=Last_IO_Errno +--let $slave_param_value=1236 +--source include/wait_for_slave_param.inc + +--let $slave_field_result_replace= / at [0-9]*/ at XXX/ +--let $status_items= Last_IO_Errno, Last_IO_Error +--source include/show_slave_status.inc + +# +# Cleanup +# + +--connection master +reset master; + +--connection slave +stop slave; +reset slave; +# Table was created from binlog, it may not be created if SQL thread is running +# slowly and IO thread reaches incident before SQL thread applies it. +--disable_warnings +drop table if exists t; +--enable_warnings +reset master; + +--echo End of the tests +--let $rpl_only_running_threads= 1 +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_checksum.inc b/mysql-test/extra/rpl_tests/rpl_checksum.inc new file mode 100644 index 0000000000000..14664fd585a30 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_checksum.inc @@ -0,0 +1,333 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# WL2540 replication events checksum +# Testing configuration parameters + +--source include/master-slave.inc +--source include/have_debug.inc +--source include/have_binlog_format_mixed.inc + +call mtr.add_suppression('Slave can not handle replication events with the checksum that master is configured to log'); +call mtr.add_suppression('Replication event checksum verification failed'); +# due to C failure simulation +call mtr.add_suppression('Relay log write failure: could not queue event from master'); +call mtr.add_suppression('Master is configured to log replication events with checksum, but will not send such events to slaves that cannot process them'); + +# A. read/write access to the global vars: +# binlog_checksum master_verify_checksum slave_sql_verify_checksum + +connection master; + +set @master_save_binlog_checksum= @@global.binlog_checksum; +set @save_master_verify_checksum = @@global.master_verify_checksum; + +select @@global.binlog_checksum as 'must be CRC32 because of the command line option'; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.binlog_checksum as 'no session var'; + +select @@global.master_verify_checksum as 'must be zero because of default'; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.master_verify_checksum as 'no session var'; + +connection slave; + +set @slave_save_binlog_checksum= @@global.binlog_checksum; +set @save_slave_sql_verify_checksum = @@global.slave_sql_verify_checksum; + +select @@global.slave_sql_verify_checksum as 'must be one because of default'; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.slave_sql_verify_checksum as 'no session var'; + +connection master; + +source include/show_binary_logs.inc; +set @@global.binlog_checksum = NONE; +select @@global.binlog_checksum; +--echo *** must be rotations seen *** +source include/show_binary_logs.inc; + +set @@global.binlog_checksum = default; +select @@global.binlog_checksum; + +# testing lack of side-effects in non-effective update of binlog_checksum: +set @@global.binlog_checksum = CRC32; +select @@global.binlog_checksum; +set @@global.binlog_checksum = CRC32; + +set @@global.master_verify_checksum = 0; +set @@global.master_verify_checksum = default; + +--error ER_WRONG_VALUE_FOR_VAR +set @@global.binlog_checksum = ADLER32; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.master_verify_checksum = 2; # the var is of bool type + +connection slave; + +set @@global.slave_sql_verify_checksum = 0; +set @@global.slave_sql_verify_checksum = default; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.slave_sql_verify_checksum = 2; # the var is of bool type + +# +# B. Old Slave to New master conditions +# +# while master does not send a checksum-ed binlog the Old Slave can +# work with the New Master + +connection master; + +set @@global.binlog_checksum = NONE; +create table t1 (a int); + +# testing that binlog rotation preserves opt_binlog_checksum value +flush logs; +flush logs; +flush logs; + +sync_slave_with_master; +#connection slave; +# checking that rotation on the slave side leaves slave stable +flush logs; +flush logs; +flush logs; +select count(*) as zero from t1; + +source include/stop_slave.inc; + +connection master; +set @@global.binlog_checksum = CRC32; +-- source include/wait_for_binlog_checkpoint.inc +insert into t1 values (1) /* will not be applied on slave due to simulation */; + +# instruction to the dump thread + +connection slave; +set @@global.debug_dbug='d,simulate_slave_unaware_checksum'; +start slave; +--let $slave_io_errno= 1236 +--let $show_slave_io_error= 1 +source include/wait_for_slave_io_error.inc; + +select count(*) as zero from t1; + +###connection master; +set @@global.debug_dbug=''; + +connection slave; +source include/start_slave.inc; + +# +# C. checksum failure simulations +# + +# C1. Failure by a client thread +connection master; +set @@global.master_verify_checksum = 1; +set @@session.debug_dbug='d,simulate_checksum_test_failure'; +--error ER_ERROR_WHEN_EXECUTING_COMMAND +show binlog events; +set @@session.debug_dbug=''; +set @@global.master_verify_checksum = default; + +#connection master; +sync_slave_with_master; + +connection slave; +source include/stop_slave.inc; + +connection master; +create table t2 (a int); +let $pos_master= query_get_value(SHOW MASTER STATUS, Position, 1); + +connection slave; + +# C2. Failure by IO thread +# instruction to io thread +set @@global.debug_dbug='d,simulate_checksum_test_failure'; +start slave io_thread; +# When the checksum error is detected, the slave sets error code 1913 +# (ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE) in queue_event(), then immediately +# sets error 1595 (ER_SLAVE_RELAY_LOG_WRITE_FAILURE) in handle_slave_io(). +# So we usually get 1595, but it is occasionally possible to get 1913. +--let $slave_io_errno= 1595,1913 +--let $show_slave_io_error= 0 +source include/wait_for_slave_io_error.inc; +set @@global.debug_dbug=''; + +# to make IO thread re-read it again w/o the failure +start slave io_thread; +let $slave_param= Read_Master_Log_Pos; +let $slave_param_value= $pos_master; +source include/wait_for_slave_param.inc; + +# C3. Failure by SQL thread +# instruction to sql thread; +set @@global.slave_sql_verify_checksum = 1; + +set @@global.debug_dbug='d,simulate_checksum_test_failure'; + +start slave sql_thread; +--let $slave_sql_errno= 1593 +--let $show_slave_sql_error= 1 +source include/wait_for_slave_sql_error.inc; + +# resuming SQL thread to parse out the event w/o the failure + +set @@global.debug_dbug=''; +source include/start_slave.inc; + +connection master; +sync_slave_with_master; + +#connection slave; +select count(*) as 'must be zero' from t2; + +# +# D. Reset slave, Change-Master, Binlog & Relay-log rotations with +# random value on binlog_checksum on both master and slave +# +connection slave; +stop slave; +reset slave; + +# randomize slave server's own checksum policy +set @@global.binlog_checksum= IF(floor((rand()*1000)%2), "CRC32", "NONE"); +flush logs; + +connection master; +set @@global.binlog_checksum= CRC32; +reset master; +flush logs; +create table t3 (a int, b char(5)); + +connection slave; +source include/start_slave.inc; + +connection master; +sync_slave_with_master; + +#connection slave; +select count(*) as 'must be zero' from t3; +source include/stop_slave.inc; +--replace_result $MASTER_MYPORT MASTER_PORT +eval change master to master_host='127.0.0.1',master_port=$MASTER_MYPORT, master_user='root'; + +connection master; +flush logs; +reset master; +insert into t3 value (1, @@global.binlog_checksum); + +connection slave; +source include/start_slave.inc; +flush logs; + +connection master; +sync_slave_with_master; + +#connection slave; +select count(*) as 'must be one' from t3; + +connection master; +set @@global.binlog_checksum= IF(floor((rand()*1000)%2), "CRC32", "NONE"); +insert into t3 value (1, @@global.binlog_checksum); +sync_slave_with_master; + +#connection slave; + +#clean-up + +connection master; +drop table t1, t2, t3; +set @@global.binlog_checksum = @master_save_binlog_checksum; +set @@global.master_verify_checksum = @save_master_verify_checksum; + +# +# BUG#58564: flush_read_lock fails in mysql-trunk-bugfixing after merging with WL#2540 +# +# Sanity check that verifies that no assertions are triggered because +# of old FD events (generated by versions prior to server released with +# checksums feature) +# +# There is no need for query log, if something wrong this should trigger +# an assertion + +--disable_query_log + +BINLOG ' +MfmqTA8BAAAAZwAAAGsAAAABAAQANS41LjctbTMtZGVidWctbG9nAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAx+apMEzgNAAgAEgAEBAQEEgAAVAAEGggAAAAICAgCAA== +'; + +--enable_query_log + +#connection slave; +sync_slave_with_master; + + +--echo *** Bug#59123 / MDEV-5799: INCIDENT_EVENT checksum written to error log as garbage characters *** + +--connection master + +--source include/wait_for_binlog_checkpoint.inc +CREATE TABLE t4 (a INT PRIMARY KEY); +INSERT INTO t4 VALUES (1); + +SET sql_log_bin=0; +CALL mtr.add_suppression("\\[ERROR\\] Can't generate a unique log-filename"); +SET sql_log_bin=1; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET debug_dbug= '+d,binlog_inject_new_name_error'; +--error ER_NO_UNIQUE_LOGFILE +FLUSH LOGS; +SET debug_dbug= @old_dbug; + +INSERT INTO t4 VALUES (2); + +--connection slave +--let $slave_sql_errno= 1590 +--source include/wait_for_slave_sql_error.inc + +# Search the error log for the error message. +# The bug was that 4 garbage bytes were output in the middle of the error +# message; by searching for a pattern that spans that location, we can +# catch the error. +let $log_error_= `SELECT @@GLOBAL.log_error`; +if(!$log_error_) +{ + # MySQL Server on windows is started with --console and thus + # does not know the location of its .err log, use default location + let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.2.err; +} +--let SEARCH_FILE= $log_error_ +--let SEARCH_RANGE=-50000 +--let SEARCH_PATTERN= Slave SQL: The incident LOST_EVENTS occurred on the master\. Message: error writing to the binary log, Internal MariaDB error code: 1590 +--source include/search_pattern_in_file.inc + +SELECT * FROM t4 ORDER BY a; +STOP SLAVE IO_THREAD; +SET sql_slave_skip_counter= 1; +--source include/start_slave.inc + +--connection master +--save_master_pos + +--connection slave +--sync_with_master +SELECT * FROM t4 ORDER BY a; + + +--connection slave +set @@global.binlog_checksum = @slave_save_binlog_checksum; +set @@global.slave_sql_verify_checksum = @save_slave_sql_verify_checksum; + +--echo End of tests + +--connection master +DROP TABLE t4; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_checksum_cache.inc b/mysql-test/extra/rpl_tests/rpl_checksum_cache.inc new file mode 100644 index 0000000000000..a10c9721f700e --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_checksum_cache.inc @@ -0,0 +1,261 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +-- source include/have_innodb.inc +-- source include/master-slave.inc + +--disable_warnings +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. Statement: insert into t2 set data=repeat.*'a', @act_size.*"); +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. Statement: insert into t1 values.* NAME_CONST.*'n',.*, @data .*"); +--enable_warnings + +connection master; +set @save_binlog_cache_size = @@global.binlog_cache_size; +set @save_binlog_checksum = @@global.binlog_checksum; +set @save_master_verify_checksum = @@global.master_verify_checksum; +set @@global.binlog_cache_size = 4096; +set @@global.binlog_checksum = CRC32; +set @@global.master_verify_checksum = 1; + +# restart slave to force the dump thread to verify events (on master side) +connection slave; +source include/stop_slave.inc; +source include/start_slave.inc; + +connection master; + +# +# Testing a critical part of checksum handling dealing with transaction cache. +# The cache's buffer size is set to be less than the transaction's footprint +# in binlog. +# +# To verify combined buffer-by-buffer read out of the file and fixing crc per event +# there are the following parts: +# +# 1. the event size is much less than the cache's buffer +# 2. the event size is bigger than the cache's buffer +# 3. the event size if approximately the same as the cache's buffer +# 4. all in above + +# +# 1. the event size is much less than the cache's buffer +# + +flush status; +show status like "binlog_cache_use"; +show status like "binlog_cache_disk_use"; +--disable_warnings +drop table if exists t1; +--enable_warnings + +# +# parameter to ensure the test slightly varies binlog content +# between different invocations +# +let $deviation_size=32; +eval create table t1 (a int PRIMARY KEY, b CHAR($deviation_size)) engine=innodb; + +# Now we are going to create transaction which is long enough so its +# transaction binlog will be flushed to disk... + +delimiter |; +create procedure test.p_init (n int, size int) +begin + while n > 0 do + select round(RAND() * size) into @act_size; + set @data = repeat('a', @act_size); + insert into t1 values(n, @data ); + set n= n-1; + end while; +end| + +delimiter ;| + +let $1 = 4000; # PB2 can run it slow to time out on following sync_slave_with_master:s + +begin; +--disable_warnings +# todo: check if it is really so. +#+Note 1592 Unsafe statement binlogged in statement format since BINLOG_FORMAT = STATEMENT. Reason for unsafeness: Statement uses a system function whose value may differ on slave. +eval call test.p_init($1, $deviation_size); +--enable_warnings +commit; + +show status like "binlog_cache_use"; +--echo *** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; + +sync_slave_with_master; + +let $diff_tables=master:test.t1, slave:test.t1; +source include/diff_tables.inc; + +# undoing changes with verifying the above once again +connection master; + +begin; +delete from t1; +commit; + +sync_slave_with_master; + + +# +# 2. the event size is bigger than the cache's buffer +# +connection master; + +flush status; +let $t2_data_size= `select 3 * @@global.binlog_cache_size`; +let $t2_aver_size= `select 2 * @@global.binlog_cache_size`; +let $t2_max_rand= `select 1 * @@global.binlog_cache_size`; + +eval create table t2(a int auto_increment primary key, data VARCHAR($t2_data_size)) ENGINE=Innodb; +let $1=100; +--disable_query_log +begin; +while ($1) +{ + eval select round($t2_aver_size + RAND() * $t2_max_rand) into @act_size; + set @data = repeat('a', @act_size); + insert into t2 set data = @data; + dec $1; +} +commit; +--enable_query_log +show status like "binlog_cache_use"; +--echo *** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; + +sync_slave_with_master; + +let $diff_tables=master:test.t2, slave:test.t2; +source include/diff_tables.inc; + +# undoing changes with verifying the above once again +connection master; + +begin; +delete from t2; +commit; + +sync_slave_with_master; + +# +# 3. the event size if approximately the same as the cache's buffer +# + +connection master; + +flush status; +let $t3_data_size= `select 2 * @@global.binlog_cache_size`; +let $t3_aver_size= `select (9 * @@global.binlog_cache_size) / 10`; +let $t3_max_rand= `select (2 * @@global.binlog_cache_size) / 10`; + +eval create table t3(a int auto_increment primary key, data VARCHAR($t3_data_size)) engine=innodb; + +let $1= 300; +--disable_query_log +begin; +while ($1) +{ + eval select round($t3_aver_size + RAND() * $t3_max_rand) into @act_size; + insert into t3 set data= repeat('a', @act_size); + dec $1; +} +commit; +--enable_query_log +show status like "binlog_cache_use"; +--echo *** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; + +sync_slave_with_master; + +let $diff_tables=master:test.t3, slave:test.t3; +source include/diff_tables.inc; + +# undoing changes with verifying the above once again +connection master; + +begin; +delete from t3; +commit; + +sync_slave_with_master; + + +# +# 4. all in above +# + +connection master; +flush status; + +delimiter |; +eval create procedure test.p1 (n int) +begin + while n > 0 do + case (select (round(rand()*100) % 3) + 1) + when 1 then + select round(RAND() * $deviation_size) into @act_size; + set @data = repeat('a', @act_size); + insert into t1 values(n, @data); + when 2 then + begin + select round($t2_aver_size + RAND() * $t2_max_rand) into @act_size; + insert into t2 set data=repeat('a', @act_size); + end; + when 3 then + begin + select round($t3_aver_size + RAND() * $t3_max_rand) into @act_size; + insert into t3 set data= repeat('a', @act_size); + end; + end case; + set n= n-1; + end while; +end| +delimiter ;| + +let $1= 1000; +set autocommit= 0; +begin; +--disable_warnings +eval call test.p1($1); +--enable_warnings +commit; + +show status like "binlog_cache_use"; +--echo *** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; + +sync_slave_with_master; + +let $diff_tables=master:test.t1, slave:test.t1; +source include/diff_tables.inc; + +let $diff_tables=master:test.t2, slave:test.t2; +source include/diff_tables.inc; + +let $diff_tables=master:test.t3, slave:test.t3; +source include/diff_tables.inc; + + +connection master; + +begin; +delete from t1; +delete from t2; +delete from t3; +commit; + +drop table t1, t2, t3; +set @@global.binlog_cache_size = @save_binlog_cache_size; +set @@global.binlog_checksum = @save_binlog_checksum; +set @@global.master_verify_checksum = @save_master_verify_checksum; +drop procedure test.p_init; +drop procedure test.p1; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_corruption.inc b/mysql-test/extra/rpl_tests/rpl_corruption.inc new file mode 100644 index 0000000000000..048a9d74249a6 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_corruption.inc @@ -0,0 +1,176 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +############################################################ +# Purpose: WL#5064 Testing with corrupted events. +# The test emulates the corruption at the vary stages +# of replication: +# - in binlog file +# - in network +# - in relay log +############################################################ + +# +# The tests intensively utilize @@global.debug. Note, +# Bug#11765758 - 58754, +# @@global.debug is read by the slave threads through dbug-interface. +# Hence, before a client thread set @@global.debug we have to ensure that: +# (a) the slave threads are stopped, or (b) the slave threads are in +# sync and waiting. + +--source include/have_debug.inc +--source include/master-slave.inc + +# Block legal errors for MTR +call mtr.add_suppression('Found invalid event in binary log'); +call mtr.add_suppression('Slave I/O: Relay log write failure: could not queue event from master'); +call mtr.add_suppression('event read from binlog did not pass crc check'); +call mtr.add_suppression('Replication event checksum verification failed'); +call mtr.add_suppression('Event crc check failed! Most likely there is event corruption'); +call mtr.add_suppression('Slave SQL: Error initializing relay log position: I/O error reading event at position .*, error.* 1593'); + +SET @old_master_verify_checksum = @@master_verify_checksum; + +# Creating test table/data and set corruption position for testing +--echo # 1. Creating test table/data and set corruption position for testing +--connection master +--echo * insert/update/delete rows in table t1 * +# Corruption algorithm modifies only the first event and +# then will be reset. To avoid checking always the first event +# from binlog (usually it is FD) we randomly execute different +# statements and set position for corruption inside events. + +CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY, b VARCHAR(10), c VARCHAR(100)); +--disable_query_log +let $i=`SELECT 3+CEILING(10*RAND())`; +let $j=1; +let $pos=0; +while ($i) { + eval INSERT INTO t1 VALUES ($j, 'a', NULL); + if (`SELECT RAND() > 0.7`) + { + eval UPDATE t1 SET c = REPEAT('a', 20) WHERE a = $j; + } + if (`SELECT RAND() > 0.8`) + { + eval DELETE FROM t1 WHERE a = $j; + } + if (!$pos) { + let $pos= query_get_value(SHOW MASTER STATUS, Position, 1); + --sync_slave_with_master + --source include/stop_slave.inc + --disable_query_log + --connection master + } + dec $i; + inc $j; +} +--enable_query_log + + +# Emulate corruption in binlog file when SHOW BINLOG EVENTS is executing +--echo # 2. Corruption in master binlog and SHOW BINLOG EVENTS +SET GLOBAL debug_dbug="+d,corrupt_read_log_event_char"; +--echo SHOW BINLOG EVENTS; +--disable_query_log +send_eval SHOW BINLOG EVENTS FROM $pos; +--enable_query_log +--error ER_ERROR_WHEN_EXECUTING_COMMAND +reap; + +SET GLOBAL debug_dbug="-d,corrupt_read_log_event_char"; + +# Emulate corruption on master with crc checking on master +--echo # 3. Master read a corrupted event from binlog and send the error to slave + +# We have a rare but nasty potential race here: if the dump thread on +# the master for the _old_ slave connection has not yet discovered +# that the slave has disconnected, we will inject the corrupt event on +# the wrong connection, and the test will fail +# (+d,corrupt_read_log_event2 corrupts only one event). +# So kill any lingering dump thread (we need to kill; otherwise dump thread +# could manage to send all events down the socket before seeing it close, and +# hang forever waiting for new binlog events to be created). +let $id= `select id from information_schema.processlist where command = "Binlog Dump"`; +if ($id) +{ + --disable_query_log + --error 0,1094 + eval kill $id; + --enable_query_log +} +let $wait_condition= + SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE command = 'Binlog Dump'; +--source include/wait_condition.inc + +SET GLOBAL debug_dbug="+d,corrupt_read_log_event2_set"; +--connection slave +START SLAVE IO_THREAD; +let $slave_io_errno= 1236; +--let $slave_timeout= 10 +--source include/wait_for_slave_io_error.inc +--connection master +SET GLOBAL debug_dbug="-d,corrupt_read_log_event2_set"; + +# Emulate corruption on master without crc checking on master +--echo # 4. Master read a corrupted event from binlog and send it to slave +--connection master +SET GLOBAL master_verify_checksum=0; +SET GLOBAL debug_dbug="+d,corrupt_read_log_event2_set"; +--connection slave +START SLAVE IO_THREAD; +# When the checksum error is detected, the slave sets error code 1913 +# (ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE) in queue_event(), then immediately +# sets error 1595 (ER_SLAVE_RELAY_LOG_WRITE_FAILURE) in handle_slave_io(). +# So we usually get 1595, but it is occasionally possible to get 1913. +let $slave_io_errno= 1595,1913; +--source include/wait_for_slave_io_error.inc +--connection master +SET GLOBAL debug_dbug="-d,corrupt_read_log_event2_set"; +SET GLOBAL debug_dbug= ""; +SET GLOBAL master_verify_checksum=1; + +# Emulate corruption in network +--echo # 5. Slave. Corruption in network +--connection slave +SET GLOBAL debug_dbug="+d,corrupt_queue_event"; +START SLAVE IO_THREAD; +let $slave_io_errno= 1595,1913; +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,corrupt_queue_event"; + +# Emulate corruption in relay log +--echo # 6. Slave. Corruption in relay log + +SET GLOBAL debug_dbug="+d,corrupt_read_log_event_char"; + +START SLAVE SQL_THREAD; +let $slave_sql_errno= 1593; +--source include/wait_for_slave_sql_error.inc + +SET GLOBAL debug_dbug="-d,corrupt_read_log_event_char"; +SET GLOBAL debug_dbug= ""; + +# Start normal replication and compare same table on master +# and slave +--echo # 7. Seek diff for tables on master and slave +--connection slave +--source include/start_slave.inc +--connection master +--sync_slave_with_master +let $diff_tables= master:test.t1, slave:test.t1; +--source include/diff_tables.inc + +# Clean up +--echo # 8. Clean up +--connection master +SET GLOBAL debug_dbug= ""; +SET GLOBAL master_verify_checksum = @old_master_verify_checksum; +DROP TABLE t1; +--sync_slave_with_master +SET GLOBAL debug_dbug= ""; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_gtid_basic.inc b/mysql-test/extra/rpl_tests/rpl_gtid_basic.inc new file mode 100644 index 0000000000000..ab7d23f70acc0 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_gtid_basic.inc @@ -0,0 +1,568 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +--source include/have_innodb.inc +--let $rpl_topology=1->2->3->4 +--source include/rpl_init.inc + +# Set up a 4-deep replication topology, then test various fail-overs +# using GTID. +# +# A -> B -> C -> D + +connection server_1; +--source include/wait_for_binlog_checkpoint.inc +--let $binlog_file = query_get_value(SHOW MASTER STATUS,File,1) +--let $binlog_pos = query_get_value(SHOW MASTER STATUS,Position,1) +--echo *** GTID position should be empty here *** +--replace_result $binlog_file $binlog_pos +eval SELECT BINLOG_GTID_POS('$binlog_file',$binlog_pos); + +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=MyISAM; +CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, "m1"); +INSERT INTO t1 VALUES (2, "m2"), (3, "m3"), (4, "m4"); +INSERT INTO t2 VALUES (1, "i1"); +BEGIN; +INSERT INTO t2 VALUES (2, "i2"), (3, "i3"); +INSERT INTO t2 VALUES (4, "i4"); +COMMIT; +save_master_pos; +source include/wait_for_binlog_checkpoint.inc; +--let $binlog_file = query_get_value(SHOW MASTER STATUS,File,1) +--let $binlog_pos = query_get_value(SHOW MASTER STATUS,Position,1) +--let $gtid_pos_server_1 = `SELECT @@gtid_binlog_pos` +--echo *** GTID position should be non-empty here *** +--replace_result $binlog_file $binlog_pos $gtid_pos_server_1 +eval SELECT BINLOG_GTID_POS('$binlog_file',$binlog_pos); + +connection server_2; +sync_with_master; +source include/wait_for_binlog_checkpoint.inc; +--let $binlog_file = query_get_value(SHOW MASTER STATUS,File,1) +--let $binlog_pos = query_get_value(SHOW MASTER STATUS,Position,1) +--echo *** GTID position should be the same as on server_1 *** +--replace_result $binlog_file $binlog_pos $gtid_pos_server_1 +eval SELECT BINLOG_GTID_POS('$binlog_file',$binlog_pos); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +save_master_pos; + +connection server_3; +sync_with_master; +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +save_master_pos; + +connection server_4; +sync_with_master; +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + + +--echo *** Now take out D, let it fall behind a bit, and then test re-attaching it to A *** +connection server_4; +--source include/stop_slave.inc + +connection server_1; +INSERT INTO t1 VALUES (5, "m1a"); +INSERT INTO t2 VALUES (5, "i1a"); +save_master_pos; + +connection server_4; +--replace_result $MASTER_MYPORT MASTER_PORT +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT, + MASTER_USE_GTID=CURRENT_POS; +--source include/start_slave.inc +sync_with_master; +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Now move B to D (C is still replicating from B) *** +connection server_2; +--source include/stop_slave.inc +--replace_result $SERVER_MYPORT_4 SERVER_MYPORT_4 +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SERVER_MYPORT_4, + MASTER_USE_GTID=CURRENT_POS; +--source include/start_slave.inc + +connection server_4; +UPDATE t2 SET b="j1a" WHERE a=5; +save_master_pos; + +connection server_2; +sync_with_master; +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Now move C to D, after letting it fall a little behind *** +connection server_3; +--source include/stop_slave.inc + +connection server_1; +INSERT INTO t2 VALUES (6, "i6b"); +INSERT INTO t2 VALUES (7, "i7b"); +--source include/save_master_gtid.inc + +connection server_3; +--replace_result $SERVER_MYPORT_4 SERVER_MYPORT_4 +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SERVER_MYPORT_4, + MASTER_USE_GTID=CURRENT_POS; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t2 ORDER BY a; + +--echo *** Now change everything back to what it was, to make rpl_end.inc happy +# Also check that MASTER_USE_GTID=CURRENT_POS is still enabled. +connection server_2; +# We need to sync up server_2 before switching. If it happened to have reached +# the point 'UPDATE t2 SET b="j1a" WHERE a=5' it will fail to connect to +# server_1, which is (deliberately) missing that transaction. +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +--replace_result $MASTER_MYPORT MASTER_MYPORT +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT; +--source include/start_slave.inc +--source include/wait_for_slave_to_start.inc + +connection server_3; +--source include/stop_slave.inc +--replace_result $SLAVE_MYPORT SLAVE_MYPORT +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SLAVE_MYPORT; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +connection server_4; +--source include/stop_slave.inc +--replace_result $SERVER_MYPORT_3 SERVER_MYPORT_3 +eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SERVER_MYPORT_3; +--source include/start_slave.inc + +connection server_1; +DROP TABLE t1,t2; +--source include/save_master_gtid.inc + +--echo *** A few more checks for BINLOG_GTID_POS function *** +--let $valid_binlog_name = query_get_value(SHOW BINARY LOGS,Log_name,1) +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT BINLOG_GTID_POS(); +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT BINLOG_GTID_POS('a'); +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT BINLOG_GTID_POS('a',1,NULL); +SELECT BINLOG_GTID_POS(1,'a'); +SELECT BINLOG_GTID_POS(NULL,NULL); +SELECT BINLOG_GTID_POS('',1); +SELECT BINLOG_GTID_POS('a',1); +eval SELECT BINLOG_GTID_POS('$valid_binlog_name',-1); +eval SELECT BINLOG_GTID_POS('$valid_binlog_name',0); +eval SELECT BINLOG_GTID_POS('$valid_binlog_name',18446744073709551615); +eval SELECT BINLOG_GTID_POS('$valid_binlog_name',18446744073709551616); + + +--echo *** Some tests of @@GLOBAL.gtid_binlog_state *** +--connection server_2 +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc + +--connection server_1 +SET @old_state= @@GLOBAL.gtid_binlog_state; + +--error ER_BINLOG_MUST_BE_EMPTY +SET GLOBAL gtid_binlog_state = ''; +RESET MASTER; +SET GLOBAL gtid_binlog_state = ''; +FLUSH LOGS; +--source include/show_binary_logs.inc +SET GLOBAL gtid_binlog_state = '0-1-10,1-2-20,0-3-30'; +--source include/show_binary_logs.inc +--let $binlog_file= master-bin.000001 +--let $binlog_start= 4 +--source include/show_binlog_events.inc +#SELECT @@GLOBAL.gtid_binlog_pos; +#SELECT @@GLOBAL.gtid_binlog_state; +--error ER_BINLOG_MUST_BE_EMPTY +SET GLOBAL gtid_binlog_state = @old_state; +RESET MASTER; +SET GLOBAL gtid_binlog_state = @old_state; + +# Check that slave can reconnect again, despite the RESET MASTER, as we +# restored the state. + +CREATE TABLE t1 (a INT PRIMARY KEY); +SET gtid_seq_no=100; +INSERT INTO t1 VALUES (1); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/start_slave.inc +# We cannot just use sync_with_master as we've done RESET MASTER, so +# slave old-style position is wrong. +# So sync on gtid position instead. +--source include/sync_with_master_gtid.inc + +SELECT * FROM t1; +# Check that the IO gtid position in SHOW SLAVE STATUS is also correct. +--let $status_items= Gtid_IO_Pos +--source include/show_slave_status.inc + +--echo *** Test @@LAST_GTID and MASTER_GTID_WAIT() *** + +--connection server_1 +DROP TABLE t1; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +--save_master_pos + +--connection server_2 +--sync_with_master +--source include/stop_slave.inc + +--connect (m1,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +SELECT @@last_gtid; +SET gtid_seq_no=110; +SELECT @@last_gtid; +BEGIN; +SELECT @@last_gtid; +INSERT INTO t1 VALUES (2); +SELECT @@last_gtid; +COMMIT; +SELECT @@last_gtid; +--let $pos= `SELECT @@gtid_binlog_pos` + +--connect (s1,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +eval SET @pos= '$pos'; +# Check NULL argument. +SELECT master_gtid_wait(NULL); +# Check empty argument returns immediately. +SELECT master_gtid_wait('', NULL); +# Check this gets counted +SHOW STATUS LIKE 'Master_gtid_wait_count'; +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +SHOW STATUS LIKE 'Master_gtid_wait_time'; +# Let's check that we get a timeout +SELECT master_gtid_wait(@pos, 0.5); +SELECT * FROM t1 ORDER BY a; +# Now actually wait until the slave reaches the position +send SELECT master_gtid_wait(@pos); + +--connection server_2 +--source include/start_slave.inc + +--connection s1 +reap; +SELECT * FROM t1 ORDER BY a; + +# Test waiting on a domain that does not exist yet. +--source include/stop_slave.inc + +--connection server_1 +SET gtid_domain_id= 1; +INSERT INTO t1 VALUES (3); +--let $pos= `SELECT @@gtid_binlog_pos` + +--connection s1 +--replace_result $pos POS +eval SET @pos= '$pos'; +SELECT master_gtid_wait(@pos, 0); +SELECT * FROM t1 WHERE a >= 3; +send SELECT master_gtid_wait(@pos, -1); + +--connection server_2 +--source include/start_slave.inc + +--connection s1 +reap; +SELECT * FROM t1 WHERE a >= 3; +# Waiting for only part of the position. +SELECT master_gtid_wait('1-1-1', 0); + +# Now test a lot of parallel master_gtid_wait() calls, completing in different +# order, and some of which time out or get killed on the way. + +--connection s1 +send SELECT master_gtid_wait('2-1-1,1-1-4,0-1-110'); + +--connect (s2,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +# This will time out. No event 0-1-1000 exists +send SELECT master_gtid_wait('0-1-1000', 0.5); + +--connect (s3,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +# This one we will kill +--let $kill1_id= `SELECT connection_id()` +send SELECT master_gtid_wait('0-1-2000'); + +--connect (s4,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +send SELECT master_gtid_wait('2-1-10'); + +--connect (s5,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +send SELECT master_gtid_wait('2-1-6', 1); + +# This one we will kill also. +--connect (s6,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +--let $kill2_id= `SELECT connection_id()` +send SELECT master_gtid_wait('2-1-5'); + +--connect (s7,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +send SELECT master_gtid_wait('2-1-10'); + +--connect (s8,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +send SELECT master_gtid_wait('2-1-5,1-1-4,0-1-110'); + +--connect (s9,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +send SELECT master_gtid_wait('2-1-2'); + +--connection server_2 +# This one completes immediately. +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +SHOW STATUS LIKE 'Master_gtid_wait_count'; +SELECT master_gtid_wait('1-1-1'); +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +SHOW STATUS LIKE 'Master_gtid_wait_count'; +let $wait_time = query_get_value(SHOW STATUS LIKE 'Master_gtid_wait_time', Value, 1); +--replace_result $wait_time MASTER_GTID_WAIT_TIME +eval SET @a= $wait_time; +SELECT IF(@a <= 100*1000*1000, "OK", CONCAT("Error: wait time ", @a, " is larger than expected")) + AS Master_gtid_wait_time_as_expected; + + +--connect (s10,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +send SELECT master_gtid_wait('0-1-109'); + +--connection server_2 +# This one should time out. +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +SHOW STATUS LIKE 'Master_gtid_wait_count'; +SELECT master_gtid_wait('2-1-2', 0.5); +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +SHOW STATUS LIKE 'Master_gtid_wait_count'; +let $wait_time = query_get_value(SHOW STATUS LIKE 'Master_gtid_wait_time', Value, 1); +--replace_result $wait_time MASTER_GTID_WAIT_TIME +eval SET @a= $wait_time; +# We expect a wait time of just a bit over 0.5 seconds. But thread scheduling +# and timer inaccuracies could introduce significant jitter. So allow a +# generous interval. +SELECT IF(@a BETWEEN 0.4*1000*1000 AND 100*1000*1000, "OK", CONCAT("Error: wait time ", @a, " not as expected")) AS Master_gtid_wait_time_as_expected; + +--replace_result $kill1_id KILL_ID +eval KILL QUERY $kill1_id; +--connection s3 +--error ER_QUERY_INTERRUPTED +reap; + +--connection server_1 +SET gtid_domain_id=2; +SET gtid_seq_no=2; +INSERT INTO t1 VALUES (4); + +--connection s9 +reap; + +--connection server_2 +--replace_result $kill2_id KILL_ID +eval KILL CONNECTION $kill2_id; + +--connection s6 +--error 2013,ER_CONNECTION_KILLED +reap; + +--connection server_1 +SET gtid_domain_id=1; +SET gtid_seq_no=4; +INSERT INTO t1 VALUES (5); +SET gtid_domain_id=2; +SET gtid_seq_no=5; +INSERT INTO t1 VALUES (6); + +--connection s8 +reap; +--connection s1 +reap; +--connection s2 +reap; +--connection s5 +reap; +--connection s10 +reap; + +--connection server_1 +SET gtid_domain_id=2; +SET gtid_seq_no=10; +INSERT INTO t1 VALUES (7); + +--connection s4 +reap; +--connection s7 +reap; + + +--echo *** Test gtid_slave_pos when used with GTID *** + +--connection server_2 +--source include/stop_slave.inc + +--connection server_1 +SET gtid_domain_id=2; +SET gtid_seq_no=1000; +INSERT INTO t1 VALUES (10); +INSERT INTO t1 VALUES (11); +--save_master_pos + +--connection server_2 +SET sql_slave_skip_counter= 1; +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +SELECT IF(LOCATE("2-1-1001", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1001 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + +--source include/stop_slave.inc + +--connection server_1 +SET gtid_domain_id=2; +SET gtid_seq_no=1010; +INSERT INTO t1 VALUES (12); +INSERT INTO t1 VALUES (13); +--save_master_pos + +--connection server_2 +SET sql_slave_skip_counter= 2; +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +SELECT IF(LOCATE("2-1-1011", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1011 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + +--source include/stop_slave.inc + +--connection server_1 +SET gtid_domain_id=2; +SET gtid_seq_no=1020; +INSERT INTO t1 VALUES (14); +INSERT INTO t1 VALUES (15); +INSERT INTO t1 VALUES (16); +--save_master_pos + +--connection server_2 +SET sql_slave_skip_counter= 3; +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +SELECT IF(LOCATE("2-1-1022", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1022 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + +--source include/stop_slave.inc + +--connection server_1 +SET gtid_domain_id=2; +SET gtid_seq_no=1030; +INSERT INTO t1 VALUES (17); +INSERT INTO t1 VALUES (18); +INSERT INTO t1 VALUES (19); +--save_master_pos + +--connection server_2 +SET sql_slave_skip_counter= 5; +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +SELECT IF(LOCATE("2-1-1032", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1032 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + + +--source include/stop_slave.inc + +--connection server_1 +SET gtid_domain_id=3; +SET gtid_seq_no=100; +CREATE TABLE t2 (a INT PRIMARY KEY); +DROP TABLE t2; +SET gtid_domain_id=2; +SET gtid_seq_no=1040; +INSERT INTO t1 VALUES (20); +--save_master_pos + +--connection server_2 +SET @saved_mode= @@GLOBAL.slave_ddl_exec_mode; +SET GLOBAL slave_ddl_exec_mode=STRICT; +SET sql_slave_skip_counter=1; +START SLAVE UNTIL master_gtid_pos="3-1-100"; +--let $master_pos=3-1-100 +--source include/sync_with_master_gtid.inc +--source include/wait_for_slave_to_stop.inc +--error ER_NO_SUCH_TABLE +SELECT * FROM t2; +SELECT IF(LOCATE("3-1-100", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-100 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + +# Start the slave again, it should fail on the DROP TABLE as the table is not there. +SET sql_log_bin=0; +CALL mtr.add_suppression("Slave: Unknown table 'test\\.t2' Error_code: 1051"); +SET sql_log_bin=1; +START SLAVE; +--let $slave_sql_errno=1051 +--source include/wait_for_slave_sql_error.inc +SELECT IF(LOCATE("3-1-100", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-100 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + +STOP SLAVE IO_THREAD; +SET sql_slave_skip_counter=2; +--source include/start_slave.inc +--sync_with_master + +SELECT * FROM t1 WHERE a >= 20 ORDER BY a; +SELECT IF(LOCATE("3-1-101", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-101 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +SELECT IF(LOCATE("2-1-1040", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1040 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; + +SET GLOBAL slave_ddl_exec_mode= @saved_mode; + + +--echo *** Test GTID-connecting to a master with out-of-order sequence numbers in the binlog. *** + +# Create an out-of-order binlog on server 2. +# Let server 3 replicate to an out-of-order point, stop it, restart it, +# and check that it replicates correctly despite the out-of-order. + +--connection server_1 +SET gtid_domain_id= @@GLOBAL.gtid_domain_id; +INSERT INTO t1 VALUES (31); +--save_master_pos + +--connection server_2 +--sync_with_master +SET gtid_domain_id= @@GLOBAL.gtid_domain_id; +INSERT INTO t1 VALUES (32); + +--connection server_1 +INSERT INTO t1 VALUES (33); +--save_master_pos + +--connection server_2 +--sync_with_master +--save_master_pos + +--connection server_3 +--sync_with_master +--source include/stop_slave.inc + +--connection server_1 +INSERT INTO t1 VALUES (34); +--save_master_pos + +--connection server_2 +--sync_with_master +--save_master_pos + +--connection server_3 +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; +--save_master_pos + +--connection server_4 +--sync_with_master +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; + + +# Clean up. +--connection server_1 +DROP TABLE t1; + + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_incident.inc b/mysql-test/extra/rpl_tests/rpl_incident.inc new file mode 100644 index 0000000000000..189240e3b7556 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_incident.inc @@ -0,0 +1,54 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +--source include/master-slave.inc +--source include/have_debug.inc + +--echo **** On Master **** +CREATE TABLE t1 (a INT); + +INSERT INTO t1 VALUES (1),(2),(3); +SELECT * FROM t1; + +let $debug_save= `SELECT @@GLOBAL.debug`; +SET GLOBAL debug_dbug= '+d,incident_database_resync_on_replace,*'; + +# This will generate an incident log event and store it in the binary +# log before the replace statement. +REPLACE INTO t1 VALUES (4); +--save_master_pos +SELECT * FROM t1; + +--disable_query_log +eval SET GLOBAL debug_dbug= '$debug_save'; +--enable_query_log + +connection slave; +# Wait until SQL thread stops with error LOST_EVENT on master +call mtr.add_suppression("Slave SQL.*The incident LOST_EVENTS occurred on the master.* 1590"); +let $slave_sql_errno= 1590; +let $show_slave_sql_error= 1; +source include/wait_for_slave_sql_error.inc; + +# The 4 should not be inserted into the table, since the incident log +# event should have stop the slave. +--echo **** On Slave **** +SELECT * FROM t1; + +SET GLOBAL SQL_SLAVE_SKIP_COUNTER=1; +START SLAVE; +--sync_with_master + +# Now, we should have inserted the row into the table and the slave +# should be running. We should also have rotated to a new binary log. + +SELECT * FROM t1; +source include/check_slave_is_running.inc; + +connection master; +DROP TABLE t1; +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_init_slave_errors.inc b/mysql-test/extra/rpl_tests/rpl_init_slave_errors.inc new file mode 100644 index 0000000000000..a8ac4e3cd6e7f --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_init_slave_errors.inc @@ -0,0 +1,95 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +###################################################################### +# Some errors that cause the slave SQL thread to stop are not shown in +# the Slave_SQL_Error column of "SHOW SLAVE STATUS". Instead, the error +# is only in the server's error log. +# +# Two failures and their respective reporting are verified: +# +# 1 - Failures during slave thread initialization +# 2 - Failures while processing queries passed through the init_slave +# option. +# +# In order to check the first type of failure, we inject a fault in the +# SQL/IO Threads through SET GLOBAL debug. +# +# To check the second type, we set @@global.init_slave to an invalid +# command thus preventing the initialization of the SQL Thread. +# +# Obs: +# 1 - Note that testing failures while initializing the relay log position +# is hard as the same function is called before the code reaches the point +# that we want to test. +# +# 2 - This test does not target failures that are reported while applying +# events such as duplicate keys, errors while reading the relay-log.bin*, +# etc. Such errors are already checked on other tests. +###################################################################### + +###################################################################### +# Configuring the Environment +###################################################################### +source include/have_debug.inc; +source include/master-slave.inc; +source include/have_log_bin.inc; + +connection slave; + +--disable_warnings +stop slave; +--enable_warnings +reset slave; + +###################################################################### +# Injecting faults in the threads' initialization +###################################################################### +connection slave; + +# Set debug flags on slave to force errors to occur +SET GLOBAL debug_dbug= "d,simulate_io_slave_error_on_init,simulate_sql_slave_error_on_init"; + +start slave; + +# +# slave is going to stop because of emulated failures +# but there won't be any crashes nor asserts hit. +# +# 1593 = ER_SLAVE_FATAL_ERROR +--let $slave_sql_errno= 1593 +--let $show_slave_sql_error= 1 +--source include/wait_for_slave_sql_error.inc + +call mtr.add_suppression("Failed during slave.* thread initialization"); + +SET GLOBAL debug_dbug= ""; + +###################################################################### +# Injecting faults in the init_slave option +###################################################################### +connection slave; + +reset slave; + +SET GLOBAL init_slave= "garbage"; + +start slave; +# 1064 = ER_PARSE_ERROR +--let $slave_sql_errno= 1064 +--let $show_slave_sql_error= 1 +--source include/wait_for_slave_sql_error.inc + +###################################################################### +# Clean up +###################################################################### +SET GLOBAL init_slave= ""; + +# Clean up Last_SQL_Error +--source include/stop_slave_io.inc +RESET SLAVE; +--let $rpl_only_running_threads= 1 +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_loaddata_local.inc b/mysql-test/extra/rpl_tests/rpl_loaddata_local.inc new file mode 100644 index 0000000000000..9604ca83d5e4c --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_loaddata_local.inc @@ -0,0 +1,246 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# See if "LOAD DATA LOCAL INFILE" is well replicated +# (LOAD DATA LOCAL INFILE is not written to the binlog +# the same way as LOAD DATA INFILE : Append_blocks are smaller). +# In MySQL 4.0 <4.0.12 there were 2 bugs with LOAD DATA LOCAL INFILE : +# - the loaded file was not written entirely to the master's binlog, +# only the first 4KB, 8KB or 16KB usually. +# - the loaded file's first line was not written entirely to the +# master's binlog (1st char was absent) +source include/master-slave.inc; + +create table t1(a int); +let $1=10000; +disable_query_log; +set SQL_LOG_BIN=0; +while ($1) +{ + insert into t1 values(1); + dec $1; +} +set SQL_LOG_BIN=1; +enable_query_log; +let $MYSQLD_DATADIR= `select @@datadir`; +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval select * into outfile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; +#This will generate a 20KB file, now test LOAD DATA LOCAL +truncate table t1; +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval load data local infile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; +--remove_file $MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile +sync_slave_with_master; +select a,count(*) from t1 group by a; +connection master; +drop table t1; +sync_slave_with_master; + +# End of 4.1 tests + +# +# Now let us test how well we replicate LOAD DATA LOCAL in situation when +# we met duplicates in tables to which we are adding rows. +# (It supposed that LOAD DATA LOCAL ignores such errors) +# +connection master; +create table t1(a int); +insert into t1 values (1), (2), (2), (3); +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval select * into outfile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; +drop table t1; +create table t1(a int primary key); +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval load data local infile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; +--remove_file $MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile +SELECT * FROM t1 ORDER BY a; +save_master_pos; +connection slave; +sync_with_master; +SELECT * FROM t1 ORDER BY a; +connection master; +drop table t1; +save_master_pos; +connection slave; +sync_with_master; + + +# +# Bug22504 load data infile sql statement in replication architecture get error +# +--echo ==== Bug22504 Initialize ==== + +--echo [on master] +--connection master + +SET sql_mode='ignore_space'; +CREATE TABLE t1(a int); +insert into t1 values (1), (2), (3), (4); +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval select * into outfile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; +truncate table t1; +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval load data local infile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; +--remove_file $MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile +SELECT * FROM t1 ORDER BY a; + +--echo [on slave] +sync_slave_with_master; +SELECT * FROM t1 ORDER BY a; + +--echo ==== Clean up ==== + +--echo [on master] +connection master; +DROP TABLE t1; + +--echo [on slave] +sync_slave_with_master; + +--echo +--echo Bug #43746: +--echo "return wrong query string when parse 'load data infile' sql statement" +--echo + +--echo [master] +connection master; +let $MYSQLD_DATADIR= `select @@datadir`; +SELECT @@SESSION.sql_mode INTO @old_mode; + +SET sql_mode='ignore_space'; + +CREATE TABLE t1(a int); +INSERT INTO t1 VALUES (1), (2), (3), (4); + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval SELECT * INTO OUTFILE '$MYSQLD_DATADIR/bug43746.sql' FROM t1; +TRUNCATE TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD/* look mum, with comments in weird places! */DATA/* oh hai */LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql'/* we are */INTO/* from the internets */TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' /*!10000 INTO */ TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' /*!10000 INTO TABLE */ t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA /*!10000 LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE */ t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql'/*!10000 INTO*/TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql'/* empty */INTO TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO/* empty */TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD/*!999999 special comments that do not expand */DATA/*!999999 code from the future */LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql'/*!999999 have flux capacitor */INTO/*!999999 will travel */TABLE t1; + +SET sql_mode='PIPES_AS_CONCAT,ANSI_QUOTES,NO_KEY_OPTIONS,NO_TABLE_OPTIONS,NO_FIELD_OPTIONS,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER'; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; + +--echo [slave] +sync_slave_with_master; + +--echo +--echo Bug #59267: +--echo "LOAD DATA LOCAL INFILE not executed on slave with SBR" +--echo + +--echo [master] +connection master; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval SELECT * INTO OUTFILE '$MYSQLD_DATADIR/bug59267.sql' FROM t1; +TRUNCATE TABLE t1; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug59267.sql' INTO TABLE t1; + +SELECT 'Master', COUNT(*) FROM t1; + +--echo [slave] +--sync_slave_with_master +SELECT 'Slave', COUNT(*) FROM t1; + +# cleanup +--echo [master] +connection master; + +--remove_file $MYSQLD_DATADIR/bug43746.sql +--remove_file $MYSQLD_DATADIR/bug59267.sql + +DROP TABLE t1; +SET SESSION sql_mode=@old_mode; + +--echo [slave] +sync_slave_with_master; + +connection master; + +--echo +--echo Bug #60580/#11902767: +--echo "statement improperly replicated crashes slave sql thread" +--echo + +--echo [master] +connection master; +let $MYSQLD_DATADIR= `select @@datadir`; + +CREATE TABLE t1(f1 INT, f2 INT); +CREATE TABLE t2(f1 INT, f2 TIMESTAMP); + +INSERT INTO t2 VALUES(1, '2011-03-22 21:01:28'); +INSERT INTO t2 VALUES(2, '2011-03-21 21:01:28'); +INSERT INTO t2 VALUES(3, '2011-03-20 21:01:28'); + +CREATE TABLE t3 AS SELECT * FROM t2; + +CREATE VIEW v1 AS SELECT * FROM t2 + WHERE f1 IN (SELECT f1 FROM t3 WHERE (t3.f2 IS NULL)); + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval SELECT 1 INTO OUTFILE '$MYSQLD_DATADIR/bug60580.csv' FROM DUAL; + +--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR +eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug60580.csv' INTO TABLE t1 (@f1) SET f2 = (SELECT f1 FROM v1 WHERE f1=@f1); + +SELECT * FROM t1; + +sleep 1; + +--echo [slave] +sync_slave_with_master; + +SELECT * FROM t1; + +--remove_file $MYSQLD_DATADIR/bug60580.csv + +--echo [master] +connection master; + +DROP VIEW v1; +DROP TABLE t1, t2, t3; + +--echo [slave] +sync_slave_with_master; + +connection master; +--source include/rpl_end.inc + +--echo # End of 5.1 tests diff --git a/mysql-test/extra/rpl_tests/rpl_loadfile.inc b/mysql-test/extra/rpl_tests/rpl_loadfile.inc new file mode 100644 index 0000000000000..e43c003b29cb6 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_loadfile.inc @@ -0,0 +1,120 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +############################################################################# +# Original Author: JBM # +# Original Date: Aug/18/2005 # +############################################################################# +# TEST: To test the LOAD_FILE() in rbr # +############################################################################# +# Change Author: JBM +# Change Date: 2006-01-16 +########## + +# Includes +-- source include/have_binlog_format_mixed_or_row.inc +-- source include/master-slave.inc + +-- source extra/rpl_tests/rpl_loadfile.test + +# BUG#39701: Mixed binlog format does not switch to row mode on LOAD_FILE +# +# DESCRIPTION +# +# Problem: when using load_file string function and mixed binlogging format +# there was no switch to row based binlogging format. This leads +# to scenarios on which the slave replicates the statement and it +# will try to load the file from local file system, which in most +# likely it will not exist. +# +# Solution: +# Marking this function as unsafe for statement format, makes the +# statement using it to be logged in row based format. As such, data +# replicated from the master, becomes the content of the loaded file. +# Consequently, the slave receives the necessary data to complete +# the load_file instruction correctly. +# +# IMPLEMENTATION +# +# The test is implemented as follows: +# +# On Master, +# i) write to file the desired content. +# ii) create table and stored procedure with load_file +# iii) stop slave +# iii) execute load_file +# iv) remove file +# +# On Slave, +# v) start slave +# vi) sync it with master so that it gets the updates from binlog (which +# should have bin logged in row format). +# +# If the the binlog format does not change to row, then the assertion +# done in the following step fails. This happens because tables differ +# since the file does not exist anymore, meaning that when slave +# attempts to execute LOAD_FILE statement it inserts NULL on table +# instead of the same contents that the master loaded when it executed +# the procedure (which was executed when file existed). +# +# vii) assert that the contents of master and slave +# table are the same + +--source include/rpl_reset.inc + +connection master; +let $file= $MYSQLTEST_VARDIR/tmp/bug_39701.data; + +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval SELECT repeat('x',20) INTO OUTFILE '$file' + +disable_warnings; +DROP TABLE IF EXISTS t1; +enable_warnings; + +CREATE TABLE t1 (t text); +DELIMITER |; +CREATE PROCEDURE p(file varchar(4096)) + BEGIN + INSERT INTO t1 VALUES (LOAD_FILE(file)); + END| +DELIMITER ;| + +# stop slave before issuing the load_file on master +connection slave; +source include/stop_slave.inc; + +connection master; + +# test: check that logging falls back to rbr. +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval CALL p('$file') + +# test: remove the file from the filesystem and assert that slave still +# gets the loaded file +remove_file $file; + +# now that the file is removed it is safe (regarding what we want to test) +# to start slave +connection slave; +source include/start_slave.inc; + +connection master; +sync_slave_with_master; + +# assertion: assert that the slave got the updates even +# if the file was removed before the slave started, +# meaning that contents were indeed transfered +# through binlog (in row format) +let $diff_tables= master:t1, slave:t1; +source include/diff_tables.inc; + +# CLEAN UP +--connection master +DROP TABLE t1; +DROP PROCEDURE p; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_packet.inc b/mysql-test/extra/rpl_tests/rpl_packet.inc new file mode 100644 index 0000000000000..41bb374b80252 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_packet.inc @@ -0,0 +1,183 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# ==== Purpose ==== +# +# Check replication protocol packet size handling +# +# ==== Related bugs ==== +# Bug#19402 SQL close to the size of the max_allowed_packet fails on slave +# BUG#23755: Replicated event larger that max_allowed_packet infinitely re-transmits +# BUG#42914: No LAST_IO_ERROR for max_allowed_packet errors +# BUG#55322: SHOW BINLOG EVENTS increases @@SESSION.MAX_ALLOWED_PACKET + +# max-out size db name +source include/master-slave.inc; +source include/have_binlog_format_row.inc; +call mtr.add_suppression("Slave I/O: Got a packet bigger than 'slave_max_allowed_packet' bytes, .*error.* 1153"); +call mtr.add_suppression("Log entry on master is longer than slave_max_allowed_packet"); +let $db= DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________; +disable_warnings; +eval drop database if exists $db; +enable_warnings; +eval create database $db; + +connection master; +let $old_max_allowed_packet= `SELECT @@global.max_allowed_packet`; +let $old_net_buffer_length= `SELECT @@global.net_buffer_length`; +let $old_slave_max_allowed_packet= `SELECT @@global.slave_max_allowed_packet`; +SET @@global.max_allowed_packet=1024; +SET @@global.net_buffer_length=1024; + +sync_slave_with_master; +# Restart slave for setting to take effect +source include/stop_slave.inc; +source include/start_slave.inc; + +# Reconnect to master for new setting to take effect +disconnect master; + +# alas, can't use eval here; if db name changed apply the change here +connect (master,localhost,root,,DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________); + +connection master; +select @@net_buffer_length, @@max_allowed_packet; + +create table `t1` (`f1` LONGTEXT) ENGINE=MyISAM; + +INSERT INTO `t1`(`f1`) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1023'); +sync_slave_with_master; + +eval select count(*) from `$db`.`t1` /* must be 1 */; + +SHOW STATUS LIKE 'Slave_running'; +select * from information_schema.session_status where variable_name= 'SLAVE_RUNNING'; +connection master; +eval drop database $db; +sync_slave_with_master; + +# +# Bug #23755: Replicated event larger that max_allowed_packet infinitely re-transmits +# +# Check that a situation when the size of event on the master is greater than +# max_allowed_packet on the slave does not lead to infinite re-transmits. + +connection master; + +# Change the max packet size on master + +SET @@global.max_allowed_packet=4096; +SET @@global.net_buffer_length=4096; + +# Restart slave for new setting to take effect +connection slave; +source include/stop_slave.inc; +source include/start_slave.inc; + +# Reconnect to master for new setting to take effect +disconnect master; +connect (master, localhost, root); +connection master; + +CREATE TABLE `t1` (`f1` LONGTEXT) ENGINE=MyISAM; + +sync_slave_with_master; + +connection master; + +INSERT INTO `t1`(`f1`) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2048'); + + +# +# Bug#42914: The slave I/O thread must stop after trying to read the above +# event, However there is no Last_IO_Error report. +# + +# The slave I/O thread must stop after trying to read the above event +connection slave; +# 1153 = ER_NET_PACKET_TOO_LARGE +--let $slave_io_errno= 1153 +--let $show_slave_io_error= 1 +--source include/wait_for_slave_io_error.inc + +# TODO: this is needed because of BUG#55790. Remove once that is fixed. +--source include/stop_slave_sql.inc + +# +# Bug#42914: On the master, if a binary log event is larger than +# max_allowed_packet, the error message ER_MASTER_FATAL_ERROR_READING_BINLOG +# is sent to a slave when it requests a dump from the master, thus leading the +# I/O thread to stop. However, there is no Last_IO_Error reported. +# + +--let $rpl_only_running_threads= 1 +--source include/rpl_reset.inc +--connection master +DROP TABLE t1; +--sync_slave_with_master + + +connection master; +CREATE TABLE t1 (f1 int PRIMARY KEY, f2 LONGTEXT, f3 LONGTEXT) ENGINE=MyISAM; +sync_slave_with_master; + +connection master; +INSERT INTO t1(f1, f2, f3) VALUES(1, REPEAT('a', @@global.max_allowed_packet), REPEAT('b', @@global.max_allowed_packet)); + +connection slave; +# The slave I/O thread must stop after receiving +# 1153 = ER_NET_PACKET_TOO_LARGE +--let $slave_io_errno= 1153 +--let $show_slave_io_error= 1 +--source include/wait_for_slave_io_error.inc + +# Remove the bad binlog and clear error status on slave. +STOP SLAVE; +RESET SLAVE; +--connection master +RESET MASTER; + + +# +# BUG#55322: SHOW BINLOG EVENTS increases @@SESSION.MAX_ALLOWED_PACKET +# +# In BUG#55322, @@session.max_allowed_packet increased each time SHOW +# BINLOG EVENTS was issued. To verify that this bug is fixed, we +# execute SHOW BINLOG EVENTS twice and check that max_allowed_packet +# never changes. We turn off the result log because we don't care +# about the contents of the binlog. + +--disable_result_log +SET @max_allowed_packet_0= @@session.max_allowed_packet; +SHOW BINLOG EVENTS; +SET @max_allowed_packet_1= @@session.max_allowed_packet; +SHOW BINLOG EVENTS; +SET @max_allowed_packet_2= @@session.max_allowed_packet; +--enable_result_log +if (`SELECT NOT(@max_allowed_packet_0 = @max_allowed_packet_1 AND @max_allowed_packet_1 = @max_allowed_packet_2)`) +{ + --echo ERROR: max_allowed_packet changed after executing SHOW BINLOG EVENTS + --source include/show_rpl_debug_info.inc + SELECT @max_allowed_packet_0, @max_allowed_packet_1, @max_allowed_packet_2; + --die @max_allowed_packet changed after executing SHOW BINLOG EVENTS +} + + +--echo ==== clean up ==== +connection master; +DROP TABLE t1; +eval SET @@global.max_allowed_packet= $old_max_allowed_packet; +eval SET @@global.net_buffer_length= $old_net_buffer_length; +eval SET @@global.slave_max_allowed_packet= $old_slave_max_allowed_packet; +# slave is stopped +connection slave; +DROP TABLE t1; + +# Clear Last_IO_Error +RESET SLAVE; + +--source include/rpl_end.inc +# End of tests diff --git a/mysql-test/extra/rpl_tests/rpl_parallel.inc b/mysql-test/extra/rpl_tests/rpl_parallel.inc new file mode 100644 index 0000000000000..8248f03e1850e --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_parallel.inc @@ -0,0 +1,2479 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc + +# Test various aspects of parallel replication. + +--connection server_2 +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +--error ER_SLAVE_MUST_STOP +SET GLOBAL slave_parallel_threads=10; +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; + +# Check that we do not spawn any worker threads when no slave is running. +SELECT IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; + +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +# Check that worker threads get spawned when slave starts. +SELECT IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; +# ... and that worker threads get removed when slave stops. +--source include/stop_slave.inc +SELECT IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; +--source include/start_slave.inc +SELECT IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; + +--echo *** Test long-running query in domain 1 can run in parallel with short queries in domain 0 *** + +--connection server_1 +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=MyISAM; +CREATE TABLE t2 (a int PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1); +--save_master_pos + +--connection server_2 +--sync_with_master + +# Block the table t1 to simulate a replicated query taking a long time. +--connect (con_temp1,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +LOCK TABLE t1 WRITE; + +--connection server_1 +SET gtid_domain_id=1; +# This query will be blocked on the slave until UNLOCK TABLES. +INSERT INTO t1 VALUES (2); +SET gtid_domain_id=0; +# These t2 queries can be replicated in parallel with the prior t1 query, as +# they are in a separate replication domain. +INSERT INTO t2 VALUES (2); +INSERT INTO t2 VALUES (3); +BEGIN; +INSERT INTO t2 VALUES (4); +INSERT INTO t2 VALUES (5); +COMMIT; +INSERT INTO t2 VALUES (6); + +--connection server_2 +--let $wait_condition= SELECT COUNT(*) = 6 FROM t2 +--source include/wait_condition.inc + +SELECT * FROM t2 ORDER by a; + +--connection con_temp1 +SELECT * FROM t1; +UNLOCK TABLES; + +--connection server_2 +--let $wait_condition= SELECT COUNT(*) = 2 FROM t1 +--source include/wait_condition.inc + +SELECT * FROM t1 ORDER BY a; + + +--echo *** Test two transactions in different domains committed in opposite order on slave but in a single group commit. *** +--connection server_2 +--source include/stop_slave.inc + +--connection server_1 +# Use a stored function to inject a debug_sync into the appropriate THD. +# The function does nothing on the master, and on the slave it injects the +# desired debug_sync action(s). +SET sql_log_bin=0; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +SET @old_format= @@SESSION.binlog_format; +SET binlog_format='statement'; +SET gtid_domain_id=1; +INSERT INTO t2 VALUES (foo(10, + 'commit_before_enqueue SIGNAL ready1 WAIT_FOR cont1', + 'commit_after_release_LOCK_prepare_ordered SIGNAL ready2')); + +--connection server_2 +FLUSH LOGS; +--source include/wait_for_binlog_checkpoint.inc +SET sql_log_bin=0; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + IF d1 != '' THEN + SET debug_sync = d1; + END IF; + IF d2 != '' THEN + SET debug_sync = d2; + END IF; + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; +SET @old_format=@@GLOBAL.binlog_format; +SET GLOBAL binlog_format=statement; +# We need to restart all parallel threads for the new global setting to +# be copied to the session-level values. +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + +# First make sure the first insert is ready to commit, but not queued yet. +SET debug_sync='now WAIT_FOR ready1'; + +--connection server_1 +SET gtid_domain_id=2; +INSERT INTO t2 VALUES (foo(11, + 'commit_before_enqueue SIGNAL ready3 WAIT_FOR cont3', + 'commit_after_release_LOCK_prepare_ordered SIGNAL ready4 WAIT_FOR cont4')); +SET gtid_domain_id=0; +SELECT * FROM t2 WHERE a >= 10 ORDER BY a; + +--connection server_2 +# Now wait for the second insert to queue itself as the leader, and then +# wait for more commits to queue up. +SET debug_sync='now WAIT_FOR ready3'; +SET debug_sync='now SIGNAL cont3'; +SET debug_sync='now WAIT_FOR ready4'; +# Now allow the first insert to queue up to participate in group commit. +SET debug_sync='now SIGNAL cont1'; +SET debug_sync='now WAIT_FOR ready2'; +# Finally allow the second insert to proceed and do the group commit. +SET debug_sync='now SIGNAL cont4'; + +--let $wait_condition= SELECT COUNT(*) = 2 FROM t2 WHERE a >= 10 +--source include/wait_condition.inc +SELECT * FROM t2 WHERE a >= 10 ORDER BY a; +# The two INSERT transactions should have been committed in opposite order, +# but in the same group commit (seen by precense of cid=# in the SHOW +# BINLOG output). +--let $binlog_file= slave-bin.000002 +--source include/show_binlog_events.inc +FLUSH LOGS; +--source include/wait_for_binlog_checkpoint.inc + +# Restart all the slave parallel worker threads, to clear all debug_sync actions. +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET debug_sync='RESET'; +--source include/start_slave.inc + + +--echo *** Test that group-committed transactions on the master can replicate in parallel on the slave. *** +--connection server_1 +SET debug_sync='RESET'; +FLUSH LOGS; +--source include/wait_for_binlog_checkpoint.inc +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +# Create some sentinel rows so that the rows inserted in parallel fall into +# separate gaps and do not cause gap lock conflicts. +INSERT INTO t3 VALUES (1,1), (3,3), (5,5), (7,7); +--save_master_pos +--connection server_2 +--sync_with_master + +# We want to test that the transactions can execute out-of-order on +# the slave, but still end up committing in-order, and in a single +# group commit. +# +# The idea is to group-commit three transactions together on the master: +# A, B, and C. On the slave, C will execute the insert first, then A, +# and then B. But B manages to complete before A has time to commit, so +# all three end up committing together. +# +# So we start by setting up some row locks that will block transactions +# A and B from executing, allowing C to run first. + +--connection con_temp1 +BEGIN; +INSERT INTO t3 VALUES (2,102); +--connect (con_temp2,127.0.0.1,root,,test,$SERVER_MYPORT_2,) +BEGIN; +INSERT INTO t3 VALUES (4,104); + +# On the master, queue three INSERT transactions as a single group commit. +--connect (con_temp3,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (2, foo(12, + 'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued1 WAIT_FOR slave_cont1', + '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connect (con_temp4,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (4, foo(14, + 'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued2', + '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; + +--connect (con_temp5,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (6, foo(16, + 'group_commit_waiting_for_prior SIGNAL slave_queued3', + '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con_temp3 +REAP; +--connection con_temp4 +REAP; +--connection con_temp5 +REAP; +SET debug_sync='RESET'; + +--connection server_1 +SELECT * FROM t3 ORDER BY a; +--let $binlog_file= master-bin.000002 +--source include/show_binlog_events.inc + +# First, wait until insert 3 is ready to queue up for group commit, but is +# waiting for insert 2 to commit before it can do so itself. +--connection server_2 +SET debug_sync='now WAIT_FOR slave_queued3'; + +# Next, let insert 1 proceed, and allow it to queue up as the group commit +# leader, but let it wait for insert 2 to also queue up before proceeding. +--connection con_temp1 +ROLLBACK; +--connection server_2 +SET debug_sync='now WAIT_FOR slave_queued1'; + +# Now let insert 2 proceed and queue up. +--connection con_temp2 +ROLLBACK; +--connection server_2 +SET debug_sync='now WAIT_FOR slave_queued2'; +# And finally, we can let insert 1 proceed and do the group commit with all +# three insert transactions together. +SET debug_sync='now SIGNAL slave_cont1'; + +# Wait for the commit to complete and check that all three transactions +# group-committed together (will be seen in the binlog as all three having +# cid=# on their GTID event). +--let $wait_condition= SELECT COUNT(*) = 3 FROM t3 WHERE a IN (2,4,6) +--source include/wait_condition.inc +SELECT * FROM t3 ORDER BY a; +--let $binlog_file= slave-bin.000003 +--source include/show_binlog_events.inc + + +--echo *** Test STOP SLAVE in parallel mode *** +--connection server_2 +--source include/stop_slave.inc +# Respawn all worker threads to clear any left-over debug_sync or other stuff. +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; + +--connection server_1 +# Set up a couple of transactions. The first will be blocked halfway +# through on a lock, and while it is blocked we initiate STOP SLAVE. +# We then test that the halfway-initiated transaction is allowed to +# complete, but no subsequent ones. +# We have to use statement-based mode and set +# binlog_direct_non_transactional_updates=0; otherwise the binlog will +# be split into two event groups, one for the MyISAM part and one for the +# InnoDB part. +SET binlog_direct_non_transactional_updates=0; +SET sql_log_bin=0; +CALL mtr.add_suppression("Statement is unsafe because it accesses a non-transactional table after accessing a transactional table within the same transaction"); +SET sql_log_bin=1; +BEGIN; +INSERT INTO t2 VALUES (20); +--disable_warnings +INSERT INTO t1 VALUES (20); +--enable_warnings +INSERT INTO t2 VALUES (21); +INSERT INTO t3 VALUES (20, 20); +COMMIT; +INSERT INTO t3 VALUES(21, 21); +INSERT INTO t3 VALUES(22, 22); +SET binlog_format=@old_format; +--save_master_pos + +# Start a connection that will block the replicated transaction halfway. +--connection con_temp1 +BEGIN; +INSERT INTO t2 VALUES (21); + +--connection server_2 +START SLAVE; +# Wait for the MyISAM change to be visible, after which replication will wait +# for con_temp1 to roll back. +--let $wait_condition= SELECT COUNT(*) = 1 FROM t1 WHERE a=20 +--source include/wait_condition.inc + +--connection con_temp2 +# Initiate slave stop. It will have to wait for the current event group +# to complete. +# The dbug injection causes debug_sync to signal 'wait_for_done_waiting' +# when the SQL driver thread is ready. +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; +send STOP SLAVE; + +--connection con_temp1 +SET debug_sync='now WAIT_FOR wait_for_done_waiting'; +ROLLBACK; + +--connection con_temp2 +reap; +SET GLOBAL debug_dbug=@old_dbug; +SET debug_sync='RESET'; + +--connection server_2 +--source include/wait_for_slave_to_stop.inc +# We should see the first transaction applied, but not the two others. +SELECT * FROM t1 WHERE a >= 20 ORDER BY a; +SELECT * FROM t2 WHERE a >= 20 ORDER BY a; +SELECT * FROM t3 WHERE a >= 20 ORDER BY a; + +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t1 WHERE a >= 20 ORDER BY a; +SELECT * FROM t2 WHERE a >= 20 ORDER BY a; +SELECT * FROM t3 WHERE a >= 20 ORDER BY a; + + +--connection server_2 +# Respawn all worker threads to clear any left-over debug_sync or other stuff. +--source include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** Test killing slave threads at various wait points *** +--echo *** 1. Test killing transaction waiting in commit for previous transaction to commit *** + +# Set up three transactions on the master that will be group-committed +# together so they can be replicated in parallel on the slave. +--connection con_temp3 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (31, foo(31, + 'commit_before_prepare_ordered WAIT_FOR t2_waiting', + 'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con_temp4 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +BEGIN; +# This insert is just so we can get T2 to wait while a query is running that we +# can see in SHOW PROCESSLIST so we can get its thread_id to kill later. +INSERT INTO t3 VALUES (32, foo(32, + 'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', + '')); +# This insert sets up debug_sync points so that T2 will tell when it is at its +# wait point where we want to kill it - and when it has been killed. +INSERT INTO t3 VALUES (33, foo(33, + 'group_commit_waiting_for_prior SIGNAL t2_waiting', + 'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); +send COMMIT; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; + +--connection con_temp5 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (34, foo(34, + '', + '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con_temp3 +REAP; +--connection con_temp4 +REAP; +--connection con_temp5 +REAP; + +--connection server_1 +SELECT * FROM t3 WHERE a >= 30 ORDER BY a; +SET debug_sync='RESET'; + +--connection server_2 +SET sql_log_bin=0; +CALL mtr.add_suppression("Query execution was interrupted"); +CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); +CALL mtr.add_suppression("Slave: Connection was killed"); +SET sql_log_bin=1; +# Wait until T2 is inside executing its insert of 32, then find it in SHOW +# PROCESSLIST to know its thread id for KILL later. +SET debug_sync='now WAIT_FOR t2_query'; +--let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(32%' AND INFO NOT LIKE '%LIKE%'` +SET debug_sync='now SIGNAL t2_cont'; + +# Wait until T2 has entered its wait for T1 to commit, and T1 has +# progressed into its commit phase. +SET debug_sync='now WAIT_FOR t1_ready'; + +# Now kill the transaction T2. +--replace_result $thd_id THD_ID +eval KILL $thd_id; + +# Wait until T2 has reacted on the kill. +SET debug_sync='now WAIT_FOR t2_killed'; + +# Now we can allow T1 to proceed. +SET debug_sync='now SIGNAL t1_cont'; + +--let $slave_sql_errno= 1317,1927,1964 +--source include/wait_for_slave_sql_error.inc +STOP SLAVE IO_THREAD; +SELECT * FROM t3 WHERE a >= 30 ORDER BY a; + +# Now we have to disable the debug_sync statements, so they do not trigger +# when the events are retried. +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +--connection server_1 +INSERT INTO t3 VALUES (39,0); +--save_master_pos + +--connection server_2 +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t3 WHERE a >= 30 ORDER BY a; +# Restore the foo() function. +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + IF d1 != '' THEN + SET debug_sync = d1; + END IF; + IF d2 != '' THEN + SET debug_sync = d2; + END IF; + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + + +--connection server_2 +# Respawn all worker threads to clear any left-over debug_sync or other stuff. +--source include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** 2. Same as (1), but without restarting IO thread after kill of SQL threads *** + +# Set up three transactions on the master that will be group-committed +# together so they can be replicated in parallel on the slave. +--connection con_temp3 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (41, foo(41, + 'commit_before_prepare_ordered WAIT_FOR t2_waiting', + 'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con_temp4 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +BEGIN; +# This insert is just so we can get T2 to wait while a query is running that we +# can see in SHOW PROCESSLIST so we can get its thread_id to kill later. +INSERT INTO t3 VALUES (42, foo(42, + 'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', + '')); +# This insert sets up debug_sync points so that T2 will tell when it is at its +# wait point where we want to kill it - and when it has been killed. +INSERT INTO t3 VALUES (43, foo(43, + 'group_commit_waiting_for_prior SIGNAL t2_waiting', + 'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); +send COMMIT; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; + +--connection con_temp5 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (44, foo(44, + '', + '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con_temp3 +REAP; +--connection con_temp4 +REAP; +--connection con_temp5 +REAP; + +--connection server_1 +SELECT * FROM t3 WHERE a >= 40 ORDER BY a; +SET debug_sync='RESET'; + +--connection server_2 +# Wait until T2 is inside executing its insert of 42, then find it in SHOW +# PROCESSLIST to know its thread id for KILL later. +SET debug_sync='now WAIT_FOR t2_query'; +--let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(42%' AND INFO NOT LIKE '%LIKE%'` +SET debug_sync='now SIGNAL t2_cont'; + +# Wait until T2 has entered its wait for T1 to commit, and T1 has +# progressed into its commit phase. +SET debug_sync='now WAIT_FOR t1_ready'; + +# Now kill the transaction T2. +--replace_result $thd_id THD_ID +eval KILL $thd_id; + +# Wait until T2 has reacted on the kill. +SET debug_sync='now WAIT_FOR t2_killed'; + +# Now we can allow T1 to proceed. +SET debug_sync='now SIGNAL t1_cont'; + +--let $slave_sql_errno= 1317,1927,1964 +--source include/wait_for_slave_sql_error.inc + +# Now we have to disable the debug_sync statements, so they do not trigger +# when the events are retried. +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +--connection server_1 +INSERT INTO t3 VALUES (49,0); +--save_master_pos + +--connection server_2 +START SLAVE SQL_THREAD; +--sync_with_master +SELECT * FROM t3 WHERE a >= 40 ORDER BY a; +# Restore the foo() function. +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + IF d1 != '' THEN + SET debug_sync = d1; + END IF; + IF d2 != '' THEN + SET debug_sync = d2; + END IF; + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + + +--connection server_2 +# Respawn all worker threads to clear any left-over debug_sync or other stuff. +--source include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** 3. Same as (2), but not using gtid mode *** + +--connection server_2 +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=no; +--source include/start_slave.inc + +--connection server_1 +# Set up three transactions on the master that will be group-committed +# together so they can be replicated in parallel on the slave. +--connection con_temp3 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (51, foo(51, + 'commit_before_prepare_ordered WAIT_FOR t2_waiting', + 'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con_temp4 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +BEGIN; +# This insert is just so we can get T2 to wait while a query is running that we +# can see in SHOW PROCESSLIST so we can get its thread_id to kill later. +INSERT INTO t3 VALUES (52, foo(52, + 'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', + '')); +# This insert sets up debug_sync points so that T2 will tell when it is at its +# wait point where we want to kill it - and when it has been killed. +INSERT INTO t3 VALUES (53, foo(53, + 'group_commit_waiting_for_prior SIGNAL t2_waiting', + 'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); +send COMMIT; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; + +--connection con_temp5 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +send INSERT INTO t3 VALUES (54, foo(54, + '', + '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con_temp3 +REAP; +--connection con_temp4 +REAP; +--connection con_temp5 +REAP; + +--connection server_1 +SELECT * FROM t3 WHERE a >= 50 ORDER BY a; +SET debug_sync='RESET'; + +--connection server_2 +# Wait until T2 is inside executing its insert of 52, then find it in SHOW +# PROCESSLIST to know its thread id for KILL later. +SET debug_sync='now WAIT_FOR t2_query'; +--let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(52%' AND INFO NOT LIKE '%LIKE%'` +SET debug_sync='now SIGNAL t2_cont'; + +# Wait until T2 has entered its wait for T1 to commit, and T1 has +# progressed into its commit phase. +SET debug_sync='now WAIT_FOR t1_ready'; + +# Now kill the transaction T2. +--replace_result $thd_id THD_ID +eval KILL $thd_id; + +# Wait until T2 has reacted on the kill. +SET debug_sync='now WAIT_FOR t2_killed'; + +# Now we can allow T1 to proceed. +SET debug_sync='now SIGNAL t1_cont'; + +--let $slave_sql_errno= 1317,1927,1964 +--source include/wait_for_slave_sql_error.inc +SELECT * FROM t3 WHERE a >= 50 ORDER BY a; + +# Now we have to disable the debug_sync statements, so they do not trigger +# when the events are retried. +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +--connection server_1 +INSERT INTO t3 VALUES (59,0); +--save_master_pos + +--connection server_2 +START SLAVE SQL_THREAD; +--sync_with_master +SELECT * FROM t3 WHERE a >= 50 ORDER BY a; +# Restore the foo() function. +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + IF d1 != '' THEN + SET debug_sync = d1; + END IF; + IF d2 != '' THEN + SET debug_sync = d2; + END IF; + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + + +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +--connection server_2 +# Respawn all worker threads to clear any left-over debug_sync or other stuff. +--source include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=4; +--source include/start_slave.inc + + +--echo *** 4. Test killing thread that is waiting to start transaction until previous transaction commits *** + +# We set up four transactions T1, T2, T3, and T4 on the master. T2, T3, and T4 +# can run in parallel with each other (same group commit and commit id), +# but not in parallel with T1. +# +# We use four worker threads, each Ti will be queued on each their own +# worker thread. We will delay T1 commit, T3 will wait for T1 to begin +# commit before it can start. We will kill T3 during this wait, and +# check that everything works correctly. +# +# It is rather tricky to get the correct thread id of the worker to kill. +# We start by injecting four dummy transactions in a debug_sync-controlled +# manner to be able to get known thread ids for the workers in a pool with +# just 4 worker threads. Then we let in each of the real test transactions +# T1-T4 one at a time in a way which allows us to know which transaction +# ends up with which thread id. + +--connection server_1 +SET binlog_format=statement; +SET gtid_domain_id=2; +BEGIN; +# This debug_sync will linger on and be used to control T4 later. +INSERT INTO t3 VALUES (70, foo(70, + 'rpl_parallel_start_waiting_for_prior SIGNAL t4_waiting', '')); +INSERT INTO t3 VALUES (60, foo(60, + 'ha_write_row_end SIGNAL d2_query WAIT_FOR d2_cont2', + 'rpl_parallel_end_of_group SIGNAL d2_done WAIT_FOR d2_cont')); +COMMIT; +SET gtid_domain_id=0; + +--connection server_2 +SET debug_sync='now WAIT_FOR d2_query'; +--let $d2_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(60%' AND INFO NOT LIKE '%LIKE%'` + +--connection server_1 +SET gtid_domain_id=1; +BEGIN; +# These debug_sync's will linger on and be used to control T3 later. +INSERT INTO t3 VALUES (61, foo(61, + 'rpl_parallel_start_waiting_for_prior SIGNAL t3_waiting', + 'rpl_parallel_start_waiting_for_prior_killed SIGNAL t3_killed')); +INSERT INTO t3 VALUES (62, foo(62, + 'ha_write_row_end SIGNAL d1_query WAIT_FOR d1_cont2', + 'rpl_parallel_end_of_group SIGNAL d1_done WAIT_FOR d1_cont')); +COMMIT; +SET gtid_domain_id=0; + +--connection server_2 +SET debug_sync='now WAIT_FOR d1_query'; +--let $d1_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(62%' AND INFO NOT LIKE '%LIKE%'` + +--connection server_1 +SET gtid_domain_id=0; +INSERT INTO t3 VALUES (63, foo(63, + 'ha_write_row_end SIGNAL d0_query WAIT_FOR d0_cont2', + 'rpl_parallel_end_of_group SIGNAL d0_done WAIT_FOR d0_cont')); + +--connection server_2 +SET debug_sync='now WAIT_FOR d0_query'; +--let $d0_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(63%' AND INFO NOT LIKE '%LIKE%'` + +--connection server_1 +SET gtid_domain_id=3; +BEGIN; +# These debug_sync's will linger on and be used to control T2 later. +INSERT INTO t3 VALUES (68, foo(68, + 'rpl_parallel_start_waiting_for_prior SIGNAL t2_waiting', '')); +INSERT INTO t3 VALUES (69, foo(69, + 'ha_write_row_end SIGNAL d3_query WAIT_FOR d3_cont2', + 'rpl_parallel_end_of_group SIGNAL d3_done WAIT_FOR d3_cont')); +COMMIT; +SET gtid_domain_id=0; + +--connection server_2 +SET debug_sync='now WAIT_FOR d3_query'; +--let $d3_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(69%' AND INFO NOT LIKE '%LIKE%'` + +SET debug_sync='now SIGNAL d2_cont2'; +SET debug_sync='now WAIT_FOR d2_done'; +SET debug_sync='now SIGNAL d1_cont2'; +SET debug_sync='now WAIT_FOR d1_done'; +SET debug_sync='now SIGNAL d0_cont2'; +SET debug_sync='now WAIT_FOR d0_done'; +SET debug_sync='now SIGNAL d3_cont2'; +SET debug_sync='now WAIT_FOR d3_done'; + +# Now prepare the real transactions T1, T2, T3, T4 on the master. + +--connection con_temp3 +# Create transaction T1. +SET binlog_format=statement; +INSERT INTO t3 VALUES (64, foo(64, + 'rpl_parallel_before_mark_start_commit SIGNAL t1_waiting WAIT_FOR t1_cont', '')); + +# Create transaction T2, as a group commit leader on the master. +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2 WAIT_FOR master_cont2'; +send INSERT INTO t3 VALUES (65, foo(65, '', '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; + +--connection con_temp4 +# Create transaction T3, participating in T2's group commit. +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +send INSERT INTO t3 VALUES (66, foo(66, '', '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued3'; + +--connection con_temp5 +# Create transaction T4, participating in group commit with T2 and T3. +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued4'; +send INSERT INTO t3 VALUES (67, foo(67, '', '')); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued4'; +SET debug_sync='now SIGNAL master_cont2'; + +--connection con_temp3 +REAP; +--connection con_temp4 +REAP; +--connection con_temp5 +REAP; + +--connection server_1 +SELECT * FROM t3 WHERE a >= 60 ORDER BY a; +SET debug_sync='RESET'; + +--connection server_2 +# Now we have the four transactions pending for replication on the slave. +# Let them be queued for our three worker threads in a controlled fashion. +# We put them at a stage where T1 is delayed and T3 is waiting for T1 to +# commit before T3 can start. Then we kill T3. + +# Make the worker D0 free, and wait for T1 to be queued in it. +SET debug_sync='now SIGNAL d0_cont'; +SET debug_sync='now WAIT_FOR t1_waiting'; + +# Make the worker D3 free, and wait for T2 to be queued in it. +SET debug_sync='now SIGNAL d3_cont'; +SET debug_sync='now WAIT_FOR t2_waiting'; + +# Now release worker D1, and wait for T3 to be queued in it. +# T3 will wait for T1 to commit before it can start. +SET debug_sync='now SIGNAL d1_cont'; +SET debug_sync='now WAIT_FOR t3_waiting'; + +# Release worker D2. Wait for T4 to be queued, so we are sure it has +# received the debug_sync signal (else we might overwrite it with the +# next debug_sync). +SET debug_sync='now SIGNAL d2_cont'; +SET debug_sync='now WAIT_FOR t4_waiting'; + +# Now we kill the waiting transaction T3 in worker D1. +--replace_result $d1_thd_id THD_ID +eval KILL $d1_thd_id; + +# Wait until T3 has reacted on the kill. +SET debug_sync='now WAIT_FOR t3_killed'; + +# Now we can allow T1 to proceed. +SET debug_sync='now SIGNAL t1_cont'; + +--let $slave_sql_errno= 1317,1927,1964 +--source include/wait_for_slave_sql_error.inc +STOP SLAVE IO_THREAD; +# Since T2, T3, and T4 run in parallel, we can not be sure if T2 will have time +# to commit or not before the stop. However, T1 should commit, and T3/T4 may +# not have committed. (After slave restart we check that all become committed +# eventually). +SELECT * FROM t3 WHERE a >= 60 AND a != 65 ORDER BY a; + +# Now we have to disable the debug_sync statements, so they do not trigger +# when the events are retried. +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +--connection server_1 +UPDATE t3 SET b=b+1 WHERE a=60; +--save_master_pos + +--connection server_2 +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t3 WHERE a >= 60 ORDER BY a; +# Restore the foo() function. +SET sql_log_bin=0; +DROP FUNCTION foo; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + IF d1 != '' THEN + SET debug_sync = d1; + END IF; + IF d2 != '' THEN + SET debug_sync = d2; + END IF; + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +--connection server_2 +# Respawn all worker threads to clear any left-over debug_sync or other stuff. +--source include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** 5. Test killing thread that is waiting for queue of max length to shorten *** + +# Find the thread id of the driver SQL thread that we want to kill. +--let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE '%Slave has read all relay log%' +--source include/wait_condition.inc +--let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE '%Slave has read all relay log%'` +SET @old_max_queued= @@GLOBAL.slave_parallel_max_queued; +SET GLOBAL slave_parallel_max_queued=9000; + +--connection server_1 +--let bigstring= `SELECT REPEAT('x', 10000)` +SET binlog_format=statement; +# Create an event that will wait to be signalled. +INSERT INTO t3 VALUES (80, foo(0, + 'ha_write_row_end SIGNAL query_waiting WAIT_FOR query_cont', '')); + +--connection server_2 +SET debug_sync='now WAIT_FOR query_waiting'; +# Inject that the SQL driver thread will signal `wait_queue_ready' to debug_sync +# as it goes to wait for the event queue to become smaller than the value of +# @@slave_parallel_max_queued. +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_queue_max"; + +--connection server_1 +--disable_query_log +# Create an event that will fill up the queue. +# The Xid event at the end of the event group will have to wait for the Query +# event with the INSERT to drain so the queue becomes shorter. However that in +# turn waits for the prior event group to continue. +eval INSERT INTO t3 VALUES (81, LENGTH('$bigstring')); +--enable_query_log +SELECT * FROM t3 WHERE a >= 80 ORDER BY a; + +--connection server_2 +SET debug_sync='now WAIT_FOR wait_queue_ready'; + +--replace_result $thd_id THD_ID +eval KILL $thd_id; + +SET debug_sync='now WAIT_FOR wait_queue_killed'; +SET debug_sync='now SIGNAL query_cont'; + +--let $slave_sql_errno= 1317,1927,1964 +--source include/wait_for_slave_sql_error.inc +STOP SLAVE IO_THREAD; + +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_max_queued= @old_max_queued; + +--connection server_1 +INSERT INTO t3 VALUES (82,0); +SET binlog_format=@old_format; +--save_master_pos + +--connection server_2 +SET debug_sync='RESET'; +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t3 WHERE a >= 80 ORDER BY a; + + +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + +--echo *** MDEV-5788 Incorrect free of rgi->deferred_events in parallel replication *** + +--connection server_2 +# Use just two worker threads, so we are sure to get the rpl_group_info added +# to the free list, which is what triggered the bug. +--source include/stop_slave.inc +SET GLOBAL replicate_ignore_table="test.t3"; +SET GLOBAL slave_parallel_threads=2; +--source include/start_slave.inc + +--connection server_1 +INSERT INTO t3 VALUES (100, rand()); +INSERT INTO t3 VALUES (101, rand()); + +--save_master_pos + +--connection server_2 +--sync_with_master + +--connection server_1 +INSERT INTO t3 VALUES (102, rand()); +INSERT INTO t3 VALUES (103, rand()); +INSERT INTO t3 VALUES (104, rand()); +INSERT INTO t3 VALUES (105, rand()); + +--save_master_pos + +--connection server_2 +--sync_with_master +--source include/stop_slave.inc +SET GLOBAL replicate_ignore_table=""; +--source include/start_slave.inc + +--connection server_1 +INSERT INTO t3 VALUES (106, rand()); +INSERT INTO t3 VALUES (107, rand()); +--save_master_pos + +--connection server_2 +--sync_with_master +--replace_column 2 # +SELECT * FROM t3 WHERE a >= 100 ORDER BY a; + + +--echo *** MDEV-5921: In parallel replication, an error is not correctly signalled to the next transaction *** + +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + +--connection server_1 +INSERT INTO t3 VALUES (110, 1); +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t3 WHERE a >= 110 ORDER BY a; +# Inject a duplicate key error. +SET sql_log_bin=0; +INSERT INTO t3 VALUES (111, 666); +SET sql_log_bin=1; + +--connection server_1 + +# Create a group commit with two inserts, the first one conflicts with a row on the slave +--connect (con1,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +send INSERT INTO t3 VALUES (111, 2); +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connect (con2,127.0.0.1,root,,test,$SERVER_MYPORT_1,) +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +send INSERT INTO t3 VALUES (112, 3); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con1 +REAP; +--connection con2 +REAP; +SET debug_sync='RESET'; +--save_master_pos + +--connection server_2 +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +--source include/wait_for_slave_sql_to_stop.inc +# We should not see the row (112,3) here, it should be rolled back due to +# error signal from the prior transaction. +SELECT * FROM t3 WHERE a >= 110 ORDER BY a; +SET sql_log_bin=0; +DELETE FROM t3 WHERE a=111 AND b=666; +SET sql_log_bin=1; +START SLAVE SQL_THREAD; +--sync_with_master +SELECT * FROM t3 WHERE a >= 110 ORDER BY a; + + +--echo ***MDEV-5914: Parallel replication deadlock due to InnoDB lock conflicts *** +--connection server_2 +--source include/stop_slave.inc + +--connection server_1 +CREATE TABLE t4 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB; +INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); + +# Create a group commit with UPDATE and DELETE, in that order. +# The bug was that while the UPDATE's row lock does not block the DELETE, the +# DELETE's gap lock _does_ block the UPDATE. This could cause a deadlock +# on the slave. +--connection con1 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +send UPDATE t4 SET b=NULL WHERE a=6; +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con2 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +send DELETE FROM t4 WHERE b <= 3; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con1 +REAP; +--connection con2 +REAP; +SET debug_sync='RESET'; +--save_master_pos + +--connection server_2 +--source include/start_slave.inc +--sync_with_master +--source include/stop_slave.inc + +SELECT * FROM t4 ORDER BY a; + + +# Another example, this one with INSERT vs. DELETE +--connection server_1 +DELETE FROM t4; +INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); + +# Create a group commit with INSERT and DELETE, in that order. +# The bug was that while the INSERT's insert intention lock does not block +# the DELETE, the DELETE's gap lock _does_ block the INSERT. This could cause +# a deadlock on the slave. +--connection con1 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +send INSERT INTO t4 VALUES (7, NULL); +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con2 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +send DELETE FROM t4 WHERE b <= 3; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con1 +REAP; +--connection con2 +REAP; +SET debug_sync='RESET'; +--save_master_pos + +--connection server_2 +--source include/start_slave.inc +--sync_with_master +--source include/stop_slave.inc + +SELECT * FROM t4 ORDER BY a; + + +# MDEV-6549, failing to update gtid_slave_pos for a transaction that was retried. +# The problem was that when a transaction updates the mysql.gtid_slave_pos +# table, it clears the flag that marks that there is a GTID position that +# needs to be updated. Then, if the transaction got killed after that due +# to a deadlock, the subsequent retry would fail to notice that the GTID needs +# to be recorded in gtid_slave_pos. +# +# (In the original bug report, the symptom was an assertion; this was however +# just a side effect of the missing update of gtid_slave_pos, which also +# happened to cause a missing clear of OPTION_GTID_BEGIN). +--connection server_1 +DELETE FROM t4; +INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); + +# Create two transactions that can run in parallel on the slave but cause +# a deadlock if the second runs before the first. +--connection con1 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +send UPDATE t4 SET b=NULL WHERE a=6; +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con2 +# Must use statement-based binlogging. Otherwise the transaction will not be +# binlogged at all, as it modifies no rows. +SET @old_format= @@SESSION.binlog_format; +SET binlog_format='statement'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +send DELETE FROM t4 WHERE b <= 1; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con1 +REAP; +--connection con2 +REAP; +SET @old_format=@@GLOBAL.binlog_format; +SET debug_sync='RESET'; +--save_master_pos +--let $last_gtid= `SELECT @@last_gtid` + +--connection server_2 +# Disable the usual skip of gap locks for transactions that are run in +# parallel, using DBUG. This allows the deadlock to occur, and this in turn +# triggers a retry of the second transaction, and the code that was buggy and +# caused the gtid_slave_pos update to be skipped in the retry. +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,disable_thd_need_ordering_with"; +--source include/start_slave.inc +--sync_with_master +SET GLOBAL debug_dbug=@old_dbug; + +SELECT * FROM t4 ORDER BY a; +# Check that the GTID of the second transaction was correctly recorded in +# gtid_slave_pos, in the variable as well as in the table. +--replace_result $last_gtid GTID +eval SET @last_gtid= '$last_gtid'; +SELECT IF(@@gtid_slave_pos LIKE CONCAT('%',@last_gtid,'%'), "GTID found ok", + CONCAT("GTID ", @last_gtid, " not found in gtid_slave_pos=", @@gtid_slave_pos)) + AS result; +SELECT "ROW FOUND" AS `Is the row found?` + FROM mysql.gtid_slave_pos + WHERE CONCAT(domain_id, "-", server_id, "-", seq_no) = @last_gtid; + + +--echo *** MDEV-5938: Exec_master_log_pos not updated at log rotate in parallel replication *** +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=1; +SET DEBUG_SYNC= 'RESET'; +--source include/start_slave.inc + +--connection server_1 +CREATE TABLE t5 (a INT PRIMARY KEY, b INT); +INSERT INTO t5 VALUES (1,1); +INSERT INTO t5 VALUES (2,2), (3,8); +INSERT INTO t5 VALUES (4,16); +--save_master_pos + +--connection server_2 +--sync_with_master +let $io_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1); +let $io_pos= query_get_value(SHOW SLAVE STATUS, Read_Master_Log_Pos, 1); +let $sql_file= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +let $sql_pos= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1); +--disable_query_log +eval SELECT IF('$io_file' = '$sql_file', "OK", "Not ok, $io_file <> $sql_file") AS test_check; +eval SELECT IF('$io_pos' = '$sql_pos', "OK", "Not ok, $io_pos <> $sql_pos") AS test_check; +--enable_query_log + +--connection server_1 +FLUSH LOGS; +--source include/wait_for_binlog_checkpoint.inc +--save_master_pos + +--connection server_2 +--sync_with_master +let $io_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1); +let $io_pos= query_get_value(SHOW SLAVE STATUS, Read_Master_Log_Pos, 1); +let $sql_file= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +let $sql_pos= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1); +--disable_query_log +eval SELECT IF('$io_file' = '$sql_file', "OK", "Not ok, $io_file <> $sql_file") AS test_check; +eval SELECT IF('$io_pos' = '$sql_pos', "OK", "Not ok, $io_pos <> $sql_pos") AS test_check; +--enable_query_log + + +--echo *** MDEV_6435: Incorrect error handling when query binlogged partially on master with "killed" error *** + +--connection server_1 +CREATE TABLE t6 (a INT) ENGINE=MyISAM; +CREATE TRIGGER tr AFTER INSERT ON t6 FOR EACH ROW SET @a = 1; + +--connection con1 +SET @old_format= @@binlog_format; +SET binlog_format= statement; +--let $conid = `SELECT CONNECTION_ID()` +SET debug_sync='sp_head_execute_before_loop SIGNAL ready WAIT_FOR cont'; +send INSERT INTO t6 VALUES (1), (2), (3); + +--connection server_1 +SET debug_sync='now WAIT_FOR ready'; +--replace_result $conid CONID +eval KILL QUERY $conid; +SET debug_sync='now SIGNAL cont'; + +--connection con1 +--error ER_QUERY_INTERRUPTED +--reap +SET binlog_format= @old_format; +SET debug_sync='RESET'; +--let $after_error_gtid_pos= `SELECT @@gtid_binlog_pos` + +--connection server_1 +SET debug_sync='RESET'; + + +--connection server_2 +--let $slave_sql_errno= 1317 +--source include/wait_for_slave_sql_error.inc +STOP SLAVE IO_THREAD; +--replace_result $after_error_gtid_pos AFTER_ERROR_GTID_POS +eval SET GLOBAL gtid_slave_pos= '$after_error_gtid_pos'; +--source include/start_slave.inc + +--connection server_1 +INSERT INTO t6 VALUES (4); +SELECT * FROM t6 ORDER BY a; +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t6 ORDER BY a; + + +--echo *** MDEV-6551: Some replication errors are ignored if slave_parallel_threads > 0 *** + +--connection server_1 +INSERT INTO t2 VALUES (31); +--let $gtid1= `SELECT @@LAST_GTID` +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads= 0; +--source include/start_slave.inc + +# Force a duplicate key error on the slave. +SET sql_log_bin= 0; +INSERT INTO t2 VALUES (32); +SET sql_log_bin= 1; + +--connection server_1 +INSERT INTO t2 VALUES (32); +--let $gtid2= `SELECT @@LAST_GTID` +# Rotate the binlog; the bug is triggered when the master binlog file changes +# after the event group that causes the duplicate key error. +FLUSH LOGS; +INSERT INTO t2 VALUES (33); +INSERT INTO t2 VALUES (34); +SELECT * FROM t2 WHERE a >= 30 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc + +--connection server_2 +--source include/stop_slave_io.inc +SET GLOBAL slave_parallel_threads=10; +START SLAVE; + +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc + +# Note: IO thread is still running at this point. +# The bug seems to have been that restarting the SQL thread after an error with +# the IO thread still running, somehow picks up a later relay log position and +# thus ends up skipping the failing event, rather than re-executing. + +START SLAVE SQL_THREAD; +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc + +SELECT * FROM t2 WHERE a >= 30 ORDER BY a; + +# Skip the duplicate error, so we can proceed. +--error ER_SLAVE_SKIP_NOT_IN_GTID +SET sql_slave_skip_counter= 1; +--source include/stop_slave_io.inc +--disable_query_log +eval SET GLOBAL gtid_slave_pos = REPLACE(@@gtid_slave_pos, "$gtid1", "$gtid2"); +--enable_query_log +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +SELECT * FROM t2 WHERE a >= 30 ORDER BY a; + + +--echo *** MDEV-6775: Wrong binlog order in parallel replication *** +--connection server_1 +# A bit tricky bug to reproduce. On the master, we binlog in statement-mode +# two transactions, an UPDATE followed by a DELETE. On the slave, we replicate +# with binlog-mode set to ROW, which means the DELETE, which modifies no rows, +# is not binlogged. Then we inject a wait in the group commit code on the +# slave, shortly before the actual commit of the UPDATE. The bug was that the +# DELETE could wake up from wait_for_prior_commit() before the commit of the +# UPDATE. So the test could see the slave position updated to after DELETE, +# while the UPDATE was still not visible. +DELETE FROM t4; +INSERT INTO t4 VALUES (1,NULL), (3,NULL), (4,4), (5, NULL), (6, 6); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,inject_binlog_commit_before_get_LOCK_log"; +SET @old_format=@@GLOBAL.binlog_format; +SET GLOBAL binlog_format=ROW; +# Re-spawn the worker threads to be sure they pick up the new binlog format +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; + +--connection con1 +SET @old_format= @@binlog_format; +SET binlog_format= statement; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +send UPDATE t4 SET b=NULL WHERE a=6; +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con2 +SET @old_format= @@binlog_format; +SET binlog_format= statement; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +send DELETE FROM t4 WHERE b <= 3; + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con1 +REAP; +SET binlog_format= @old_format; +--connection con2 +REAP; +SET binlog_format= @old_format; +SET debug_sync='RESET'; +--save_master_pos +SELECT * FROM t4 ORDER BY a; + +--connection server_2 +--source include/start_slave.inc +SET debug_sync= 'now WAIT_FOR waiting'; +--sync_with_master +SELECT * FROM t4 ORDER BY a; +SET debug_sync= 'now SIGNAL cont'; + +# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL binlog_format= @old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** MDEV-7237: Parallel replication: incorrect relaylog position after stop/start the slave *** +--connection server_1 +INSERT INTO t2 VALUES (40); +--save_master_pos + +--connection server_2 +--sync_with_master +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=no; +SET @old_dbug= @@GLOBAL.debug_dbug; +# This DBUG injection causes a DEBUG_SYNC signal "scheduled_gtid_0_x_100" when +# GTID 0-1-100 has been scheduled for and fetched by a worker thread. +SET GLOBAL debug_dbug="+d,rpl_parallel_scheduled_gtid_0_x_100"; +# This DBUG injection causes a DEBUG_SYNC signal "wait_for_done_waiting" when +# STOP SLAVE has signalled all worker threads to stop. +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; +# Reset worker threads to make DBUG setting catch on. +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; + + +--connection server_1 +# Setup some transaction for the slave to replicate. +INSERT INTO t2 VALUES (41); +INSERT INTO t2 VALUES (42); +# Need to log the DELETE in statement format, so we can see it in processlist. +SET @old_format= @@binlog_format; +SET binlog_format= statement; +DELETE FROM t2 WHERE a=40; +SET binlog_format= @old_format; +INSERT INTO t2 VALUES (43); +INSERT INTO t2 VALUES (44); +# Force the slave to switch to a new relay log file. +FLUSH LOGS; +INSERT INTO t2 VALUES (45); +# Inject a GTID 0-1-100, which will trigger a DEBUG_SYNC signal when this +# transaction has been fetched by a worker thread. +SET gtid_seq_no=100; +INSERT INTO t2 VALUES (46); +--save_master_pos + +--connection con_temp2 +# Temporarily block the DELETE on a=40 from completing. +BEGIN; +SELECT * FROM t2 WHERE a=40 FOR UPDATE; + + +--connection server_2 +--source include/start_slave.inc + +# Wait for a worker thread to start on the DELETE that will be blocked +# temporarily by the SELECT FOR UPDATE. +--let $wait_condition= SELECT count(*) > 0 FROM information_schema.processlist WHERE state='updating' and info LIKE '%DELETE FROM t2 WHERE a=40%' +--source include/wait_condition.inc + +# The DBUG injection set above will make the worker thread signal the following +# debug_sync when the GTID 0-1-100 has been reached by a worker thread. +# Thus, at this point, the SQL driver thread has reached the next +# relay log file name, while a worker thread is still processing a +# transaction in the previous relay log file, blocked on the SELECT FOR +# UPDATE. +SET debug_sync= 'now WAIT_FOR scheduled_gtid_0_x_100'; +# At this point, the SQL driver thread is in the new relay log file, while +# the DELETE from the old relay log file is not yet complete. We will stop +# the slave at this point. The bug was that the DELETE statement would +# update the slave position to the _new_ relay log file name instead of +# its own old file name. Thus, by stoping and restarting the slave at this +# point, we would get an error at restart due to incorrect position. (If +# we would let the slave catch up before stopping, the incorrect position +# would be corrected by a later transaction). + +send STOP SLAVE; + +--connection con_temp2 +# Wait for STOP SLAVE to have proceeded sufficiently that it has signalled +# all worker threads to stop; this ensures that we will stop after the DELETE +# transaction (and not after a later transaction that might have been able +# to set a fixed position). +SET debug_sync= 'now WAIT_FOR wait_for_done_waiting'; +# Now release the row lock that was blocking the replication of DELETE. +ROLLBACK; + +--connection server_2 +reap; +--source include/wait_for_slave_sql_to_stop.inc +SELECT * FROM t2 WHERE a >= 40 ORDER BY a; +# Now restart the slave. With the bug present, this would start at an +# incorrect relay log position, causing relay log read error (or if unlucky, +# silently skip a number of events). +--source include/start_slave.inc +--sync_with_master +SELECT * FROM t2 WHERE a >= 40 ORDER BY a; +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET DEBUG_SYNC= 'RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + + +--echo *** MDEV-7326 Server deadlock in connection with parallel replication *** +# We use three transactions, each in a separate group commit. +# T1 does mark_start_commit(), then gets a deadlock error. +# T2 wakes up and starts running +# T1 does unmark_start_commit() +# T3 goes to wait for T2 to start its commit +# T2 does mark_start_commit() +# The bug was that at this point, T3 got deadlocked. Because T1 has unmarked(), +# T3 did not yet see the count_committing_event_groups reach its target value +# yet. But when T1 later re-did mark_start_commit(), it failed to send a wakeup +# to T3. + +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=3; +SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; +--source include/start_slave.inc + +--connection server_1 +SET @old_format= @@SESSION.binlog_format; +SET binlog_format= STATEMENT; +# This debug_sync will linger on and be used to control T3 later. +INSERT INTO t1 VALUES (foo(50, + "rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", + "rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); +--save_master_pos +--connection server_2 +# Wait for the debug_sync point for T3 to be set. But let the preparation +# transaction remain hanging, so that T1 and T2 will be scheduled for the +# remaining two worker threads. +SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; + +--connection server_1 +INSERT INTO t2 VALUES (foo(50, + "rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", + "rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); +--save_master_pos + +--connection server_2 +SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; +# T1 has now done mark_start_commit(). It will later do a rollback and retry. + +--connection server_1 +# Use a MyISAM table for T2 and T3, so they do not trigger the +# rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event. +INSERT INTO t1 VALUES (foo(51, + "rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", + "rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); + +--connection server_2 +SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; +# T2 has now started running, but has not yet done mark_start_commit() +SET DEBUG_SYNC= "now SIGNAL t1_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; +# T1 has now done unmark_start_commit() in preparation for its retry. + +--connection server_1 +INSERT INTO t1 VALUES (52); +SET BINLOG_FORMAT= @old_format; +SELECT * FROM t2 WHERE a>=50 ORDER BY a; +SELECT * FROM t1 WHERE a>=50 ORDER BY a; + +--connection server_2 +# Let the preparation transaction complete, so that the same worker thread +# can continue with the transaction T3. +SET DEBUG_SYNC= "now SIGNAL prep_cont"; +SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; +# T3 has now gone to wait for T2 to start committing +SET DEBUG_SYNC= "now SIGNAL t2_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; +# T2 has now done mark_start_commit(). +# Let things run, and check that T3 does not get deadlocked. +SET DEBUG_SYNC= "now SIGNAL t1_cont2"; +--sync_with_master + +--connection server_1 +--save_master_pos +--connection server_2 +--sync_with_master +SELECT * FROM t2 WHERE a>=50 ORDER BY a; +SELECT * FROM t1 WHERE a>=50 ORDER BY a; +SET DEBUG_SYNC="reset"; + +# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** MDEV-7326 Server deadlock in connection with parallel replication *** +# Similar to the previous test, but with T2 and T3 in the same GCO. +# We use three transactions, T1 in one group commit and T2/T3 in another. +# T1 does mark_start_commit(), then gets a deadlock error. +# T2 wakes up and starts running +# T1 does unmark_start_commit() +# T3 goes to wait for T1 to start its commit +# T2 does mark_start_commit() +# The bug was that at this point, T3 got deadlocked. T2 increments the +# count_committing_event_groups but does not signal T3, as they are in +# the same GCO. Then later when T1 increments, it would also not signal +# T3, because now the count_committing_event_groups is not equal to the +# wait_count of T3 (it is one larger). + +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=3; +SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; +--source include/start_slave.inc + +--connection server_1 +SET @old_format= @@SESSION.binlog_format; +SET binlog_format= STATEMENT; +# This debug_sync will linger on and be used to control T3 later. +INSERT INTO t1 VALUES (foo(60, + "rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", + "rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); +--save_master_pos +--connection server_2 +# Wait for the debug_sync point for T3 to be set. But let the preparation +# transaction remain hanging, so that T1 and T2 will be scheduled for the +# remaining two worker threads. +SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; + +--connection server_1 +INSERT INTO t2 VALUES (foo(60, + "rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", + "rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); +--save_master_pos + +--connection server_2 +SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; +# T1 has now done mark_start_commit(). It will later do a rollback and retry. + +# Do T2 and T3 in a single group commit. +# Use a MyISAM table for T2 and T3, so they do not trigger the +# rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event. +--connection con_temp3 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +send INSERT INTO t1 VALUES (foo(61, + "rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", + "rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued1'; + +--connection con_temp4 +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +send INSERT INTO t6 VALUES (62); + +--connection server_1 +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; + +--connection con_temp3 +REAP; +--connection con_temp4 +REAP; + +--connection server_1 +SET debug_sync='RESET'; +SET BINLOG_FORMAT= @old_format; +SELECT * FROM t2 WHERE a>=60 ORDER BY a; +SELECT * FROM t1 WHERE a>=60 ORDER BY a; +SELECT * FROM t6 WHERE a>=60 ORDER BY a; + +--connection server_2 +SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; +# T2 has now started running, but has not yet done mark_start_commit() +SET DEBUG_SYNC= "now SIGNAL t1_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; +# T1 has now done unmark_start_commit() in preparation for its retry. + +--connection server_2 +# Let the preparation transaction complete, so that the same worker thread +# can continue with the transaction T3. +SET DEBUG_SYNC= "now SIGNAL prep_cont"; +SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; +# T3 has now gone to wait for T2 to start committing +SET DEBUG_SYNC= "now SIGNAL t2_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; +# T2 has now done mark_start_commit(). +# Let things run, and check that T3 does not get deadlocked. +SET DEBUG_SYNC= "now SIGNAL t1_cont2"; +--sync_with_master + +--connection server_1 +--save_master_pos +--connection server_2 +--sync_with_master +SELECT * FROM t2 WHERE a>=60 ORDER BY a; +SELECT * FROM t1 WHERE a>=60 ORDER BY a; +SELECT * FROM t6 WHERE a>=60 ORDER BY a; +SET DEBUG_SYNC="reset"; + +# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + +--echo *** MDEV-7335: Potential parallel slave deadlock with specific binlog corruption *** + +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=1; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,slave_discard_xid_for_gtid_0_x_1000"; + +--connection server_1 +INSERT INTO t2 VALUES (101); +INSERT INTO t2 VALUES (102); +INSERT INTO t2 VALUES (103); +INSERT INTO t2 VALUES (104); +INSERT INTO t2 VALUES (105); +# Inject a partial event group (missing XID at the end). The bug was that such +# partial group was not handled appropriately, leading to server deadlock. +SET gtid_seq_no=1000; +INSERT INTO t2 VALUES (106); +INSERT INTO t2 VALUES (107); +INSERT INTO t2 VALUES (108); +INSERT INTO t2 VALUES (109); +INSERT INTO t2 VALUES (110); +INSERT INTO t2 VALUES (111); +INSERT INTO t2 VALUES (112); +INSERT INTO t2 VALUES (113); +INSERT INTO t2 VALUES (114); +INSERT INTO t2 VALUES (115); +INSERT INTO t2 VALUES (116); +INSERT INTO t2 VALUES (117); +INSERT INTO t2 VALUES (118); +INSERT INTO t2 VALUES (119); +INSERT INTO t2 VALUES (120); +INSERT INTO t2 VALUES (121); +INSERT INTO t2 VALUES (122); +INSERT INTO t2 VALUES (123); +INSERT INTO t2 VALUES (124); +INSERT INTO t2 VALUES (125); +INSERT INTO t2 VALUES (126); +INSERT INTO t2 VALUES (127); +INSERT INTO t2 VALUES (128); +INSERT INTO t2 VALUES (129); +INSERT INTO t2 VALUES (130); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +# The partial event group (a=106) should be rolled back and thus missing. +SELECT * FROM t2 WHERE a >= 100 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + +--echo *** MDEV-6676 - test syntax of @@slave_parallel_mode *** +--connection server_2 + +--let $status_items= Parallel_Mode +--source include/show_slave_status.inc +--source include/stop_slave.inc +SET GLOBAL slave_parallel_mode='aggressive'; +--let $status_items= Parallel_Mode +--source include/show_slave_status.inc +SET GLOBAL slave_parallel_mode='conservative'; +--let $status_items= Parallel_Mode +--source include/show_slave_status.inc + + +--echo *** MDEV-6676 - test that empty parallel_mode does not replicate in parallel *** +--connection server_1 +INSERT INTO t2 VALUES (1040); +--source include/save_master_gtid.inc + +--connection server_2 +SET GLOBAL slave_parallel_mode='none'; +# Test that we do not use parallel apply, by injecting an unconditional +# crash in the parallel apply code. +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,slave_crash_if_parallel_apply"; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 1040 ORDER BY a; +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; + + +--echo *** MDEV-6676 - test disabling domain-based parallel replication *** +--connection server_1 +# Let's do a bunch of transactions that will conflict if run out-of-order in +# domain-based parallel replication mode. +SET gtid_domain_id = 1; +INSERT INTO t2 VALUES (1041); +INSERT INTO t2 VALUES (1042); +INSERT INTO t2 VALUES (1043); +INSERT INTO t2 VALUES (1044); +INSERT INTO t2 VALUES (1045); +INSERT INTO t2 VALUES (1046); +DELETE FROM t2 WHERE a >= 1041; +SET gtid_domain_id = 2; +INSERT INTO t2 VALUES (1041); +INSERT INTO t2 VALUES (1042); +INSERT INTO t2 VALUES (1043); +INSERT INTO t2 VALUES (1044); +INSERT INTO t2 VALUES (1045); +INSERT INTO t2 VALUES (1046); +SET gtid_domain_id = 0; +--source include/save_master_gtid.inc +--connection server_2 +SET GLOBAL slave_parallel_mode=minimal; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 1040 ORDER BY a; +--source include/stop_slave.inc +SET GLOBAL slave_parallel_mode='conservative'; +--source include/start_slave.inc + + +--echo *** MDEV-7847: "Slave worker thread retried transaction 10 time(s) in vain, giving up", followed by replication hanging *** +--echo *** MDEV-7882: Excessive transaction retry in parallel replication *** + +--connection server_1 +CREATE TABLE t7 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; +CREATE TABLE t8 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; +--save_master_pos + +--connection server_2 +--sync_with_master +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=40; +SELECT @old_retries:=@@GLOBAL.slave_transaction_retries; +SET GLOBAL slave_transaction_retries= 5; + + +# Using dbug error injection, we artificially create event groups with a lot of +# conflicting transactions in each event group. The bugs were originally seen +# "in the wild" with transactions that did not conflict on the master, and only +# conflicted very rarely on the slave (maybe some edge case with InnoDB btree +# page splits or something like that). The event groups here loosely reflect +# the structure of the original failure's group commits. + + +--connection server_1 +INSERT INTO t7 VALUES (1,1), (2,2), (3,3), (4,4), (5,5); +SET @old_dbug= @@SESSION.debug_dbug; +SET @commit_id= 42; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +INSERT INTO t8 VALUES (1,1); +INSERT INTO t8 VALUES (2,2); +INSERT INTO t8 VALUES (3,3); +INSERT INTO t8 VALUES (4,4); +INSERT INTO t8 VALUES (5,5); +INSERT INTO t8 VALUES (6,6); +INSERT INTO t8 VALUES (7,7); +INSERT INTO t8 VALUES (8,8); + +UPDATE t7 SET b=9 WHERE a=3; +UPDATE t7 SET b=10 WHERE a=3; +UPDATE t7 SET b=11 WHERE a=3; + +INSERT INTO t8 VALUES (12,12); +INSERT INTO t8 VALUES (13,13); + +UPDATE t7 SET b=14 WHERE a=3; +UPDATE t7 SET b=15 WHERE a=3; + +INSERT INTO t8 VALUES (16,16); + +UPDATE t7 SET b=17 WHERE a=3; + +INSERT INTO t8 VALUES (18,18); +INSERT INTO t8 VALUES (19,19); + +UPDATE t7 SET b=20 WHERE a=3; + +INSERT INTO t8 VALUES (21,21); + +UPDATE t7 SET b=22 WHERE a=3; + +INSERT INTO t8 VALUES (23,24); +INSERT INTO t8 VALUES (24,24); + +UPDATE t7 SET b=25 WHERE a=3; + +INSERT INTO t8 VALUES (26,26); + +UPDATE t7 SET b=27 WHERE a=3; + +BEGIN; +INSERT INTO t8 VALUES (28,28); +INSERT INTO t8 VALUES (29,28), (30,28); +INSERT INTO t8 VALUES (31,28); +INSERT INTO t8 VALUES (32,28); +INSERT INTO t8 VALUES (33,28); +INSERT INTO t8 VALUES (34,28); +INSERT INTO t8 VALUES (35,28); +INSERT INTO t8 VALUES (36,28); +INSERT INTO t8 VALUES (37,28); +INSERT INTO t8 VALUES (38,28); +INSERT INTO t8 VALUES (39,28); +INSERT INTO t8 VALUES (40,28); +INSERT INTO t8 VALUES (41,28); +INSERT INTO t8 VALUES (42,28); +COMMIT; + + +SET @commit_id=43; +INSERT INTO t8 VALUES (43,43); +INSERT INTO t8 VALUES (44,44); + +UPDATE t7 SET b=45 WHERE a=3; + +INSERT INTO t8 VALUES (46,46); +INSERT INTO t8 VALUES (47,47); + +UPDATE t7 SET b=48 WHERE a=3; + +INSERT INTO t8 VALUES (49,49); +INSERT INTO t8 VALUES (50,50); + + +SET @commit_id=44; +INSERT INTO t8 VALUES (51,51); +INSERT INTO t8 VALUES (52,52); + +UPDATE t7 SET b=53 WHERE a=3; + +INSERT INTO t8 VALUES (54,54); +INSERT INTO t8 VALUES (55,55); + +UPDATE t7 SET b=56 WHERE a=3; + +INSERT INTO t8 VALUES (57,57); + +UPDATE t7 SET b=58 WHERE a=3; + +INSERT INTO t8 VALUES (58,58); +INSERT INTO t8 VALUES (59,59); +INSERT INTO t8 VALUES (60,60); +INSERT INTO t8 VALUES (61,61); + +UPDATE t7 SET b=62 WHERE a=3; + +INSERT INTO t8 VALUES (63,63); +INSERT INTO t8 VALUES (64,64); +INSERT INTO t8 VALUES (65,65); +INSERT INTO t8 VALUES (66,66); + +UPDATE t7 SET b=67 WHERE a=3; + +INSERT INTO t8 VALUES (68,68); + +UPDATE t7 SET b=69 WHERE a=3; +UPDATE t7 SET b=70 WHERE a=3; +UPDATE t7 SET b=71 WHERE a=3; + +INSERT INTO t8 VALUES (72,72); + +UPDATE t7 SET b=73 WHERE a=3; +UPDATE t7 SET b=74 WHERE a=3; +UPDATE t7 SET b=75 WHERE a=3; +UPDATE t7 SET b=76 WHERE a=3; + +INSERT INTO t8 VALUES (77,77); + +UPDATE t7 SET b=78 WHERE a=3; + +INSERT INTO t8 VALUES (79,79); + +UPDATE t7 SET b=80 WHERE a=3; + +INSERT INTO t8 VALUES (81,81); + +UPDATE t7 SET b=82 WHERE a=3; + +INSERT INTO t8 VALUES (83,83); + +UPDATE t7 SET b=84 WHERE a=3; + + +SET @commit_id=45; +INSERT INTO t8 VALUES (85,85); +UPDATE t7 SET b=86 WHERE a=3; +INSERT INTO t8 VALUES (87,87); + + +SET @commit_id=46; +INSERT INTO t8 VALUES (88,88); +INSERT INTO t8 VALUES (89,89); +INSERT INTO t8 VALUES (90,90); + +SET SESSION debug_dbug=@old_dbug; + +INSERT INTO t8 VALUES (91,91); +INSERT INTO t8 VALUES (92,92); +INSERT INTO t8 VALUES (93,93); +INSERT INTO t8 VALUES (94,94); +INSERT INTO t8 VALUES (95,95); +INSERT INTO t8 VALUES (96,96); +INSERT INTO t8 VALUES (97,97); +INSERT INTO t8 VALUES (98,98); +INSERT INTO t8 VALUES (99,99); + + +SELECT * FROM t7 ORDER BY a; +SELECT * FROM t8 ORDER BY a; +--source include/save_master_gtid.inc + + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t7 ORDER BY a; +SELECT * FROM t8 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL slave_transaction_retries= @old_retries; +SET GLOBAL slave_parallel_threads=10; +--source include/start_slave.inc + + +--echo *** MDEV-7888: ANALYZE TABLE does wakeup_subsequent_commits(), causing wrong binlog order and parallel replication hang *** + +--connection server_2 +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_analyze_table_sleep'; + +--connection server_1 +# Inject two group commits. The bug was that ANALYZE TABLE would call +# wakeup_subsequent_commits() too early, allowing the following transaction +# in the same group to run ahead and binlog and free the GCO. Then we get +# wrong binlog order and later access freed GCO, which causes lost wakeup +# of following GCO and thus replication hang. +# We injected a small sleep in ANALYZE to make the race easier to hit (this +# can only cause false negatives in versions with the bug, not false positives, +# so sleep is ok here. And it's in general not possible to trigger reliably +# the race with debug_sync, since the bugfix makes the race impossible). + +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; + +# Group commit with cid=10000, two event groups. +SET @commit_id= 10000; +ANALYZE TABLE t2; +INSERT INTO t3 VALUES (120, 0); + +# Group commit with cid=10001, one event group. +SET @commit_id= 10001; +INSERT INTO t3 VALUES (121, 0); + +SET SESSION debug_dbug=@old_dbug; + +SELECT * FROM t3 WHERE a >= 120 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +SELECT * FROM t3 WHERE a >= 120 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL debug_dbug= @old_dbug; +--source include/start_slave.inc + + +--echo *** MDEV-7929: record_gtid() for non-transactional event group calls wakeup_subsequent_commits() too early, causing slave hang. *** + +--connection server_2 +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_record_gtid_serverid_100_sleep'; + +--connection server_1 +# Inject two group commits. The bug was that record_gtid for a +# non-transactional event group would commit its own transaction, which would +# cause ha_commit_trans() to call wakeup_subsequent_commits() too early. This +# in turn lead to access to freed group_commit_orderer object, losing a wakeup +# and causing slave threads to hang. +# We inject a small sleep in the corresponding record_gtid() to make the race +# easier to hit. + +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; + +# Group commit with cid=10010, two event groups. +SET @old_server_id= @@SESSION.server_id; +SET SESSION server_id= 100; +SET @commit_id= 10010; +ALTER TABLE t1 COMMENT "Hulubulu!"; +SET SESSION server_id= @old_server_id; +INSERT INTO t3 VALUES (130, 0); + +# Group commit with cid=10011, one event group. +SET @commit_id= 10011; +INSERT INTO t3 VALUES (131, 0); + +SET SESSION debug_dbug=@old_dbug; + +SELECT * FROM t3 WHERE a >= 130 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +SELECT * FROM t3 WHERE a >= 130 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL debug_dbug= @old_dbug; +--source include/start_slave.inc + + +--echo *** MDEV-8031: Parallel replication stops on "connection killed" error (probably incorrectly handled deadlock kill) *** + +--connection server_1 +INSERT INTO t3 VALUES (201,0), (202,0); +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_mdev8031'; + +--connection server_1 +# We artificially create a situation that hopefully resembles the original +# bug which was only seen "in the wild", and only once. +# Setup a fake group commit with lots of conflicts that will lead to deadloc +# kill. The slave DBUG injection causes the slave to be deadlock killed at +# a particular point during the retry, and then later do a small sleep at +# another critical point where the prior transaction then has a chance to +# complete. Finally an extra KILL check catches an unhandled, lingering +# deadlock kill. So rather artificial, but at least it exercises the +# relevant code paths. +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; + +SET @commit_id= 10200; +INSERT INTO t3 VALUES (203, 1); +INSERT INTO t3 VALUES (204, 1); +INSERT INTO t3 VALUES (205, 1); +UPDATE t3 SET b=b+1 WHERE a=201; +UPDATE t3 SET b=b+1 WHERE a=201; +UPDATE t3 SET b=b+1 WHERE a=201; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=203; +UPDATE t3 SET b=b+1 WHERE a=203; +UPDATE t3 SET b=b+1 WHERE a=204; +UPDATE t3 SET b=b+1 WHERE a=204; +UPDATE t3 SET b=b+1 WHERE a=204; +UPDATE t3 SET b=b+1 WHERE a=203; +UPDATE t3 SET b=b+1 WHERE a=205; +UPDATE t3 SET b=b+1 WHERE a=205; +SET SESSION debug_dbug=@old_dbug; + +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +--source include/stop_slave.inc +SET GLOBAL debug_dbug= @old_dbug; +--source include/start_slave.inc + + +--echo *** Check getting deadlock killed inside open_binlog() during retry. *** + +--connection server_2 +--source include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_retry_event_group_open_binlog_kill'; +SET @old_max= @@GLOBAL.max_relay_log_size; +SET GLOBAL max_relay_log_size= 4096; + +--connection server_1 +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; + +--let $large= `SELECT REPEAT("*", 8192)` +SET @commit_id= 10210; +--echo Omit long queries that cause relaylog rotations and transaction retries... +--disable_query_log +eval UPDATE t3 SET b=b+1 WHERE a=201 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=201 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=201 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=203 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=203 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=204 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=204 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=204 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=203 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=205 /* $large */; +eval UPDATE t3 SET b=b+1 WHERE a=205 /* $large */; +--enable_query_log +SET SESSION debug_dbug=@old_dbug; + +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +--source include/stop_slave.inc +SET GLOBAL debug_dbug= @old_debg; +SET GLOBAL max_relay_log_size= @old_max; +--source include/start_slave.inc + + +--echo *** MDEV-8302: Duplicate key with parallel replication *** + +--connection server_2 +--source include/stop_slave.inc +/* Inject a small sleep which makes the race easier to hit. */ +SET @old_dbug=@@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,inject_mdev8302"; + + +--connection server_1 +INSERT INTO t7 VALUES (100,1), (101,2), (102,3), (103,4), (104,5); + +# Artificially create a bunch of group commits with conflicting transactions. +# The bug happened when T1 and T2 was in one group commit, and T3 was in the +# following group commit. T2 is a DELETE of a row with same primary key as a +# row that T3 inserts. T1 and T2 can conflict, causing T2 to be deadlock +# killed after starting to commit. The bug was that T2 could roll back before +# doing unmark_start_commit(); this could allow T3 to run before the retry +# of T2, causing duplicate key violation. + +SET @old_dbug= @@SESSION.debug_dbug; +SET @commit_id= 20000; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; + +--let $n = 100 +--disable_query_log +while ($n) +{ + eval UPDATE t7 SET b=b+1 WHERE a=100+($n MOD 5); + eval DELETE FROM t7 WHERE a=100+($n MOD 5); + + SET @commit_id = @commit_id + 1; + eval INSERT INTO t7 VALUES (100+($n MOD 5), $n); + SET @commit_id = @commit_id + 1; + dec $n; +} +--enable_query_log +SET SESSION debug_dbug=@old_dbug; + + +SELECT * FROM t7 ORDER BY a; +--source include/save_master_gtid.inc + + +--connection server_2 +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc +SELECT * FROM t7 ORDER BY a; + +--source include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +--source include/start_slave.inc + + + +--echo *** MDEV-8725: Assertion on ROLLBACK statement in the binary log *** +--connection server_1 +# Inject an event group terminated by ROLLBACK, by mixing MyISAM and InnoDB +# in a transaction. The bug was an assertion on the ROLLBACK due to +# mark_start_commit() being already called. +--disable_warnings +BEGIN; +INSERT INTO t2 VALUES (2000); +INSERT INTO t1 VALUES (2000); +INSERT INTO t2 VALUES (2001); +ROLLBACK; +--enable_warnings +SELECT * FROM t1 WHERE a>=2000 ORDER BY a; +SELECT * FROM t2 WHERE a>=2000 ORDER BY a; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 WHERE a>=2000 ORDER BY a; +SELECT * FROM t2 WHERE a>=2000 ORDER BY a; + + +# Clean up. +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc +SET DEBUG_SYNC= 'RESET'; + +--connection server_1 +DROP function foo; +DROP TABLE t1,t2,t3,t4,t5,t6,t7,t8; +SET DEBUG_SYNC= 'RESET'; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_parallel_show_binlog_events_purge_logs.inc b/mysql-test/extra/rpl_tests/rpl_parallel_show_binlog_events_purge_logs.inc new file mode 100644 index 0000000000000..9cbcf01f46b48 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_parallel_show_binlog_events_purge_logs.inc @@ -0,0 +1,40 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# BUG#13979418: SHOW BINLOG EVENTS MAY CRASH THE SERVER +# +# The function mysql_show_binlog_events has a local stack variable +# 'LOG_INFO linfo;', which is assigned to thd->current_linfo, however +# this variable goes out of scope and is destroyed before clean +# thd->current_linfo. +# +# This test case runs SHOW BINLOG EVENTS and FLUSH LOGS to make sure +# that with the fix local variable linfo is valid along all +# mysql_show_binlog_events function scope. +# +--source include/have_debug_sync.inc +--source include/master-slave.inc + +--echo [connection slave] +--connection slave +SET DEBUG_SYNC= 'after_show_binlog_events SIGNAL on_show_binlog_events WAIT_FOR end'; +--send SHOW BINLOG EVENTS + +--connection slave1 +--echo [connection slave1] +SET DEBUG_SYNC= 'now WAIT_FOR on_show_binlog_events'; +FLUSH LOGS; +SET DEBUG_SYNC= 'now SIGNAL end'; + +--echo [connection slave] +--connection slave +--disable_result_log +--reap +--enable_result_log +SET DEBUG_SYNC= 'RESET'; + +--connection master +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_relayrotate.inc b/mysql-test/extra/rpl_tests/rpl_relayrotate.inc new file mode 100644 index 0000000000000..ce638e419ff10 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_relayrotate.inc @@ -0,0 +1,18 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +####################################################### +# Wrapper for rpl_relayrotate.test to allow multi # +# Engines to reuse test code. By JBM 2006-02-15 # +####################################################### +-- source include/have_innodb.inc +# Slow test, don't run during staging part +-- source include/not_staging.inc +-- source include/master-slave.inc + +let $engine_type=innodb; +-- source extra/rpl_tests/rpl_relayrotate.test +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_semi_sync.inc b/mysql-test/extra/rpl_tests/rpl_semi_sync.inc new file mode 100644 index 0000000000000..456a750e89f68 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_semi_sync.inc @@ -0,0 +1,580 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +source include/have_semisync.inc; +source include/not_embedded.inc; +source include/have_innodb.inc; +source include/master-slave.inc; + +let $engine_type= InnoDB; +#let $engine_type= MyISAM; + +# Suppress warnings that might be generated during the test +connection master; +call mtr.add_suppression("Timeout waiting for reply of binlog"); +call mtr.add_suppression("Read semi-sync reply"); +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +connection slave; +call mtr.add_suppression("Master server does not support semi-sync"); +call mtr.add_suppression("Semi-sync slave .* reply"); +call mtr.add_suppression("Slave SQL.*Request to stop slave SQL Thread received while applying a group that has non-transactional changes; waiting for completion of the group"); +connection master; + +# wait for dying connections (if any) to disappear +let $wait_condition= select count(*) = 0 from information_schema.processlist where command='killed'; +--source include/wait_condition.inc + +# After fix of BUG#45848, semi-sync slave should not create any extra +# connections on master, save the count of connections before start +# semi-sync slave for comparison below. +let $_connections_normal_slave= query_get_value(SHOW STATUS LIKE 'Threads_connected', Value, 1); + +--echo # +--echo # Uninstall semi-sync plugins on master and slave +--echo # +connection slave; +source include/stop_slave.inc; +reset slave; +set global rpl_semi_sync_master_enabled= 0; +set global rpl_semi_sync_slave_enabled= 0; + +connection master; +reset master; +set global rpl_semi_sync_master_enabled= 0; +set global rpl_semi_sync_slave_enabled= 0; + +--echo # +--echo # Main test of semi-sync replication start here +--echo # + +connection master; +echo [ on master ]; + +set global rpl_semi_sync_master_timeout= 60000; # 60s + +echo [ default state of semi-sync on master should be OFF ]; +show variables like 'rpl_semi_sync_master_enabled'; + +echo [ enable semi-sync on master ]; +set global rpl_semi_sync_master_enabled = 1; +show variables like 'rpl_semi_sync_master_enabled'; + +echo [ status of semi-sync on master should be ON even without any semi-sync slaves ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +--echo # +--echo # BUG#45672 Semisync repl: ActiveTranx:insert_tranx_node: transaction node allocation failed +--echo # BUG#45673 Semisynch reports correct operation even if no slave is connected +--echo # + +# BUG#45672 When semi-sync is enabled on master, it would allocate +# transaction node even without semi-sync slave connected, and would +# finally result in transaction node allocation error. +# +# Semi-sync master will pre-allocate 'max_connections' transaction +# nodes, so here we do more than that much transactions to check if it +# will fail or not. +# select @@global.max_connections + 1; +let $i= `select @@global.max_connections + 1`; +disable_query_log; +eval create table t1 (a int) engine=$engine_type; +while ($i) +{ + eval insert into t1 values ($i); + dec $i; +} +drop table t1; +enable_query_log; + +# BUG#45673 +echo [ status of semi-sync on master should be OFF ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +--replace_result 305 304 +show status like 'Rpl_semi_sync_master_yes_tx'; + +# reset master to make sure the following test will start with a clean environment +reset master; + +connection slave; +echo [ on slave ]; + +echo [ default state of semi-sync on slave should be OFF ]; +show variables like 'rpl_semi_sync_slave_enabled'; + +echo [ enable semi-sync on slave ]; +set global rpl_semi_sync_slave_enabled = 1; +show variables like 'rpl_semi_sync_slave_enabled'; +source include/start_slave.inc; + +connection master; +echo [ on master ]; + +# NOTE: Rpl_semi_sync_master_client will only be updated when +# semi-sync slave has started binlog dump request +let $status_var= Rpl_semi_sync_master_clients; +let $status_var_value= 1; +source include/wait_for_status_var.inc; + +echo [ initial master state after the semi-sync slave connected ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +replace_result $engine_type ENGINE_TYPE; +eval create table t1(a int) engine = $engine_type; + +echo [ master state after CREATE TABLE statement ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +# After fix of BUG#45848, semi-sync slave should not create any extra +# connections on master. +let $_connections_semisync_slave= query_get_value(SHOW STATUS LIKE 'Threads_connected', Value, 1); +replace_result $_connections_normal_slave CONNECTIONS_NORMAL_SLAVE $_connections_semisync_slave CONNECTIONS_SEMISYNC_SLAVE; +eval select $_connections_semisync_slave - $_connections_normal_slave as 'Should be 0'; + +echo [ insert records to table ]; +insert t1 values (10); +insert t1 values (9); +insert t1 values (8); +insert t1 values (7); +insert t1 values (6); +insert t1 values (5); +insert t1 values (4); +insert t1 values (3); +insert t1 values (2); +insert t1 values (1); + +echo [ master status after inserts ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +sync_slave_with_master; +echo [ on slave ]; + +echo [ slave status after replicated inserts ]; +show status like 'Rpl_semi_sync_slave_status'; + +select count(distinct a) from t1; +select min(a) from t1; +select max(a) from t1; + +--echo +--echo # BUG#50157 +--echo # semi-sync replication crashes when replicating a transaction which +--echo # include 'CREATE TEMPORARY TABLE `MyISAM_t` SELECT * FROM `Innodb_t` ; + +connection master; +echo [ on master ]; +SET SESSION AUTOCOMMIT= 0; +CREATE TABLE t2(c1 INT) ENGINE=innodb; +sync_slave_with_master; + +connection master; +BEGIN; +--echo +--echo # Even though it is in a transaction, this statement is binlogged into binlog +--echo # file immediately. +--disable_warnings +CREATE TEMPORARY TABLE t3 SELECT c1 FROM t2 where 1=1; +--enable_warnings +--echo +--echo # These statements will not be binlogged until the transaction is committed +INSERT INTO t2 VALUES(11); +INSERT INTO t2 VALUES(22); +COMMIT; + +DROP TABLE t2, t3; +SET SESSION AUTOCOMMIT= 1; +sync_slave_with_master; + + +--echo # +--echo # Test semi-sync master will switch OFF after one transaction +--echo # timeout waiting for slave reply. +--echo # +connection slave; +source include/stop_slave.inc; + +connection master; +echo [ on master ]; +set global rpl_semi_sync_master_timeout= 5000; + +# The first semi-sync check should be on because after slave stop, +# there are no transactions on the master. +echo [ master status should be ON ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +--replace_result 305 304 +show status like 'Rpl_semi_sync_master_yes_tx'; +show status like 'Rpl_semi_sync_master_clients'; + +echo [ semi-sync replication of these transactions will fail ]; +insert into t1 values (500); + +# Wait for the semi-sync replication of this transaction to timeout +let $status_var= Rpl_semi_sync_master_status; +let $status_var_value= OFF; +source include/wait_for_status_var.inc; + +# The second semi-sync check should be off because one transaction +# times out during waiting. +echo [ master status should be OFF ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +--replace_result 305 304 +show status like 'Rpl_semi_sync_master_yes_tx'; + +# Semi-sync status on master is now OFF, so all these transactions +# will be replicated asynchronously. +delete from t1 where a=10; +delete from t1 where a=9; +delete from t1 where a=8; +delete from t1 where a=7; +delete from t1 where a=6; +delete from t1 where a=5; +delete from t1 where a=4; +delete from t1 where a=3; +delete from t1 where a=2; +delete from t1 where a=1; + +insert into t1 values (100); + +echo [ master status should be OFF ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +--replace_result 305 304 +show status like 'Rpl_semi_sync_master_yes_tx'; + +--echo # +--echo # Test semi-sync status on master will be ON again when slave catches up +--echo # + +# Save the master position for later use. +save_master_pos; + +connection slave; +echo [ on slave ]; + +echo [ slave status should be OFF ]; +show status like 'Rpl_semi_sync_slave_status'; +source include/start_slave.inc; +sync_with_master; + +echo [ slave status should be ON ]; +show status like 'Rpl_semi_sync_slave_status'; + +select count(distinct a) from t1; +select min(a) from t1; +select max(a) from t1; + +connection master; +echo [ on master ]; + +# The master semi-sync status should be on again after slave catches up. +echo [ master status should be ON again after slave catches up ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +--replace_result 305 304 +show status like 'Rpl_semi_sync_master_yes_tx'; +show status like 'Rpl_semi_sync_master_clients'; + +--echo # +--echo # Test disable/enable master semi-sync on the fly. +--echo # + +drop table t1; +sync_slave_with_master; +echo [ on slave ]; + +source include/stop_slave.inc; + +--echo # +--echo # Flush status +--echo # +connection master; +echo [ Semi-sync master status variables before FLUSH STATUS ]; +SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; +SHOW STATUS LIKE 'Rpl_semi_sync_master_yes_tx'; +# Do not write the FLUSH STATUS to binlog, to make sure we'll get a +# clean status after this. +FLUSH NO_WRITE_TO_BINLOG STATUS; +echo [ Semi-sync master status variables after FLUSH STATUS ]; +SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; +SHOW STATUS LIKE 'Rpl_semi_sync_master_yes_tx'; + +connection master; +echo [ on master ]; + +source include/show_master_logs.inc; +show variables like 'rpl_semi_sync_master_enabled'; + +echo [ disable semi-sync on the fly ]; +set global rpl_semi_sync_master_enabled=0; +show variables like 'rpl_semi_sync_master_enabled'; +show status like 'Rpl_semi_sync_master_status'; + +echo [ enable semi-sync on the fly ]; +set global rpl_semi_sync_master_enabled=1; +show variables like 'rpl_semi_sync_master_enabled'; +show status like 'Rpl_semi_sync_master_status'; + +--echo # +--echo # Test RESET MASTER/SLAVE +--echo # + +connection slave; +echo [ on slave ]; + +source include/start_slave.inc; + +connection master; +echo [ on master ]; + +replace_result $engine_type ENGINE_TYPE; +eval create table t1 (a int) engine = $engine_type; +drop table t1; + +##show status like 'Rpl_semi_sync_master_status'; + +sync_slave_with_master; +--replace_column 2 # +show status like 'Rpl_relay%'; + +echo [ test reset master ]; +connection master; +echo [ on master]; + +reset master; + +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +connection slave; +echo [ on slave ]; + +source include/stop_slave.inc; +reset slave; + +# Kill the dump thread on master for previous slave connection and +# wait for it to exit +connection master; +let $_tid= `select id from information_schema.processlist where command = 'Binlog Dump' limit 1`; +if ($_tid) +{ + --replace_result $_tid _tid + eval kill query $_tid; + + # After dump thread exit, Rpl_semi_sync_master_clients will be 0 + let $status_var= Rpl_semi_sync_master_clients; + let $status_var_value= 0; + source include/wait_for_status_var.inc; +} + +connection slave; +source include/start_slave.inc; + +connection master; +echo [ on master ]; + +# Wait for dump thread to start, Rpl_semi_sync_master_clients will be +# 1 after dump thread started. +let $status_var= Rpl_semi_sync_master_clients; +let $status_var_value= 1; +source include/wait_for_status_var.inc; + +replace_result $engine_type ENGINE_TYPE; +eval create table t1 (a int) engine = $engine_type; +insert into t1 values (1); +insert into t1 values (2), (3); + +sync_slave_with_master; +echo [ on slave ]; + +select * from t1; + +connection master; +echo [ on master ]; + +echo [ master semi-sync status should be ON ]; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +--echo # +--echo # Start semi-sync replication without SUPER privilege +--echo # +connection slave; +source include/stop_slave.inc; +reset slave; +connection master; +echo [ on master ]; +reset master; + +# Kill the dump thread on master for previous slave connection and wait for it to exit +let $_tid= `select id from information_schema.processlist where command = 'Binlog Dump' limit 1`; +if ($_tid) +{ + --replace_result $_tid _tid + eval kill query $_tid; + + # After dump thread exit, Rpl_semi_sync_master_clients will be 0 + let $status_var= Rpl_semi_sync_master_clients; + let $status_var_value= 0; + source include/wait_for_status_var.inc; +} + +# Do not binlog the following statement because it will generate +# different events for ROW and STATEMENT format +set sql_log_bin=0; +grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; +flush privileges; +set sql_log_bin=1; +connection slave; +echo [ on slave ]; +grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; +flush privileges; +change master to master_user='rpl',master_password='rpl_password'; +source include/start_slave.inc; +show status like 'Rpl_semi_sync_slave_status'; +connection master; +echo [ on master ]; + +# Wait for the semi-sync binlog dump thread to start +let $status_var= Rpl_semi_sync_master_clients; +let $status_var_value= 1; +source include/wait_for_status_var.inc; +echo [ master semi-sync should be ON ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; +insert into t1 values (4); +insert into t1 values (5); +echo [ master semi-sync should be ON ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_no_tx'; +show status like 'Rpl_semi_sync_master_yes_tx'; + +--echo # +--echo # Test semi-sync slave connect to non-semi-sync master +--echo # + +# Disable semi-sync on master +connection slave; +echo [ on slave ]; +source include/stop_slave.inc; +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; + +connection master; +echo [ on master ]; + +# Kill the dump thread on master for previous slave connection and wait for it to exit +let $_tid= `select id from information_schema.processlist where command = 'Binlog Dump' limit 1`; +if ($_tid) +{ + --replace_result $_tid _tid + eval kill query $_tid; + + # After dump thread exit, Rpl_semi_sync_master_clients will be 0 + let $status_var= Rpl_semi_sync_master_clients; + let $status_var_value= 0; + source include/wait_for_status_var.inc; +} + +echo [ Semi-sync status on master should be ON ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +set global rpl_semi_sync_master_enabled= 0; + +connection slave; +echo [ on slave ]; +SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; +source include/start_slave.inc; +connection master; +echo [ on master ]; +insert into t1 values (8); +let $status_var= Rpl_semi_sync_master_clients; +let $status_var_value= 1; +source include/wait_for_status_var.inc; +echo [ master semi-sync clients should be 1, status should be OFF ]; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; +sync_slave_with_master; +echo [ on slave ]; +show status like 'Rpl_semi_sync_slave_status'; + +# Uninstall semi-sync plugin on master +connection slave; +source include/stop_slave.inc; +connection master; +echo [ on master ]; +set global rpl_semi_sync_master_enabled= 0; + +connection slave; +echo [ on slave ]; +SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; +source include/start_slave.inc; + +connection master; +echo [ on master ]; +insert into t1 values (10); +sync_slave_with_master; + +--echo # +--echo # Test non-semi-sync slave connect to semi-sync master +--echo # + +connection master; +set global rpl_semi_sync_master_timeout= 5000; # 5s +set global rpl_semi_sync_master_enabled= 1; + +connection slave; +echo [ on slave ]; +source include/stop_slave.inc; +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; + +echo [ uninstall semi-sync slave plugin ]; +set global rpl_semi_sync_slave_enabled= 0; + +echo [ reinstall semi-sync slave plugin and disable semi-sync ]; +SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; +source include/start_slave.inc; +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; + +--echo # +--echo # Clean up +--echo # + +connection slave; +source include/stop_slave.inc; +set global rpl_semi_sync_slave_enabled= 0; + +connection master; +set global rpl_semi_sync_master_enabled= 0; + +connection slave; +change master to master_user='root',master_password=''; +source include/start_slave.inc; + +connection master; +drop table t1; +sync_slave_with_master; + +connection master; +drop user rpl@127.0.0.1; +flush privileges; +set global rpl_semi_sync_master_timeout= default; +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_skip_replication.inc b/mysql-test/extra/rpl_tests/rpl_skip_replication.inc new file mode 100644 index 0000000000000..14e3339ff5ed8 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_skip_replication.inc @@ -0,0 +1,402 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it. +# +# Usage: +# +# --let $use_remote_mysqlbinlog= 1 # optional +# --source extra/rpl_tests/rpl_skip_replication.inc +# +# The script uses MYSQLBINLOG to verify certain results. +# By default, it uses binary logs directly. If it is undesirable, +# this behavior can be overridden by setting $use_remote_binlog +# as shown above. +# The value will be unset after every execution of the script, +# so if it is needed, it should be set explicitly before each call. +# + +--source include/master-slave.inc +--source include/have_innodb.inc + +connection slave; +# Test that SUPER is required to change @@replicate_events_marked_for_skip. +CREATE USER 'nonsuperuser'@'127.0.0.1'; +GRANT ALTER,CREATE,DELETE,DROP,EVENT,INSERT,PROCESS,REPLICATION SLAVE, + SELECT,UPDATE ON *.* TO 'nonsuperuser'@'127.0.0.1'; +connect(nonpriv, 127.0.0.1, nonsuperuser,, test, $SLAVE_MYPORT,); +connection nonpriv; +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +disconnect nonpriv; +connection slave; +DROP USER'nonsuperuser'@'127.0.0.1'; + +SELECT @@global.replicate_events_marked_for_skip; +--error ER_SLAVE_MUST_STOP +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +SELECT @@global.replicate_events_marked_for_skip; +STOP SLAVE; +--error ER_GLOBAL_VARIABLE +SET SESSION replicate_events_marked_for_skip=FILTER_ON_MASTER; +SELECT @@global.replicate_events_marked_for_skip; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +SELECT @@global.replicate_events_marked_for_skip; +START SLAVE; + +connection master; +SELECT @@skip_replication; +--error ER_LOCAL_VARIABLE +SET GLOBAL skip_replication=1; +SELECT @@skip_replication; + +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +CREATE TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=innodb; +INSERT INTO t1(a) VALUES (1); +INSERT INTO t2(a) VALUES (1); + + +# Test that master-side filtering works. +SET skip_replication=1; + +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +INSERT INTO t1(a) VALUES (2); +INSERT INTO t2(a) VALUES (2); + +# Inject a rotate event in the binlog stream sent to slave (otherwise we will +# fail sync_slave_with_master as the last event on the master is not present +# on the slave). +FLUSH NO_WRITE_TO_BINLOG LOGS; + +sync_slave_with_master; +connection slave; +SHOW TABLES; +SELECT * FROM t1; +SELECT * FROM t2; + +connection master; +DROP TABLE t3; + +FLUSH NO_WRITE_TO_BINLOG LOGS; +sync_slave_with_master; + + +# Test that slave-side filtering works. +connection slave; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +START SLAVE; + +connection master; +SET skip_replication=1; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +INSERT INTO t1(a) VALUES (3); +INSERT INTO t2(a) VALUES (3); + +# Inject a rotate event in the binlog stream sent to slave (otherwise we will +# fail sync_slave_with_master as the last event on the master is not present +# on the slave). +FLUSH NO_WRITE_TO_BINLOG LOGS; + +sync_slave_with_master; +connection slave; +SHOW TABLES; +SELECT * FROM t1; +SELECT * FROM t2; + +connection master; +DROP TABLE t3; + +FLUSH NO_WRITE_TO_BINLOG LOGS; +sync_slave_with_master; +connection slave; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; + + +# Test that events with @@skip_replication=1 are not filtered when filtering is +# not set on slave. +connection master; +SET skip_replication=1; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +INSERT INTO t3(a) VALUES(2); +sync_slave_with_master; +connection slave; +SELECT * FROM t3; +connection master; +DROP TABLE t3; + +# +# Test that the slave will preserve the @@skip_replication flag in its +# own binlog. +# + +TRUNCATE t1; +sync_slave_with_master; +connection slave; +RESET MASTER; + +connection master; +SET skip_replication=0; +INSERT INTO t1 VALUES (1,0); +SET skip_replication=1; +INSERT INTO t1 VALUES (2,0); +SET skip_replication=0; +INSERT INTO t1 VALUES (3,0); + +sync_slave_with_master; +connection slave; +# Since slave has @@replicate_events_marked_for_skip=REPLICATE, it should have +# applied all events. +SELECT * FROM t1 ORDER by a; + +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +let $SLAVE_DATADIR= `select @@datadir`; + +connection master; +TRUNCATE t1; + +# Now apply the slave binlog to the master, to check that both the slave +# and mysqlbinlog will preserve the @@skip_replication flag. + +--let $mysqlbinlog_args= $SLAVE_DATADIR/slave-bin.000001 +if ($use_remote_mysqlbinlog) +{ + --let $mysqlbinlog_args= --read-from-remote-server --protocol=tcp --host=127.0.0.1 --port=$SLAVE_MYPORT -uroot slave-bin.000001 + --let $use_remote_mysqlbinlog= 0 +} +--exec $MYSQL_BINLOG $mysqlbinlog_args > $MYSQLTEST_VARDIR/tmp/rpl_skip_replication.binlog +--exec $MYSQL test < $MYSQLTEST_VARDIR/tmp/rpl_skip_replication.binlog + +# The master should have all three events. +SELECT * FROM t1 ORDER by a; + +# The slave should be missing event 2, which is marked with the +# @@skip_replication flag. + +connection slave; +START SLAVE; + +connection master; +sync_slave_with_master; + +connection slave; +SELECT * FROM t1 ORDER by a; + +# +# Test that @@sql_slave_skip_counter does not count skipped @@skip_replication +# events. +# + +connection master; +TRUNCATE t1; + +sync_slave_with_master; +connection slave; +STOP SLAVE; +# We will skip two INSERTs (in addition to any skipped due to +# @@skip_replication). Since from 5.5 every statement is wrapped in +# BEGIN ... END, we need to skip 6 events for this. +SET GLOBAL sql_slave_skip_counter=6; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +START SLAVE; + +connection master; +# Need to fix @@binlog_format to get consistent event count. +SET @old_binlog_format= @@binlog_format; +SET binlog_format= statement; +SET skip_replication=0; +INSERT INTO t1 VALUES (1,5); +SET skip_replication=1; +INSERT INTO t1 VALUES (2,5); +SET skip_replication=0; +INSERT INTO t1 VALUES (3,5); +INSERT INTO t1 VALUES (4,5); +SET binlog_format= @old_binlog_format; + +sync_slave_with_master; +connection slave; + +# The slave should have skipped the first three inserts (number 1 and 3 due +# to @@sql_slave_skip_counter=2, number 2 due to +# @@replicate_events_marked_for_skip=FILTER_ON_SLAVE). So only number 4 +# should be left. +SELECT * FROM t1; + + +# +# Check that BINLOG statement preserves the @@skip_replication flag. +# +connection slave; +# Need row @@binlog_format for BINLOG statements containing row events. +--source include/stop_slave.inc +SET @old_slave_binlog_format= @@global.binlog_format; +SET GLOBAL binlog_format= row; +--source include/start_slave.inc + +connection master; +TRUNCATE t1; + +SET @old_binlog_format= @@binlog_format; +SET binlog_format= row; +# Format description log event. +BINLOG 'wlZOTw8BAAAA8QAAAPUAAAAAAAQANS41LjIxLU1hcmlhREItZGVidWctbG9nAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAEzgNAAgAEgAEBAQEEgAA2QAEGggAAAAICAgCAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAA371saA=='; +# INSERT INTO t1 VALUES (1,8) # with @@skip_replication=1 +BINLOG 'wlZOTxMBAAAAKgAAAGMBAAAAgCkAAAAAAAEABHRlc3QAAnQxAAIDAwAC +wlZOTxcBAAAAJgAAAIkBAAAAgCkAAAAAAAEAAv/8AQAAAAgAAAA='; +# INSERT INTO t1 VALUES (2,8) # with @@skip_replication=0 +BINLOG 'wlZOTxMBAAAAKgAAADwCAAAAACkAAAAAAAEABHRlc3QAAnQxAAIDAwAC +wlZOTxcBAAAAJgAAAGICAAAAACkAAAAAAAEAAv/8AgAAAAgAAAA='; +SET binlog_format= @old_binlog_format; + +SELECT * FROM t1 ORDER BY a; +sync_slave_with_master; +connection slave; +# Slave should have only the second insert, the first should be ignored due to +# the @@skip_replication flag. +SELECT * FROM t1 ORDER by a; + +--source include/stop_slave.inc +SET GLOBAL binlog_format= @old_slave_binlog_format; +--source include/start_slave.inc + + +# Test that it is not possible to change @@skip_replication inside a +# transaction or statement, thereby replicating only parts of statements +# or transactions. +connection master; +SET skip_replication=0; + +BEGIN; +--error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET skip_replication=0; +--error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET skip_replication=1; +ROLLBACK; +SET skip_replication=1; +BEGIN; +--error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET skip_replication=0; +--error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET skip_replication=1; +COMMIT; +SET autocommit=0; +INSERT INTO t2(a) VALUES(100); +--error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET skip_replication=1; +ROLLBACK; +SET autocommit=1; + +SET skip_replication=1; +--delimiter | +CREATE FUNCTION foo (x INT) RETURNS INT BEGIN SET SESSION skip_replication=x; RETURN x; END| +CREATE PROCEDURE bar(x INT) BEGIN SET SESSION skip_replication=x; END| +CREATE FUNCTION baz (x INT) RETURNS INT BEGIN CALL bar(x); RETURN x; END| +--delimiter ; +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +SELECT foo(0); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +SELECT baz(0); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET @a= foo(1); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +SET @a= baz(1); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +UPDATE t2 SET b=foo(0); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +UPDATE t2 SET b=baz(0); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +INSERT INTO t1 VALUES (101, foo(1)); +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION +INSERT INTO t1 VALUES (101, baz(0)); +SELECT @@skip_replication; +CALL bar(0); +SELECT @@skip_replication; +CALL bar(1); +SELECT @@skip_replication; +DROP FUNCTION foo; +DROP PROCEDURE bar; +DROP FUNCTION baz; + + +# Test that master-side filtering happens on the master side, and that +# slave-side filtering happens on the slave. + +# First test that events do not reach the slave when master-side filtering +# is configured. Do this by replicating first with only the IO thread running +# and master-side filtering; then change to no filtering and start the SQL +# thread. This should still skip the events, as master-side filtering +# means the events never reached the slave. +connection master; +SET skip_replication= 0; +TRUNCATE t1; +sync_slave_with_master; +connection slave; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +START SLAVE IO_THREAD; +connection master; +SET skip_replication= 1; +INSERT INTO t1(a) VALUES (1); +SET skip_replication= 0; +INSERT INTO t1(a) VALUES (2); +--source include/save_master_pos.inc +connection slave; +--source include/sync_io_with_master.inc +STOP SLAVE IO_THREAD; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; +connection master; +sync_slave_with_master; +connection slave; +# Now only the second insert of (2) should be visible, as the first was +# filtered on the master, so even though the SQL thread ran without skipping +# events, it will never see the event in the first place. +SELECT * FROM t1; + +# Now tests that when slave-side filtering is configured, events _do_ reach +# the slave. +connection master; +SET skip_replication= 0; +TRUNCATE t1; +sync_slave_with_master; +connection slave; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +START SLAVE IO_THREAD; +connection master; +SET skip_replication= 1; +INSERT INTO t1(a) VALUES (1); +SET skip_replication= 0; +INSERT INTO t1(a) VALUES (2); +--source include/save_master_pos.inc +connection slave; +--source include/sync_io_with_master.inc +STOP SLAVE IO_THREAD; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; +connection master; +sync_slave_with_master; +connection slave; +# Now both inserts should be visible. Since filtering was configured to be +# slave-side, the event is in the relay log, and when the SQL thread ran we +# had disabled filtering again. +SELECT * FROM t1 ORDER BY a; + + +# Clean up. +connection master; +SET skip_replication=0; +DROP TABLE t1,t2; +connection slave; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_special_charset.inc b/mysql-test/extra/rpl_tests/rpl_special_charset.inc new file mode 100644 index 0000000000000..51119b319d712 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_special_charset.inc @@ -0,0 +1,32 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +################################################################################ +# Bug#19855907 IO THREAD AUTHENTICATION ISSUE WITH SOME CHARACTER SETS +# Problem: IO thread fails to connect to master if servers are configured with +# special character sets like utf16, utf32, ucs2. +# +# Analysis: MySQL server does not support few special character sets like +# utf16,utf32 and ucs2 as "client's character set"(eg: utf16,utf32, ucs2). +# When IO thread is trying to connect to Master, it sets server's character +# set as client's character set. When Slave server is started with these +# special character sets, IO thread (a connection to Master) fails because +# of the above said reason. +# +# Fix: If server's character set is not supported as client's character set, +# then set default's client character set(latin1) as client's character set. +############################################################################### +--source include/master-slave.inc +call mtr.add_suppression("Cannot use utf16 as character_set_client"); +CREATE TABLE t1(i VARCHAR(20)); +INSERT INTO t1 VALUES (0xFFFF); +--sync_slave_with_master +--let diff_tables=master:t1, slave:t1 +--source include/diff_tables.inc +# Cleanup +--connection master +DROP TABLE t1; +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_sporadic_master.inc b/mysql-test/extra/rpl_tests/rpl_sporadic_master.inc new file mode 100644 index 0000000000000..ad4c44cbf7409 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_sporadic_master.inc @@ -0,0 +1,32 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# test to see if replication can continue when master sporadically fails on +# COM_BINLOG_DUMP and additionally limits the number of events per dump + +source include/master-slave.inc; + +create table t2(n int); +create table t1(n int not null auto_increment primary key); +insert into t1 values (NULL),(NULL); +truncate table t1; +# We have to use 4 in the following to make this test work with all table types +insert into t1 values (4),(NULL); +sync_slave_with_master; +--source include/stop_slave.inc +--source include/start_slave.inc +connection master; +insert into t1 values (NULL),(NULL); +flush logs; +truncate table t1; +insert into t1 values (10),(NULL),(NULL),(NULL),(NULL),(NULL); +sync_slave_with_master; +select * from t1 ORDER BY n; +connection master; +drop table t1,t2; +sync_slave_with_master; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_ssl.inc b/mysql-test/extra/rpl_tests/rpl_ssl.inc new file mode 100644 index 0000000000000..ad75b54ba6c92 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_ssl.inc @@ -0,0 +1,117 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +source include/have_ssl_communication.inc; +source include/master-slave.inc; + +# create a user for replication that requires ssl encryption +connection master; +create user replssl@localhost; +grant replication slave on *.* to replssl@localhost require ssl; +create table t1 (t int auto_increment, KEY(t)); + +sync_slave_with_master; + +# Set slave to use SSL for connection to master +stop slave; +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR +eval change master to + master_user='replssl', + master_password='', + master_ssl=1, + master_ssl_ca ='$MYSQL_TEST_DIR/std_data/cacert.pem', + master_ssl_cert='$MYSQL_TEST_DIR/std_data/client-cert.pem', + master_ssl_key='$MYSQL_TEST_DIR/std_data/client-key.pem'; +start slave; + +# Switch to master and insert one record, then sync it to slave +connection master; +insert into t1 values(1); +sync_slave_with_master; + +# The record should now be on slave +select * from t1; + +# The slave is synced and waiting/reading from master +# SHOW SLAVE STATUS will show "Waiting for master to send event" +let $status_items= Master_SSL_Allowed, Master_SSL_CA_Path, Master_SSL_CA_File, Master_SSL_Cert, Master_SSL_Key; +source include/show_slave_status.inc; +source include/check_slave_is_running.inc; + +# Stop the slave, as reported in bug#21871 it would hang +STOP SLAVE; + +select * from t1; + +# Do the same thing a number of times +disable_query_log; +disable_result_log; +# 2007-11-27 mats Bug #32756 Starting and stopping the slave in a loop can lose rows +# After discussions with Engineering, I'm disabling this part of the test to avoid it causing +# red trees. +disable_parsing; +let $i= 100; +while ($i) +{ + start slave; + connection master; + insert into t1 values (NULL); + select * from t1; # Some variance + connection slave; + select * from t1; # Some variance + stop slave; + dec $i; +} +enable_parsing; +START SLAVE; +enable_query_log; +enable_result_log; +connection master; +# INSERT one more record to make sure +# the sync has something to do +insert into t1 values (NULL); +let $master_count= `select count(*) from t1`; + +sync_slave_with_master; +--source include/wait_for_slave_to_start.inc +source include/show_slave_status.inc; +source include/check_slave_is_running.inc; + +let $slave_count= `select count(*) from t1`; + +if ($slave_count != $master_count) +{ + echo master and slave differed in number of rows; + echo master: $master_count; + echo slave: $slave_count; + + connection master; + echo === master ===; + select count(*) t1; + select * from t1; + connection slave; + echo === slave ===; + select count(*) t1; + select * from t1; + query_vertical show slave status; +} + +connection master; +drop user replssl@localhost; +drop table t1; +sync_slave_with_master; + +--source include/stop_slave.inc +CHANGE MASTER TO + master_user = 'root', + master_ssl = 0, + master_ssl_ca = '', + master_ssl_cert = '', + master_ssl_key = ''; + +--echo End of 5.0 tests +--let $rpl_only_running_threads= 1 +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_stm_relay_ign_space.inc b/mysql-test/extra/rpl_tests/rpl_stm_relay_ign_space.inc new file mode 100644 index 0000000000000..82c4b1881bf05 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_stm_relay_ign_space.inc @@ -0,0 +1,107 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# +# BUG#12400313 / BUG#64503 test case +# +# +# Description +# ----------- +# +# This test case starts the slave server with: +# --relay-log-space-limit=8192 --relay-log-purge --max-relay-log-size=4096 +# +# Then it issues some queries that will cause the slave to reach +# relay-log-space-limit. We lock the table so that the SQL thread is +# not able to purge the log and then we issue some more statements. +# +# The purpose is to show that the IO thread will honor the limits +# while the SQL thread is not able to purge the relay logs, which did +# not happen before this patch. In addition we assert that while +# ignoring the limit (SQL thread needs to rotate before purging), the +# IO thread does not do it in an uncontrolled manner. + +--source include/have_binlog_format_statement.inc +--source include/master-slave.inc +--source include/have_innodb.inc + +--disable_query_log +CREATE TABLE t1 (c1 TEXT) engine=InnoDB; + +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); + +--sync_slave_with_master + +# wait for the SQL thread to sleep +--let $show_statement= SHOW PROCESSLIST +--let $field= State +--let $condition= = 'Slave has read all relay log; waiting for the slave I/O thread to update it' +--source include/wait_show_condition.inc + +# now the io thread has set rli->ignore_space_limit +# lets lock the table so that once the SQL thread awakes +# it blocks there and does not set rli->ignore_space_limit +# back to zero +LOCK TABLE t1 WRITE; + +# now issue more statements that will overflow the +# rli->log_space_limit (in this case ~10K) +--connection master + +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); +INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); + +--connection slave + +# ASSERT that the IO thread waits for the SQL thread to release some +# space before continuing +--let $show_statement= SHOW PROCESSLIST +--let $field= State +--let $condition= LIKE 'Waiting for %' +# before the patch (IO would have transfered everything) +#--let $condition= = 'Waiting for master to send event' +# after the patch (now it waits for space to be freed) +#--let $condition= = 'Waiting for the slave SQL thread to free enough relay log space' +--source include/wait_show_condition.inc + +# without the patch we can uncomment the following two lines and +# watch the IO thread synchronize with the master, thus writing +# relay logs way over the space limit +#--connection master +#--source include/sync_slave_io_with_master.inc + +## ASSERT that the IO thread has honored the limit+few bytes required to be able to purge +--let $relay_log_space_while_sql_is_executing = query_get_value(SHOW SLAVE STATUS, Relay_Log_Space, 1) +--let $relay_log_space_limit = query_get_value(SHOW VARIABLES LIKE "relay_log_space_limit", Value, 1) +--let $assert_text= Assert that relay log space is close to the limit +--let $assert_cond= $relay_log_space_while_sql_is_executing <= $relay_log_space_limit * 1.15 +--source include/assert.inc + +# unlock the table and let SQL thread continue applying events +UNLOCK TABLES; + +--connection master +--sync_slave_with_master +--let $diff_tables=master:test.t1,slave:test.t1 +--source include/diff_tables.inc + +--connection master +DROP TABLE t1; +--enable_query_log +--sync_slave_with_master + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_switch_stm_row_mixed.inc b/mysql-test/extra/rpl_tests/rpl_switch_stm_row_mixed.inc new file mode 100644 index 0000000000000..e74fd6828c6c1 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_switch_stm_row_mixed.inc @@ -0,0 +1,631 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +# +# rpl_switch_stm_row_mixed tests covers +# +# - Master is switching explicitly between STATEMENT, ROW, and MIXED +# binlog format showing when it is possible and when not. +# - Master switching from MIXED to RBR implicitly listing all use +# cases, e.g a query invokes UUID(), thereafter to serve as the +# definition of MIXED binlog format +# - correctness of execution + + +-- source include/have_binlog_format_mixed_or_row.inc +-- source include/master-slave.inc + +# Since this test generates row-based events in the binary log, the +# slave SQL thread cannot be in STATEMENT mode to execute this test, +# so we only execute it for MIXED and ROW as default value of +# BINLOG_FORMAT. + +connection slave; + +connection master; +--disable_warnings +drop database if exists mysqltest1; +create database mysqltest1; +--enable_warnings +use mysqltest1; + +# Save binlog format +set @my_binlog_format= @@global.binlog_format; + +# play with switching +set session binlog_format=mixed; +show session variables like "binlog_format%"; +set session binlog_format=statement; +show session variables like "binlog_format%"; +set session binlog_format=row; +show session variables like "binlog_format%"; + +set global binlog_format=DEFAULT; +show global variables like "binlog_format%"; +set global binlog_format=MIXED; +show global variables like "binlog_format%"; +set global binlog_format=STATEMENT; +show global variables like "binlog_format%"; +set global binlog_format=ROW; +show global variables like "binlog_format%"; +show session variables like "binlog_format%"; +select @@global.binlog_format, @@session.binlog_format; + +CREATE TABLE t1 (a varchar(100)); + +prepare stmt1 from 'insert into t1 select concat(UUID(),?)'; +set @string="emergency_1_"; +insert into t1 values("work_2_"); +execute stmt1 using @string; +deallocate prepare stmt1; + +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values(concat(UUID(),"work_3_")); +execute stmt1 using @string; +deallocate prepare stmt1; + +insert into t1 values(concat("for_4_",UUID())); +insert into t1 select "yesterday_5_"; + +# verify that temp tables prevent a switch to SBR +create temporary table tmp(a char(100)); +insert into tmp values("see_6_"); +--error ER_TEMP_TABLE_PREVENTS_SWITCH_OUT_OF_RBR +set binlog_format=statement; +insert into t1 select * from tmp; +drop temporary table tmp; + +# Now we go to SBR +set binlog_format=statement; +show global variables like "binlog_format%"; +show session variables like "binlog_format%"; +select @@global.binlog_format, @@session.binlog_format; +set global binlog_format=statement; +show global variables like "binlog_format%"; +show session variables like "binlog_format%"; +select @@global.binlog_format, @@session.binlog_format; + +prepare stmt1 from 'insert into t1 select ?'; +set @string="emergency_7_"; +insert into t1 values("work_8_"); +execute stmt1 using @string; +deallocate prepare stmt1; + +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values("work_9_"); +execute stmt1 using @string; +deallocate prepare stmt1; + +insert into t1 values("for_10_"); +insert into t1 select "yesterday_11_"; + +# test statement (is not default after wl#3368) +set binlog_format=statement; +select @@global.binlog_format, @@session.binlog_format; +set global binlog_format=statement; +select @@global.binlog_format, @@session.binlog_format; + +prepare stmt1 from 'insert into t1 select ?'; +set @string="emergency_12_"; +insert into t1 values("work_13_"); +execute stmt1 using @string; +deallocate prepare stmt1; + +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values("work_14_"); +execute stmt1 using @string; +deallocate prepare stmt1; + +insert into t1 values("for_15_"); +insert into t1 select "yesterday_16_"; + +# and now the mixed mode + +set global binlog_format=mixed; +select @@global.binlog_format, @@session.binlog_format; +set binlog_format=default; +select @@global.binlog_format, @@session.binlog_format; + +prepare stmt1 from 'insert into t1 select concat(UUID(),?)'; +set @string="emergency_17_"; +insert into t1 values("work_18_"); +execute stmt1 using @string; +deallocate prepare stmt1; + +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values(concat(UUID(),"work_19_")); +execute stmt1 using @string; +deallocate prepare stmt1; + +insert into t1 values(concat("for_20_",UUID())); +insert into t1 select "yesterday_21_"; + +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values(concat(UUID(),"work_22_")); +execute stmt1 using @string; +deallocate prepare stmt1; + +insert into t1 values(concat("for_23_",UUID())); +insert into t1 select "yesterday_24_"; + +# Test of CREATE TABLE SELECT + +create table t2 ENGINE=MyISAM select rpad(UUID(),100,' '); +create table t3 select 1 union select UUID(); +--disable_warnings +create table t4 select * from t1 where 3 in (select 1 union select 2 union select UUID() union select 3); +--enable_warnings +create table t5 select * from t1 where 3 in (select 1 union select 2 union select curdate() union select 3); +# what if UUID() is first: +--disable_warnings +insert into t5 select UUID() from t1 where 3 in (select 1 union select 2 union select 3 union select * from t4); +--enable_warnings + +# inside a stored procedure + +delimiter |; +create procedure foo() +begin +insert into t1 values("work_25_"); +insert into t1 values(concat("for_26_",UUID())); +insert into t1 select "yesterday_27_"; +end| +create procedure foo2() +begin +insert into t1 values(concat("emergency_28_",UUID())); +insert into t1 values("work_29_"); +insert into t1 values(concat("for_30_",UUID())); +set session binlog_format=row; # accepted for stored procs +insert into t1 values("more work_31_"); +set session binlog_format=mixed; +end| +create function foo3() returns bigint unsigned +begin + set session binlog_format=row; # rejected for stored funcs + insert into t1 values("alarm"); + return 100; +end| +create procedure foo4(x varchar(100)) +begin +insert into t1 values(concat("work_250_",x)); +insert into t1 select "yesterday_270_"; +end| +delimiter ;| +call foo(); +call foo2(); +call foo4("hello"); +call foo4(UUID()); +call foo4("world"); + +# test that can't SET in a stored function +--error ER_STORED_FUNCTION_PREVENTS_SWITCH_BINLOG_FORMAT +select foo3(); +select * from t1 where a="alarm"; + +# Tests of stored functions/triggers/views for BUG#20930 "Mixed +# binlogging mode does not work with stored functions, triggers, +# views" + +# Function which calls procedure +drop function foo3; +delimiter |; +create function foo3() returns bigint unsigned +begin + insert into t1 values("foo3_32_"); + call foo(); + return 100; +end| +delimiter ;| +insert into t2 select foo3(); + +prepare stmt1 from 'insert into t2 select foo3()'; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; + +# Test if stored function calls stored function which calls procedure +# which requires row-based. + +delimiter |; +create function foo4() returns bigint unsigned +begin + insert into t2 select foo3(); + return 100; +end| +delimiter ;| +select foo4(); + +prepare stmt1 from 'select foo4()'; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; + +# A simple stored function +delimiter |; +create function foo5() returns bigint unsigned +begin + insert into t2 select UUID(); + return 100; +end| +delimiter ;| +select foo5(); + +prepare stmt1 from 'select foo5()'; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; + +# A simple stored function where UUID() is in the argument +delimiter |; +create function foo6(x varchar(100)) returns bigint unsigned +begin + insert into t2 select x; + return 100; +end| +delimiter ;| +select foo6("foo6_1_"); +select foo6(concat("foo6_2_",UUID())); + +prepare stmt1 from 'select foo6(concat("foo6_3_",UUID()))'; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; + + +# Test of views using UUID() + +create view v1 as select uuid(); +create table t11 (data varchar(255)); +insert into t11 select * from v1; +# Test of querying INFORMATION_SCHEMA which parses the view's body, +# to verify that it binlogs statement-based (is not polluted by +# the parsing of the view's body). +insert into t11 select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='mysqltest1' and TABLE_NAME IN ('v1','t11'); +prepare stmt1 from "insert into t11 select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='mysqltest1' and TABLE_NAME IN ('v1','t11')"; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; + +# Test of triggers with UUID() +delimiter |; +create trigger t11_bi before insert on t11 for each row +begin + set NEW.data = concat(NEW.data,UUID()); +end| +delimiter ;| +insert into t11 values("try_560_"); + +# Test that INSERT DELAYED works in mixed mode (BUG#20649) +insert delayed into t2 values("delay_1_"); +insert delayed into t2 values(concat("delay_2_",UUID())); +insert delayed into t2 values("delay_6_"); + +# Test for BUG#20633 (INSERT DELAYED RAND()/user_variable does not +# replicate fine in statement-based ; we test that in mixed mode it +# works). +insert delayed into t2 values(rand()); +set @a=2.345; +insert delayed into t2 values(@a); + +# With INSERT DELAYED, rows are written to the binlog after they are +# written to the table. Therefore, it is not enough to wait until the +# rows make it to t2 on the master (the rows may not be in the binlog +# at that time, and may still not be in the binlog when +# sync_slave_with_master is later called). Instead, we wait until the +# rows make it to t2 on the slave. We first call +# sync_slave_with_master, so that we are sure that t2 has been created +# on the slave. +sync_slave_with_master; +let $wait_condition= SELECT COUNT(*) = 19 FROM mysqltest1.t2; +--source include/wait_condition.inc +connection master; + +# If you want to do manual testing of the mixed mode regarding UDFs (not +# testable automatically as quite platform- and compiler-dependent), +# you just need to set the variable below to 1, and to +# "make udf_example.so" in sql/, and to copy sql/udf_example.so to +# MYSQL_TEST_DIR/lib/mysql. +let $you_want_to_test_UDF=0; +if ($you_want_to_test_UDF) +{ + CREATE FUNCTION metaphon RETURNS STRING SONAME 'udf_example.so'; + prepare stmt1 from 'insert into t1 select metaphon(?)'; + set @string="emergency_133_"; + insert into t1 values("work_134_"); + execute stmt1 using @string; + deallocate prepare stmt1; + prepare stmt1 from 'insert into t1 select ?'; + insert into t1 values(metaphon("work_135_")); + execute stmt1 using @string; + deallocate prepare stmt1; + insert into t1 values(metaphon("for_136_")); + insert into t1 select "yesterday_137_"; + create table t6 select metaphon("for_138_"); + create table t7 select 1 union select metaphon("for_139_"); + create table t8 select * from t1 where 3 in (select 1 union select 2 union select metaphon("for_140_") union select 3); + create table t9 select * from t1 where 3 in (select 1 union select 2 union select curdate() union select 3); +} + +create table t20 select * from t1; # save for comparing later +create table t21 select * from t2; +create table t22 select * from t3; +drop table t1,t2,t3; + +# This tests the fix to +# BUG#19630 stored function inserting into two auto_increment breaks statement-based binlog +# We verify that under the mixed binlog mode, a stored function +# modifying at least two tables having an auto_increment column, +# is binlogged row-based. Indeed in statement-based binlogging, +# only the auto_increment value generated for the first table +# is recorded in the binlog, the value generated for the 2nd table +# lacking. + +create table t1 (a int primary key auto_increment, b varchar(100)); +create table t2 (a int primary key auto_increment, b varchar(100)); +create table t3 (b varchar(100)); +delimiter |; +create function f (x varchar(100)) returns int deterministic +begin + insert into t1 values(null,x); + insert into t2 values(null,x); + return 1; +end| +delimiter ;| +select f("try_41_"); +# Two operations which compensate each other except that their net +# effect is that they advance the auto_increment counter of t2 on slave: +sync_slave_with_master; +use mysqltest1; +insert into t2 values(2,null),(3,null),(4,null); +delete from t2 where a>=2; + +connection master; +# this is the call which didn't replicate well +select f("try_42_"); +sync_slave_with_master; + +# now use prepared statement and test again, just to see that the RBB +# mode isn't set at PREPARE but at EXECUTE. + +insert into t2 values(3,null),(4,null); +delete from t2 where a>=3; + +connection master; +prepare stmt1 from 'select f(?)'; +set @string="try_43_"; +insert into t1 values(null,"try_44_"); # should be SBB +execute stmt1 using @string; # should be RBB +deallocate prepare stmt1; +sync_slave_with_master; + +# verify that if only one table has auto_inc, it does not trigger RBB +# (we'll check in binlog further below) + +connection master; +create table t12 select * from t1; # save for comparing later +drop table t1; +create table t1 (a int, b varchar(100), key(a)); +select f("try_45_"); + +# restore table's key +create table t13 select * from t1; +drop table t1; +create table t1 (a int primary key auto_increment, b varchar(100)); + +# now test if it's two functions, each of them inserts in one table + +drop function f; +# we need a unique key to have sorting of rows by mysqldump +create table t14 (unique (a)) select * from t2; +truncate table t2; +delimiter |; +create function f1 (x varchar(100)) returns int deterministic +begin + insert into t1 values(null,x); + return 1; +end| +create function f2 (x varchar(100)) returns int deterministic +begin + insert into t2 values(null,x); + return 1; +end| +delimiter ;| +select f1("try_46_"),f2("try_47_"); + +sync_slave_with_master; +insert into t2 values(2,null),(3,null),(4,null); +delete from t2 where a>=2; + +connection master; +# Test with SELECT and INSERT +select f1("try_48_"),f2("try_49_"); +insert into t3 values(concat("try_50_",f1("try_51_"),f2("try_52_"))); +sync_slave_with_master; + +# verify that if f2 does only read on an auto_inc table, this does not +# switch to RBB +connection master; +drop function f2; +delimiter |; +create function f2 (x varchar(100)) returns int deterministic +begin + declare y int; + insert into t1 values(null,x); + set y = (select count(*) from t2); + return y; +end| +delimiter ;| +select f1("try_53_"),f2("try_54_"); +sync_slave_with_master; + +# And now, a normal statement with a trigger (no stored functions) + +connection master; +drop function f2; +delimiter |; +create trigger t1_bi before insert on t1 for each row +begin + insert into t2 values(null,"try_55_"); +end| +delimiter ;| +insert into t1 values(null,"try_56_"); +# and now remove one auto_increment and verify SBB +alter table t1 modify a int, drop primary key; +insert into t1 values(null,"try_57_"); +sync_slave_with_master; + +# Test for BUG#20499 "mixed mode with temporary table breaks binlog" +# Slave used to have only 2 rows instead of 3. +connection master; +CREATE TEMPORARY TABLE t15 SELECT UUID(); +create table t16 like t15; +INSERT INTO t16 SELECT * FROM t15; +# we'll verify that this one is done RBB +insert into t16 values("try_65_"); +drop table t15; +# we'll verify that this one is done SBB +insert into t16 values("try_66_"); +sync_slave_with_master; + +# and now compare: + +connection master; + +# first check that data on master is sensible +select count(*) from t1; +select count(*) from t2; +select count(*) from t3; +select count(*) from t4; +select count(*) from t5; +select count(*) from t11; +select count(*) from t20; +select count(*) from t21; +select count(*) from t22; +select count(*) from t12; +select count(*) from t13; +select count(*) from t14; +select count(*) from t16; +if ($you_want_to_test_UDF) +{ + select count(*) from t6; + select count(*) from t7; + select count(*) from t8; + select count(*) from t9; +} + +sync_slave_with_master; + +# +# Bug#20863 If binlog format is changed between update and unlock of +# tables, wrong binlog +# + +connection master; +DROP TABLE IF EXISTS t11; +SET SESSION BINLOG_FORMAT=STATEMENT; +CREATE TABLE t11 (song VARCHAR(255)); +LOCK TABLES t11 WRITE; +SET SESSION BINLOG_FORMAT=ROW; +INSERT INTO t11 VALUES('Several Species of Small Furry Animals Gathered Together in a Cave and Grooving With a Pict'); +SET SESSION BINLOG_FORMAT=STATEMENT; +INSERT INTO t11 VALUES('Careful With That Axe, Eugene'); +UNLOCK TABLES; + +--query_vertical SELECT * FROM t11 +sync_slave_with_master; +USE mysqltest1; +--query_vertical SELECT * FROM t11 + +connection master; +DROP TABLE IF EXISTS t12; +SET SESSION BINLOG_FORMAT=MIXED; +CREATE TABLE t12 (data LONG); +LOCK TABLES t12 WRITE; +INSERT INTO t12 VALUES(UUID()); +UNLOCK TABLES; +sync_slave_with_master; + +# +# BUG#28086: SBR of USER() becomes corrupted on slave +# + +connection master; + +# Just to get something that is non-trivial, albeit still simple, we +# stuff the result of USER() and CURRENT_USER() into a variable. +--delimiter $$ +CREATE FUNCTION my_user() + RETURNS CHAR(64) +BEGIN + DECLARE user CHAR(64); + SELECT USER() INTO user; + RETURN user; +END $$ +--delimiter ; + +--delimiter $$ +CREATE FUNCTION my_current_user() + RETURNS CHAR(64) +BEGIN + DECLARE user CHAR(64); + SELECT CURRENT_USER() INTO user; + RETURN user; +END $$ +--delimiter ; + +DROP TABLE IF EXISTS t13; +CREATE TABLE t13 (data CHAR(64)); +INSERT INTO t13 VALUES (USER()); +INSERT INTO t13 VALUES (my_user()); +INSERT INTO t13 VALUES (CURRENT_USER()); +INSERT INTO t13 VALUES (my_current_user()); + +sync_slave_with_master; + +# as we're using UUID we don't SELECT but use "diff" like in rpl_row_UUID +--exec $MYSQL_DUMP --compact --order-by-primary --skip-extended-insert --no-create-info mysqltest1 > $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql +--exec $MYSQL_DUMP_SLAVE --compact --order-by-primary --skip-extended-insert --no-create-info mysqltest1 > $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_slave.sql + +# Let's compare. Note: If they match test will pass, if they do not match +# the test will show that the diff statement failed and not reject file +# will be created. You will need to go to the mysql-test dir and diff +# the files your self to see what is not matching + +diff_files $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_slave.sql; + +connection master; + +# Now test that mysqlbinlog works fine on a binlog generated by the +# mixed mode + +# BUG#11312 "DELIMITER is not written to the binary log that causes +# syntax error" makes that mysqlbinlog will fail if we pass it the +# text of queries; this forces us to use --base64-output here. + +# BUG#20929 "BINLOG command causes invalid free plus assertion +# failure" makes mysqld segfault when receiving --base64-output + +# So I can't enable this piece of test +# SIGH + +if ($enable_when_11312_or_20929_fixed) +{ +--exec $MYSQL_BINLOG --base64-output $MYSQLTEST_VARDIR/log/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_mixed.sql +drop database mysqltest1; +--exec $MYSQL < $MYSQLTEST_VARDIR/tmp/mysqlbinlog_mixed.sql +--exec $MYSQL_DUMP --compact --order-by-primary --skip-extended-insert --no-create-info mysqltest1 > $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql +# the old mysqldump output on slave is the same as what it was on +# master before restoring on master. +diff_files $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_slave.sql; +} + +drop database mysqltest1; +sync_slave_with_master; + +connection master; +# Restore binlog format setting +set global binlog_format =@my_binlog_format; +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_sync.inc b/mysql-test/extra/rpl_tests/rpl_sync.inc new file mode 100644 index 0000000000000..ede3c3c515f3e --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_sync.inc @@ -0,0 +1,159 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +######################################################################################## +# This test verifies the options --sync-relay-log-info and --relay-log-recovery by +# crashing the slave in two different situations: +# (case-1) - Corrupt the relay log with changes which were not processed by +# the SQL Thread and crashes it. +# (case-2) - Corrupt the master.info with wrong coordinates and crashes it. +# +# Case 1: +# 1 - Stops the SQL Thread +# 2 - Inserts new records into the master. +# 3 - Corrupts the relay-log.bin* which most likely has such changes. +# 4 - Crashes the slave +# 5 - Verifies if the slave is sync with the master which means that the information +# loss was circumvented by the recovery process. +# +# Case 2: +# 1 - Stops the SQL/IO Threads +# 2 - Inserts new records into the master. +# 3 - Corrupts the master.info with wrong coordinates. +# 4 - Crashes the slave +# 5 - Verifies if the slave is sync with the master which means that the information +# loss was circumvented by the recovery process. +######################################################################################## + +######################################################################################## +# Configuring the environment +######################################################################################## +--echo =====Configuring the enviroment=======; +--source include/master-slave.inc +--source include/not_embedded.inc +--source include/not_valgrind.inc +--source include/have_debug.inc +--source include/have_innodb.inc +--source include/not_crashrep.inc + +call mtr.add_suppression('Attempting backtrace'); +call mtr.add_suppression("Recovery from master pos .* and file master-bin.000001"); +# Use innodb so we do not get "table should be repaired" issues. +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +flush tables; +CREATE TABLE t1(a INT, PRIMARY KEY(a)) engine=innodb; + +insert into t1(a) values(1); +insert into t1(a) values(2); +insert into t1(a) values(3); + +######################################################################################## +# Case 1: Corrupt a relay-log.bin* +######################################################################################## +--echo =====Inserting data on the master but without the SQL Thread being running=======; +sync_slave_with_master; + +connection slave; +let $MYSQLD_SLAVE_DATADIR= `select @@datadir`; +--replace_result $MYSQLD_SLAVE_DATADIR MYSQLD_SLAVE_DATADIR +--copy_file $MYSQLD_SLAVE_DATADIR/master.info $MYSQLD_SLAVE_DATADIR/master.backup +--source include/stop_slave_sql.inc + +connection master; +insert into t1(a) values(4); +insert into t1(a) values(5); +insert into t1(a) values(6); + +--echo =====Removing relay log files and crashing/recoverying the slave=======; +connection slave; +--source include/stop_slave_io.inc + +let $file= query_get_value("SHOW SLAVE STATUS", Relay_Log_File, 1); + +--let FILE_TO_CORRUPT= $MYSQLD_SLAVE_DATADIR/$file +perl; +$file= $ENV{'FILE_TO_CORRUPT'}; +open(FILE, ">$file") || die "Unable to open $file."; +truncate(FILE,0); +print FILE "failure"; +close ($file); +EOF + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +SET SESSION debug_dbug="d,crash_before_rotate_relaylog"; +--error 2013 +FLUSH LOGS; + +--let $rpl_server_number= 2 +--source include/rpl_reconnect.inc + +--echo =====Dumping and comparing tables=======; +--source include/start_slave.inc + +connection master; +sync_slave_with_master; + +let $diff_tables=master:t1,slave:t1; +source include/diff_tables.inc; + +######################################################################################## +# Case 2: Corrupt a master.info +######################################################################################## +--echo =====Corrupting the master.info=======; +connection slave; +--source include/stop_slave.inc + +connection master; +FLUSH LOGS; + +insert into t1(a) values(7); +insert into t1(a) values(8); +insert into t1(a) values(9); + +connection slave; +let MYSQLD_SLAVE_DATADIR=`select @@datadir`; + +--perl +use strict; +use warnings; +my $src= "$ENV{'MYSQLD_SLAVE_DATADIR'}/master.backup"; +my $dst= "$ENV{'MYSQLD_SLAVE_DATADIR'}/master.info"; +open(FILE, "<", $src) or die; +my @content= ; +close FILE; +open(FILE, ">", $dst) or die; +binmode FILE; +print FILE @content; +close FILE; +EOF + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +SET SESSION debug_dbug="d,crash_before_rotate_relaylog"; +--error 2013 +FLUSH LOGS; + +--let $rpl_server_number= 2 +--source include/rpl_reconnect.inc + +--echo =====Dumping and comparing tables=======; +--source include/start_slave.inc + +connection master; +sync_slave_with_master; + +let $diff_tables=master:t1,slave:t1; +source include/diff_tables.inc; + +######################################################################################## +# Clean up +######################################################################################## +--echo =====Clean up=======; +connection master; +drop table t1; + +--remove_file $MYSQLD_SLAVE_DATADIR/master.backup +--source include/rpl_end.inc + diff --git a/mysql-test/extra/rpl_tests/rpl_temporal_format_default_to_default.inc b/mysql-test/extra/rpl_tests/rpl_temporal_format_default_to_default.inc new file mode 100644 index 0000000000000..6728ff55d6f5e --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_temporal_format_default_to_default.inc @@ -0,0 +1,82 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption). +# Please check all dependent tests after modifying it +# + +--source include/master-slave.inc + +if ($force_master_mysql56_temporal_format) +{ + connection master; + eval SET @@global.mysql56_temporal_format=$force_master_mysql56_temporal_format; +} + +if ($force_slave_mysql56_temporal_format) +{ + connection slave; + eval SET @@global.mysql56_temporal_format=$force_slave_mysql56_temporal_format; +} + +connection master; +SELECT @@global.mysql56_temporal_format AS on_master; +connection slave; +SELECT @@global.mysql56_temporal_format AS on_slave; +connection master; + +CREATE TABLE t1 +( + c0 TIME(0), + c1 TIME(1), + c2 TIME(2), + c3 TIME(3), + c4 TIME(4), + c5 TIME(5), + c6 TIME(6) +); +CREATE TABLE t2 +( + c0 TIMESTAMP(0), + c1 TIMESTAMP(1), + c2 TIMESTAMP(2), + c3 TIMESTAMP(3), + c4 TIMESTAMP(4), + c5 TIMESTAMP(5), + c6 TIMESTAMP(6) +); + +CREATE TABLE t3 +( + c0 DATETIME(0), + c1 DATETIME(1), + c2 DATETIME(2), + c3 DATETIME(3), + c4 DATETIME(4), + c5 DATETIME(5), + c6 DATETIME(6) +); +INSERT INTO t1 VALUES ('01:01:01','01:01:01.1','01:01:01.11','01:01:01.111','01:01:01.1111','01:01:01.11111','01:01:01.111111'); +INSERT INTO t2 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +INSERT INTO t3 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +sync_slave_with_master; + +connection slave; +--query_vertical SELECT * FROM t1; +--query_vertical SELECT * FROM t2; +--query_vertical SELECT * FROM t3; +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; + +connection master; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; + +connection slave; +SET @@global.mysql56_temporal_format=DEFAULT; +connection master; +SET @@global.mysql56_temporal_format=DEFAULT; + +--source include/rpl_end.inc diff --git a/mysql-test/extra/rpl_tests/rpl_typeconv.inc b/mysql-test/extra/rpl_tests/rpl_typeconv.inc new file mode 100644 index 0000000000000..0f078854ec28f --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_typeconv.inc @@ -0,0 +1,78 @@ +# +# This include file is used by more than one test suite +# (currently rpl and binlog_encryption suite). +# Please check all dependent tests after modifying it +# + +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +connection slave; +set @saved_slave_type_conversions = @@global.slave_type_conversions; +CREATE TABLE type_conversions ( + TestNo INT AUTO_INCREMENT PRIMARY KEY, + Source TEXT, + Target TEXT, + Flags TEXT, + On_Master TEXT, + On_Slave TEXT, + Expected TEXT, + Compare INT, + Error TEXT); + +SELECT @@global.slave_type_conversions; +SET GLOBAL SLAVE_TYPE_CONVERSIONS=''; +SELECT @@global.slave_type_conversions; +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_NON_LOSSY'; +SELECT @@global.slave_type_conversions; +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY'; +SELECT @@global.slave_type_conversions; +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY'; +SELECT @@global.slave_type_conversions; +--error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY,NONEXISTING_BIT'; +SELECT @@global.slave_type_conversions; + +# Checking strict interpretation of type conversions +connection slave; +SET GLOBAL SLAVE_TYPE_CONVERSIONS=''; +source extra/rpl_tests/type_conversions.test; + +# Checking lossy integer type conversions +connection slave; +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_NON_LOSSY'; +source extra/rpl_tests/type_conversions.test; + +# Checking non-lossy integer type conversions +connection slave; +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY'; +source extra/rpl_tests/type_conversions.test; + +# Checking all type conversions +connection slave; +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY'; +source extra/rpl_tests/type_conversions.test; + +connection slave; +--echo **** Result of conversions **** +disable_query_log; +SELECT RPAD(Source, 15, ' ') AS Source_Type, + RPAD(Target, 15, ' ') AS Target_Type, + RPAD(Flags, 25, ' ') AS All_Type_Conversion_Flags, + IF(Compare IS NULL AND Error IS NOT NULL, '', + IF(Compare, '', + CONCAT("'", On_Slave, "' != '", Expected, "'"))) + AS Value_On_Slave + FROM type_conversions; +enable_query_log; +DROP TABLE type_conversions; + +call mtr.add_suppression("Slave SQL.*Column 1 of table .test.t1. cannot be converted from type.* error.* 1677"); + +connection master; +DROP TABLE t1; +sync_slave_with_master; + +set global slave_type_conversions = @saved_slave_type_conversions; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/multi_source/reset_master_slave.inc b/mysql-test/include/reset_master_slave.inc similarity index 100% rename from mysql-test/suite/multi_source/reset_master_slave.inc rename to mysql-test/include/reset_master_slave.inc diff --git a/mysql-test/include/search_pattern_in_file.inc b/mysql-test/include/search_pattern_in_file.inc index 84237026ed06e..3280dbfd574db 100644 --- a/mysql-test/include/search_pattern_in_file.inc +++ b/mysql-test/include/search_pattern_in_file.inc @@ -60,25 +60,30 @@ perl; use strict; - my $search_file= $ENV{'SEARCH_FILE'} or die "SEARCH_FILE not set"; + die "SEARCH_FILE not set" unless $ENV{'SEARCH_FILE'}; + my @search_files= glob($ENV{'SEARCH_FILE'}); my $search_pattern= $ENV{'SEARCH_PATTERN'} or die "SEARCH_PATTERN not set"; my $search_range= $ENV{'SEARCH_RANGE'}; - my $file_content; + my $content; $search_range= 50000 unless $search_range =~ /-?[0-9]+/; - open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n"); - if ($search_range >= 0) { - read(FILE, $file_content, $search_range, 0); - } else { - my $size= -s $search_file; - $search_range = -$size if $size > -$search_range; - seek(FILE, $search_range, 2); - read(FILE, $file_content, -$search_range, 0); + foreach my $search_file (@search_files) { + open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n"); + my $file_content; + if ($search_range >= 0) { + read(FILE, $file_content, $search_range, 0); + } else { + my $size= -s $search_file; + $search_range = -$size if $size > -$search_range; + seek(FILE, $search_range, 2); + read(FILE, $file_content, -$search_range, 0); + } + close(FILE); + $content.= $file_content; } - close(FILE); - $search_file =~ s{^.*?([^/\\]+)$}{$1}; - if ($file_content =~ m{$search_pattern}) { - print "FOUND /$search_pattern/ in $search_file\n" + $ENV{'SEARCH_FILE'} =~ s{^.*?([^/\\]+)$}{$1}; + if ($content =~ m{$search_pattern}) { + print "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" } else { - print "NOT FOUND /$search_pattern/ in $search_file\n" + print "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" } EOF diff --git a/mysql-test/suite/multi_source/wait_for_sql_thread_read_all.inc b/mysql-test/include/wait_for_sql_thread_read_all.inc similarity index 100% rename from mysql-test/suite/multi_source/wait_for_sql_thread_read_all.inc rename to mysql-test/include/wait_for_sql_thread_read_all.inc diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 2bd89f5ae4939..2485b948bf1d4 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -170,6 +170,7 @@ END main- archive- binlog- + binlog_encryption- csv- encryption- federated- diff --git a/mysql-test/suite/binlog/t/binlog_incident.test b/mysql-test/suite/binlog/t/binlog_incident.test index 1c526ca59801f..c4270a074f097 100644 --- a/mysql-test/suite/binlog/t/binlog_incident.test +++ b/mysql-test/suite/binlog/t/binlog_incident.test @@ -1,29 +1 @@ -# The purpose of this test is to provide a reference for how the -# incident log event is represented in the output from the mysqlbinlog -# program. - -source include/have_log_bin.inc; -source include/have_debug.inc; -source include/binlog_start_pos.inc; - -let $MYSQLD_DATADIR= `select @@datadir`; -RESET MASTER; - -CREATE TABLE t1 (a INT); - -INSERT INTO t1 VALUES (1),(2),(3); -SELECT * FROM t1; - -# This will generate an incident log event and store it in the binary -# log before the replace statement. -REPLACE INTO t1 VALUES (4); - -DROP TABLE t1; -FLUSH LOGS; - -exec $MYSQL_BINLOG --start-position=$binlog_start_pos $MYSQLD_DATADIR/master-bin.000001 >$MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql; ---disable_query_log -eval SELECT cont LIKE '%RELOAD DATABASE; # Shall generate syntax error%' AS `Contain RELOAD DATABASE` FROM (SELECT load_file('$MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql') AS cont) AS tbl; ---enable_query_log - -remove_file $MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql; +--source extra/binlog_tests/binlog_incident.inc diff --git a/mysql-test/suite/binlog/t/binlog_index.test b/mysql-test/suite/binlog/t/binlog_index.test index 26f3595db2b02..1837c683bbaae 100644 --- a/mysql-test/suite/binlog/t/binlog_index.test +++ b/mysql-test/suite/binlog/t/binlog_index.test @@ -1,272 +1 @@ -# -# testing of purging of binary log files bug#18199/Bug#18453 -# -source include/have_log_bin.inc; -source include/not_embedded.inc; -# Don't test this under valgrind, memory leaks will occur ---source include/not_valgrind.inc -source include/have_debug.inc; -# Avoid CrashReporter popup on Mac ---source include/not_crashrep.inc -call mtr.add_suppression('Attempting backtrace'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); -call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); -call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); -call mtr.add_suppression('Could not open .*'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); -flush tables; - -let $old=`select @@debug`; - -RESET MASTER; - -let $MYSQLD_DATADIR= `select @@datadir`; -let $INDEX=$MYSQLD_DATADIR/master-bin.index; - -# -# testing purge binary logs TO -# - -flush logs; -flush logs; -flush logs; - -source include/show_binary_logs.inc; -remove_file $MYSQLD_DATADIR/master-bin.000001; -flush tables; - -# there must be a warning with file names -replace_regex /\.[\\\/]master/master/; ---source include/wait_for_binlog_checkpoint.inc -purge binary logs TO 'master-bin.000004'; - ---echo *** must show a list starting from the 'TO' argument of PURGE *** -source include/show_binary_logs.inc; - -# -# testing purge binary logs BEFORE -# - -reset master; - -flush logs; -flush logs; -flush logs; -remove_file $MYSQLD_DATADIR/master-bin.000001; - ---echo *** must be a warning master-bin.000001 was not found *** -let $date=`select NOW() + INTERVAL 1 MINUTE`; ---disable_query_log -replace_regex /\.[\\\/]master/master/; ---source include/wait_for_binlog_checkpoint.inc -eval purge binary logs BEFORE '$date'; ---enable_query_log - ---echo *** must show one record, of the active binlog, left in the index file after PURGE *** -source include/show_binary_logs.inc; - -# -# testing a fatal error -# Turning a binlog file into a directory must be a portable setup -# - -reset master; - -flush logs; -flush logs; -flush logs; - -remove_file $MYSQLD_DATADIR/master-bin.000001; -mkdir $MYSQLD_DATADIR/master-bin.000001; - ---source include/wait_for_binlog_checkpoint.inc ---error ER_BINLOG_PURGE_FATAL_ERR -purge binary logs TO 'master-bin.000002'; -replace_regex /\.[\\\/]master/master/; -show warnings; -rmdir $MYSQLD_DATADIR/master-bin.000001; ---disable_warnings -reset master; ---enable_warnings - ---echo # crash_purge_before_update_index -flush logs; - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug_dbug="+d,crash_purge_before_update_index"; ---source include/wait_for_binlog_checkpoint.inc ---error 2013 -purge binary logs TO 'master-bin.000002'; - ---enable_reconnect ---source include/wait_until_connected_again.inc - -file_exists $MYSQLD_DATADIR/master-bin.000001; -file_exists $MYSQLD_DATADIR/master-bin.000002; -file_exists $MYSQLD_DATADIR/master-bin.000003; ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # crash_purge_non_critical_after_update_index -flush logs; - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug_dbug="+d,crash_purge_non_critical_after_update_index"; ---source include/wait_for_binlog_checkpoint.inc ---error 2013 -purge binary logs TO 'master-bin.000004'; - ---enable_reconnect ---source include/wait_until_connected_again.inc - ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000001; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000002; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000003; ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # crash_purge_critical_after_update_index -flush logs; - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug_dbug="+d,crash_purge_critical_after_update_index"; ---source include/wait_for_binlog_checkpoint.inc ---error 2013 -purge binary logs TO 'master-bin.000006'; - ---enable_reconnect ---source include/wait_until_connected_again.inc - ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000004; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000005; -file_exists $MYSQLD_DATADIR/master-bin.000006; -file_exists $MYSQLD_DATADIR/master-bin.000007; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000008; ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # crash_create_non_critical_before_update_index ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug_dbug="+d,crash_create_non_critical_before_update_index"; ---error 2013 -flush logs; - ---enable_reconnect ---source include/wait_until_connected_again.inc - -file_exists $MYSQLD_DATADIR/master-bin.000008; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000009; ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # crash_create_critical_before_update_index ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug_dbug="+d,crash_create_critical_before_update_index"; ---error 2013 -flush logs; - ---enable_reconnect ---source include/wait_until_connected_again.inc - -file_exists $MYSQLD_DATADIR/master-bin.000009; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000010; ---error 1 -file_exists $MYSQLD_DATADIR/master-bin.000011; ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # crash_create_after_update_index ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug_dbug="+d,crash_create_after_update_index"; ---error 2013 -flush logs; - ---enable_reconnect ---source include/wait_until_connected_again.inc - -file_exists $MYSQLD_DATADIR/master-bin.000010; -file_exists $MYSQLD_DATADIR/master-bin.000011; ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # ---echo # This should put the server in unsafe state and stop ---echo # accepting any command. If we inject a fault at this ---echo # point and continue the execution the server crashes. ---echo # - ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # fault_injection_registering_index -SET SESSION debug_dbug="+d,fault_injection_registering_index"; --- replace_regex /\.[\\\/]master/master/ --- error ER_CANT_OPEN_FILE -flush logs; - ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---source include/restart_mysqld.inc - ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---echo # fault_injection_updating_index -SET SESSION debug_dbug="+d,fault_injection_updating_index"; --- replace_regex /\.[\\\/]master/master/ --- error ER_CANT_OPEN_FILE -flush logs; - ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - ---source include/restart_mysqld.inc - ---chmod 0644 $INDEX --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SET @index=LOAD_FILE('$index') --- replace_regex /\.[\\\/]master/master/ -SELECT @index; - -eval SET SESSION debug_dbug="$old"; - ---echo End of tests +--source extra/binlog_tests/binlog_index.inc diff --git a/mysql-test/suite/binlog/t/binlog_ioerr.test b/mysql-test/suite/binlog/t/binlog_ioerr.test index f23fadfc1b467..3155e14e6b0aa 100644 --- a/mysql-test/suite/binlog/t/binlog_ioerr.test +++ b/mysql-test/suite/binlog/t/binlog_ioerr.test @@ -1,30 +1 @@ -source include/have_debug.inc; -source include/have_innodb.inc; -source include/have_log_bin.inc; -source include/have_binlog_format_mixed_or_statement.inc; - -CALL mtr.add_suppression("Error writing file 'master-bin'"); - -RESET MASTER; - -CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb; -INSERT INTO t1 VALUES(0); -SET SESSION debug_dbug='+d,fail_binlog_write_1'; ---error ER_ERROR_ON_WRITE -INSERT INTO t1 VALUES(1); ---error ER_ERROR_ON_WRITE -INSERT INTO t1 VALUES(2); -SET SESSION debug_dbug=''; -INSERT INTO t1 VALUES(3); -SELECT * FROM t1; - -# Actually the output from this currently shows a bug. -# The injected IO error leaves partially written transactions in the binlog in -# the form of stray "BEGIN" events. -# These should disappear from the output if binlog error handling is improved -# (see MySQL Bug#37148 and WL#1790). ---replace_regex /\/\* xid=.* \*\//\/* XID *\// /Server ver: .*, Binlog ver: .*/Server ver: #, Binlog ver: #/ /table_id: [0-9]+/table_id: #/ ---replace_column 1 BINLOG 2 POS 5 ENDPOS -SHOW BINLOG EVENTS; - -DROP TABLE t1; +--source extra/binlog_tests/binlog_ioerr.inc diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog-cp932.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog-cp932.test index 2a210bea0e05c..58c4befa8d648 100644 --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog-cp932.test +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog-cp932.test @@ -1,26 +1 @@ -# disabled in embedded until tools running is fixed with embedded ---source include/not_embedded.inc - --- source include/have_binlog_format_mixed_or_statement.inc --- source include/have_cp932.inc --- source include/have_log_bin.inc - -RESET MASTER; - -# Bug#16217 (mysql client did not know how not switch its internal charset) -create table t3 (f text character set utf8); -create table t4 (f text character set cp932); ---exec $MYSQL --default-character-set=utf8 test -e "insert into t3 values(_utf8'ソ')" ---exec $MYSQL --default-character-set=cp932 test -e "insert into t4 values(_cp932'ƒ\');" -flush logs; -rename table t3 to t03, t4 to t04; -let $MYSQLD_DATADIR= `select @@datadir`; ---exec $MYSQL_BINLOG --short-form $MYSQLD_DATADIR/master-bin.000001 | $MYSQL --default-character-set=utf8 -# original and recovered data must be equal -select HEX(f) from t03; -select HEX(f) from t3; -select HEX(f) from t04; -select HEX(f) from t4; - -drop table t3, t4, t03, t04; ---echo End of 5.0 tests +--source extra/binlog_tests/binlog_mysqlbinlog-cp932.inc diff --git a/mysql-test/suite/binlog/t/binlog_row_annotate.test b/mysql-test/suite/binlog/t/binlog_row_annotate.test index e831b29df009f..569391e68ce23 100644 --- a/mysql-test/suite/binlog/t/binlog_row_annotate.test +++ b/mysql-test/suite/binlog/t/binlog_row_annotate.test @@ -1,191 +1 @@ -############################################################################### -# WL47: Store in binlog text of statements that caused RBR events -# new event: ANNOTATE_ROWS_EVENT -# new master option: --binlog-annotate-row-events -# new mysqlbinlog option: --skip-annotate-row-events -# -# Intended to test that: -# *** If the --binlog-annotate-row-events option is switched on on master -# then Annotate_rows events: -# - are generated; -# - are genrated only once for "multi-table-maps" rbr queries; -# - are not generated when the corresponding queries are filtered away; -# - are generated when the corresponding queries are filtered away partialy -# (e.g. in case of multi-delete). -# *** Annotate_rows events are printed by mysqlbinlog started without -# --skip-annotate-row-events options both in remote and local cases. -# *** Annotate_rows events are not printed by mysqlbinlog started with -# --skip-annotate-row-events options both in remote and local cases. -############################################################################### - ---source include/have_log_bin.inc ---source include/have_binlog_format_row.inc ---source include/binlog_start_pos.inc - ---disable_query_log - -set sql_mode=""; - -# Fix timestamp to avoid varying results -SET timestamp=1000000000; - -# Delete all existing binary logs -RESET MASTER; - ---disable_warnings -DROP DATABASE IF EXISTS test1; -DROP DATABASE IF EXISTS test2; -DROP DATABASE IF EXISTS test3; -DROP DATABASE IF EXISTS xtest1; -DROP DATABASE IF EXISTS xtest2; ---enable_warnings - -CREATE DATABASE test1; -CREATE TABLE test1.t1(a int); - -CREATE DATABASE test2; -CREATE TABLE test2.t2(a int); -CREATE VIEW test2.v2 AS SELECT * FROM test2.t2; - -CREATE DATABASE test3; -CREATE TABLE test3.t3(a int); - -CREATE DATABASE xtest1; -CREATE TABLE xtest1.xt1(a int); - -CREATE DATABASE xtest2; -CREATE TABLE xtest2.xt2(a int); - -# By default SESSION binlog_annotate_row_events = OFF - -INSERT INTO test1.t1 VALUES (1), (2), (3); - -SET SESSION binlog_annotate_row_events = ON; - -INSERT INTO test2.t2 VALUES (1), (2), (3); -INSERT INTO test3.t3 VALUES (1), (2), (3); - -# This query generates two Table maps but the Annotate -# event should appear only once before the first Table map -DELETE test1.t1, test2.t2 - FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 - WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a; - -# This event should be filtered out together with Annotate event -INSERT INTO xtest1.xt1 VALUES (1), (2), (3); - -# This event should pass the filter -INSERT INTO test2.v2 VALUES (1), (2), (3); - -# This event should pass the filter only for test2.t2 part -DELETE xtest1.xt1, test2.t2 - FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 - WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a; - -# These events should be filtered out together with Annotate events -INSERT INTO xtest1.xt1 VALUES (1), (2), (3); -INSERT INTO xtest2.xt2 VALUES (1), (2), (3); -DELETE xtest1.xt1, xtest2.xt2 - FROM xtest1.xt1 INNER JOIN xtest2.xt2 INNER JOIN test3.t3 - WHERE xtest1.xt1.a=xtest2.xt2.a AND xtest2.xt2.a=test3.t3.a; - -FLUSH LOGS; ---enable_query_log - ---echo ##################################################################################### ---echo # The following Annotate_rows events should appear below: ---echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) ---echo # - INSERT INTO test3.t3 VALUES (1), (2), (3) ---echo # - DELETE test1.t1, test2.t2 FROM <...> ---echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) ---echo # - DELETE xtest1.xt1, test2.t2 FROM <...> ---echo ##################################################################################### - -let $start_pos= `select @binlog_start_pos`; ---replace_column 2 # 5 # ---replace_result $start_pos ---replace_regex /table_id: [0-9]+/table_id: #/ /\/\* xid=.* \*\//\/* xid= *\// ---eval show binlog events in 'master-bin.000001' from $start_pos - ---echo # ---echo ##################################################################################### ---echo # mysqlbinlog ---echo # The following Annotates should appear in this output: ---echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) ---echo # - INSERT INTO test3.t3 VALUES (1), (2), (3) ---echo # - DELETE test1.t1, test2.t2 FROM <...> (with two subsequent Table maps) ---echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) ---echo # - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) ---echo ##################################################################################### - -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ ---exec $MYSQL_BINLOG --base64-output=decode-rows -v -v $MYSQLD_DATADIR/master-bin.000001 - ---echo # ---echo ##################################################################################### ---echo # mysqlbinlog --database=test1 ---echo # The following Annotate should appear in this output: ---echo # - DELETE test1.t1, test2.t2 FROM <...> ---echo ##################################################################################### - -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ ---exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v $MYSQLD_DATADIR/master-bin.000001 - ---echo # ---echo ##################################################################################### ---echo # mysqlbinlog --skip-annotate-row-events ---echo # No Annotates should appear in this output ---echo ##################################################################################### - -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ ---exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v $MYSQLD_DATADIR/master-bin.000001 - ---echo # ---echo ##################################################################################### ---echo # mysqlbinlog --read-from-remote-server ---echo # The following Annotates should appear in this output: ---echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) ---echo # - INSERT INTO test3.t3 VALUES (1), (2), (3) ---echo # - DELETE test1.t1, test2.t2 FROM <...> (with two subsequent Table maps) ---echo # - INSERT INTO test2.t2 VALUES (1), (2), (3) ---echo # - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) ---echo ##################################################################################### - -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ ---exec $MYSQL_BINLOG --base64-output=decode-rows -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 - ---echo # ---echo ##################################################################################### ---echo # mysqlbinlog --read-from-remote-server --database=test1 ---echo # The following Annotate should appear in this output: ---echo # - DELETE test1.t1, test2.t2 FROM <...> ---echo ##################################################################################### - -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ ---exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 - ---echo # ---echo ##################################################################################### ---echo # mysqlbinlog --read-from-remote-server --skip-annotate-row-events ---echo # No Annotates should appear in this output ---echo ##################################################################################### - -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ ---exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 - -# Clean-up - ---disable_query_log -DROP DATABASE test1; -DROP DATABASE test2; -DROP DATABASE test3; -DROP DATABASE xtest1; -DROP DATABASE xtest2; ---enable_query_log - +--source extra/binlog_tests/binlog_row_annotate.inc diff --git a/mysql-test/suite/binlog/t/binlog_write_error.test b/mysql-test/suite/binlog/t/binlog_write_error.test index 78f55c1bb0d5d..05f8eff6c3a38 100644 --- a/mysql-test/suite/binlog/t/binlog_write_error.test +++ b/mysql-test/suite/binlog/t/binlog_write_error.test @@ -1,102 +1 @@ -# -# === Name === -# -# binlog_write_error.test -# -# === Description === -# -# This test case check if the error of writing binlog file is properly -# reported and handled when executing statements. -# -# === Related Bugs === -# -# BUG#37148 -# - -source include/have_log_bin.inc; -source include/have_debug.inc; -source include/have_binlog_format_mixed_or_statement.inc; - ---echo # ---echo # Initialization ---echo # - -disable_warnings; -DROP TABLE IF EXISTS t1, t2; -DROP FUNCTION IF EXISTS f1; -DROP FUNCTION IF EXISTS f2; -DROP PROCEDURE IF EXISTS p1; -DROP PROCEDURE IF EXISTS p2; -DROP TRIGGER IF EXISTS tr1; -DROP TRIGGER IF EXISTS tr2; -DROP VIEW IF EXISTS v1, v2; -enable_warnings; - ---echo # ---echo # Test injecting binlog write error when executing queries ---echo # - -let $query= CREATE TABLE t1 (a INT); -source include/binlog_inject_error.inc; - -INSERT INTO t1 VALUES (1),(2),(3); - -let $query= INSERT INTO t1 VALUES (4),(5),(6); -source include/binlog_inject_error.inc; - -let $query= UPDATE t1 set a=a+1; -source include/binlog_inject_error.inc; - -let $query= DELETE FROM t1; -source include/binlog_inject_error.inc; - -let $query= CREATE TRIGGER tr1 AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t1 VALUES (new.a + 100); -source include/binlog_inject_error.inc; - -let $query= DROP TRIGGER tr1; -source include/binlog_inject_error.inc; - -let $query= ALTER TABLE t1 ADD (b INT); -source include/binlog_inject_error.inc; - -let $query= CREATE VIEW v1 AS SELECT a FROM t1; -source include/binlog_inject_error.inc; - -let $query= DROP VIEW v1; -source include/binlog_inject_error.inc; - -let $query= CREATE PROCEDURE p1(OUT rows INT) SELECT count(*) INTO rows FROM t1; -source include/binlog_inject_error.inc; - -let $query= DROP PROCEDURE p1; -source include/binlog_inject_error.inc; - -let $query= DROP TABLE t1; -source include/binlog_inject_error.inc; - -let $query= CREATE FUNCTION f1() RETURNS INT return 1; -source include/binlog_inject_error.inc; - -let $query= DROP FUNCTION f1; -source include/binlog_inject_error.inc; - -let $query= CREATE USER user1; -source include/binlog_inject_error.inc; - -let $query= REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1; -source include/binlog_inject_error.inc; - -let $query= DROP USER user1; -source include/binlog_inject_error.inc; - ---echo # ---echo # Cleanup ---echo # - -disable_warnings; -DROP TABLE IF EXISTS t1, t2; -DROP FUNCTION IF EXISTS f1; -DROP PROCEDURE IF EXISTS p1; -DROP TRIGGER IF EXISTS tr1; -DROP VIEW IF EXISTS v1, v2; -enable_warnings; +--source extra/binlog_tests/binlog_write_error.inc diff --git a/mysql-test/suite/binlog/t/binlog_xa_recover.test b/mysql-test/suite/binlog/t/binlog_xa_recover.test index 903044ca5bd4f..0e0b80433ff43 100644 --- a/mysql-test/suite/binlog/t/binlog_xa_recover.test +++ b/mysql-test/suite/binlog/t/binlog_xa_recover.test @@ -1,275 +1 @@ ---source include/have_innodb.inc ---source include/have_debug.inc ---source include/have_debug_sync.inc ---source include/have_binlog_format_row.inc -# Valgrind does not work well with test that crashes the server ---source include/not_valgrind.inc - -# (We do not need to restore these settings, as we crash the server). -SET GLOBAL max_binlog_size= 4096; -SET GLOBAL innodb_flush_log_at_trx_commit= 1; -RESET MASTER; - -CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb; -# Insert some data to force a couple binlog rotations (3), so we get some -# normal binlog checkpoints before starting the test. -INSERT INTO t1 VALUES (100, REPEAT("x", 4100)); -# Wait for the master-bin.000002 binlog checkpoint to appear. ---let $wait_for_all= 0 ---let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000002" ---let $field= Info ---let $condition= = "master-bin.000002" ---source include/wait_show_condition.inc -INSERT INTO t1 VALUES (101, REPEAT("x", 4100)); ---let $wait_for_all= 0 ---let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000003" ---let $field= Info ---let $condition= = "master-bin.000003" ---source include/wait_show_condition.inc -INSERT INTO t1 VALUES (102, REPEAT("x", 4100)); ---let $wait_for_all= 0 ---let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000004" ---let $field= Info ---let $condition= = "master-bin.000004" ---source include/wait_show_condition.inc - -# Now start a bunch of transactions that span multiple binlog -# files. Leave then in the state prepared-but-not-committed in the engine -# and crash the server. Check that crash recovery is able to recover all -# of them. -# -# We use debug_sync to get all the transactions into the prepared state before -# we commit any of them. This is because the prepare step flushes the InnoDB -# redo log - including any commits made before, so recovery would become -# unnecessary, decreasing the value of this test. -# -# We arrange to have con1 with a prepared transaction in master-bin.000004, -# con2 and con3 with a prepared transaction in master-bin.000005, and a new -# empty master-bin.000006. So the latest binlog checkpoint should be -# master-bin.000006. - -connect(con1,localhost,root,,); -# First wait after prepare and before write to binlog. -SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con1_wait WAIT_FOR con1_cont"; -# Then complete InnoDB commit in memory (but not commit checkpoint / write to -# disk), and hang until crash, leaving a transaction to be XA recovered. -SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con1_ready WAIT_FOR _ever"; -send INSERT INTO t1 VALUES (1, REPEAT("x", 4100)); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con1_wait"; - -connect(con2,localhost,root,,); -SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con2_wait WAIT_FOR con2_cont"; -SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con2_ready WAIT_FOR _ever"; -send INSERT INTO t1 VALUES (2, NULL); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con2_wait"; - -connect(con3,localhost,root,,); -SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con3_wait WAIT_FOR con3_cont"; -SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con3_ready WAIT_FOR _ever"; -send INSERT INTO t1 VALUES (3, REPEAT("x", 4100)); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con3_wait"; - -connect(con4,localhost,root,,); -SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con4_wait WAIT_FOR con4_cont"; -SET SESSION debug_dbug="+d,crash_commit_after_log"; -send INSERT INTO t1 VALUES (4, NULL); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con4_wait"; - -SET DEBUG_SYNC= "now SIGNAL con1_cont"; -SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; -SET DEBUG_SYNC= "now SIGNAL con2_cont"; -SET DEBUG_SYNC= "now WAIT_FOR con2_ready"; -SET DEBUG_SYNC= "now SIGNAL con3_cont"; -SET DEBUG_SYNC= "now WAIT_FOR con3_ready"; - -# Check that everything is committed in binary log. ---source include/show_binary_logs.inc ---let $binlog_file= master-bin.000003 ---let $binlog_start= 4 ---source include/show_binlog_events.inc ---let $binlog_file= master-bin.000004 ---source include/show_binlog_events.inc ---let $binlog_file= master-bin.000005 ---source include/show_binlog_events.inc ---let $binlog_file= master-bin.000006 ---source include/show_binlog_events.inc - - -# Check that server will not purge too much. -PURGE BINARY LOGS TO "master-bin.000006"; ---source include/show_binary_logs.inc - -# Now crash the server with one more transaction in prepared state. ---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -wait-binlog_xa_recover.test -EOF ---error 0,2006,2013 -SET DEBUG_SYNC= "now SIGNAL con4_cont"; -connection con4; ---error 2006,2013 -reap; - ---append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart-group_commit_binlog_pos.test -EOF - -connection default; ---enable_reconnect ---source include/wait_until_connected_again.inc - -# Check that all transactions are recovered. -SELECT a FROM t1 ORDER BY a; - ---echo Test that with multiple binlog checkpoints, recovery starts from the last one. -SET GLOBAL max_binlog_size= 4096; -SET GLOBAL innodb_flush_log_at_trx_commit= 1; -RESET MASTER; - -# Rotate to binlog master-bin.000003 while delaying binlog checkpoints. -# So we get multiple binlog checkpoints in master-bin.000003. -# Then complete the checkpoints, crash, and check that we only scan -# the necessary binlog file (ie. that we use the _last_ checkpoint). - -connect(con10,localhost,root,,); -SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con10_ready WAIT_FOR con10_cont"; -send INSERT INTO t1 VALUES (10, REPEAT("x", 4100)); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con10_ready"; - -connect(con11,localhost,root,,); -SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con11_ready WAIT_FOR con11_cont"; -send INSERT INTO t1 VALUES (11, REPEAT("x", 4100)); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con11_ready"; - -connect(con12,localhost,root,,); -SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con12_ready WAIT_FOR con12_cont"; -send INSERT INTO t1 VALUES (12, REPEAT("x", 4100)); - -connection default; -SET DEBUG_SYNC= "now WAIT_FOR con12_ready"; -INSERT INTO t1 VALUES (13, NULL); - ---source include/show_binary_logs.inc ---let $binlog_file= master-bin.000004 ---let $binlog_start= 4 ---source include/show_binlog_events.inc - -SET DEBUG_SYNC= "now SIGNAL con10_cont"; -connection con10; -reap; -connection default; - -# We need to sync the test case with the background processing of the -# commit checkpoint, otherwise we get nondeterministic results. -SET @old_dbug= @@global.DEBUG_DBUG; -SET GLOBAL debug_dbug="+d,binlog_background_checkpoint_processed"; - -SET DEBUG_SYNC= "now SIGNAL con12_cont"; -connection con12; -reap; -connection default; -SET DEBUG_SYNC= "now WAIT_FOR binlog_background_checkpoint_processed"; -SET GLOBAL debug_dbug= @old_dbug; - -SET DEBUG_SYNC= "now SIGNAL con11_cont"; -connection con11; -reap; - -connection default; -# Wait for the last (master-bin.000004) binlog checkpoint to appear. ---let $wait_for_all= 0 ---let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000004" ---let $field= Info ---let $condition= = "master-bin.000004" ---source include/wait_show_condition.inc - ---echo Checking that master-bin.000004 is the last binlog checkpoint ---source include/show_binlog_events.inc - ---echo Now crash the server -# It is not too easy to test XA recovery, as it runs early during server -# startup, before any connections can be made. -# What we do is set a DBUG error insert which will crash if XA recovery -# starts from any other binlog than master-bin.000004 (check the file -# binlog_xa_recover-master.opt). Then we will fail here if XA recovery -# would start from the wrong place. ---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -wait-binlog_xa_recover.test -EOF -SET SESSION debug_dbug="+d,crash_commit_after_log"; ---error 2006,2013 -INSERT INTO t1 VALUES (14, NULL); - ---append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart-group_commit_binlog_pos.test -EOF - -connection default; ---enable_reconnect ---source include/wait_until_connected_again.inc - -# Check that all transactions are recovered. -SELECT a FROM t1 ORDER BY a; - - ---echo *** Check that recovery works if we crashed early during rotate, before ---echo *** binlog checkpoint event could be written. - -SET GLOBAL max_binlog_size= 4096; -SET GLOBAL innodb_flush_log_at_trx_commit= 1; -RESET MASTER; - -# We need some initial data to reach binlog master-bin.000004. Otherwise -# crash recovery fails due to the error insert used for previous test. -INSERT INTO t1 VALUES (21, REPEAT("x", 4100)); -INSERT INTO t1 VALUES (22, REPEAT("x", 4100)); -# Wait for the master-bin.000003 binlog checkpoint to appear. ---let $wait_for_all= 0 ---let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000003" ---let $field= Info ---let $condition= = "master-bin.000003" ---source include/wait_show_condition.inc -INSERT INTO t1 VALUES (23, REPEAT("x", 4100)); -# Wait for the last (master-bin.000004) binlog checkpoint to appear. ---let $wait_for_all= 0 ---let $show_statement= SHOW BINLOG EVENTS IN "master-bin.000004" ---let $field= Info ---let $condition= = "master-bin.000004" ---source include/wait_show_condition.inc - ---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -wait-binlog_xa_recover.test -EOF -SET SESSION debug_dbug="+d,crash_before_write_checkpoint_event"; ---error 2006,2013 -INSERT INTO t1 VALUES (24, REPEAT("x", 4100)); - ---append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -restart-group_commit_binlog_pos.test -EOF - ---enable_reconnect ---source include/wait_until_connected_again.inc - -# Check that all transactions are recovered. -SELECT a FROM t1 ORDER BY a; - ---source include/show_binary_logs.inc ---let $binlog_file= master-bin.000004 ---let $binlog_start= 4 ---source include/show_binlog_events.inc - -# Cleanup -connection default; -DROP TABLE t1; +--source extra/binlog_tests/binlog_xa_recover.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_incident.combinations b/mysql-test/suite/binlog_encryption/binlog_incident.combinations new file mode 100644 index 0000000000000..2269ed4a7a152 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_incident.combinations @@ -0,0 +1,8 @@ +[stmt] +binlog-format=statement + +[mix] +binlog-format=mixed + +[row] +binlog-format=row diff --git a/mysql-test/suite/binlog_encryption/binlog_incident.result b/mysql-test/suite/binlog_encryption/binlog_incident.result new file mode 100644 index 0000000000000..6118f066f4adc --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_incident.result @@ -0,0 +1,13 @@ +RESET MASTER; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +SELECT * FROM t1; +a +1 +2 +3 +REPLACE INTO t1 VALUES (4); +DROP TABLE t1; +FLUSH LOGS; +Contain RELOAD DATABASE +0 diff --git a/mysql-test/suite/binlog_encryption/binlog_incident.test b/mysql-test/suite/binlog_encryption/binlog_incident.test new file mode 100644 index 0000000000000..d37ed3d552ded --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_incident.test @@ -0,0 +1,2 @@ +--let $use_remote_mysqlbinlog= 1 +--source extra/binlog_tests/binlog_incident.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_index.result b/mysql-test/suite/binlog_encryption/binlog_index.result new file mode 100644 index 0000000000000..8cdca86173744 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_index.result @@ -0,0 +1,187 @@ +call mtr.add_suppression('Attempting backtrace'); +call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); +call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); +call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); +call mtr.add_suppression('Could not open .*'); +call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); +flush tables; +RESET MASTER; +flush logs; +flush logs; +flush logs; +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +master-bin.000003 # +master-bin.000004 # +flush tables; +purge binary logs TO 'master-bin.000004'; +Warnings: +Warning 1612 Being purged log master-bin.000001 was not found +*** must show a list starting from the 'TO' argument of PURGE *** +show binary logs; +Log_name File_size +master-bin.000004 # +reset master; +flush logs; +flush logs; +flush logs; +*** must be a warning master-bin.000001 was not found *** +Warnings: +Warning 1612 Being purged log master-bin.000001 was not found +*** must show one record, of the active binlog, left in the index file after PURGE *** +show binary logs; +Log_name File_size +master-bin.000004 # +reset master; +flush logs; +flush logs; +flush logs; +purge binary logs TO 'master-bin.000002'; +ERROR HY000: Fatal error during log purge +show warnings; +Level Code Message +Warning 1377 a problem with deleting master-bin.000001; consider examining correspondence of your binlog index file to the actual binlog files +Error 1377 Fatal error during log purge +reset master; +# crash_purge_before_update_index +flush logs; +SET SESSION debug_dbug="+d,crash_purge_before_update_index"; +purge binary logs TO 'master-bin.000002'; +ERROR HY000: Lost connection to MySQL server during query +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000001 +master-bin.000002 +master-bin.000003 + +# crash_purge_non_critical_after_update_index +flush logs; +SET SESSION debug_dbug="+d,crash_purge_non_critical_after_update_index"; +purge binary logs TO 'master-bin.000004'; +ERROR HY000: Lost connection to MySQL server during query +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000004 +master-bin.000005 + +# crash_purge_critical_after_update_index +flush logs; +SET SESSION debug_dbug="+d,crash_purge_critical_after_update_index"; +purge binary logs TO 'master-bin.000006'; +ERROR HY000: Lost connection to MySQL server during query +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 + +# crash_create_non_critical_before_update_index +SET SESSION debug_dbug="+d,crash_create_non_critical_before_update_index"; +flush logs; +ERROR HY000: Lost connection to MySQL server during query +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 + +# crash_create_critical_before_update_index +SET SESSION debug_dbug="+d,crash_create_critical_before_update_index"; +flush logs; +ERROR HY000: Lost connection to MySQL server during query +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 + +# crash_create_after_update_index +SET SESSION debug_dbug="+d,crash_create_after_update_index"; +flush logs; +ERROR HY000: Lost connection to MySQL server during query +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 +master-bin.000010 +master-bin.000011 + +# +# This should put the server in unsafe state and stop +# accepting any command. If we inject a fault at this +# point and continue the execution the server crashes. +# +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 +master-bin.000010 +master-bin.000011 + +# fault_injection_registering_index +SET SESSION debug_dbug="+d,fault_injection_registering_index"; +flush logs; +ERROR HY000: Can't open file: 'master-bin.000012' (errno: 1 "Operation not permitted") +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 +master-bin.000010 +master-bin.000011 + +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 +master-bin.000010 +master-bin.000011 +master-bin.000012 + +# fault_injection_updating_index +SET SESSION debug_dbug="+d,fault_injection_updating_index"; +flush logs; +ERROR HY000: Can't open file: 'master-bin.000013' (errno: 1 "Operation not permitted") +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 +master-bin.000010 +master-bin.000011 +master-bin.000012 + +SET @index=LOAD_FILE('MYSQLTEST_VARDIR/mysqld.1/data//master-bin.index'); +SELECT @index; +@index +master-bin.000006 +master-bin.000007 +master-bin.000008 +master-bin.000009 +master-bin.000010 +master-bin.000011 +master-bin.000012 +master-bin.000013 + +SET SESSION debug_dbug=""; +End of tests diff --git a/mysql-test/suite/binlog_encryption/binlog_index.test b/mysql-test/suite/binlog_encryption/binlog_index.test new file mode 100644 index 0000000000000..1837c683bbaae --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_index.test @@ -0,0 +1 @@ +--source extra/binlog_tests/binlog_index.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_ioerr.result b/mysql-test/suite/binlog_encryption/binlog_ioerr.result new file mode 100644 index 0000000000000..6b3120b6d8904 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_ioerr.result @@ -0,0 +1,32 @@ +CALL mtr.add_suppression("Error writing file 'master-bin'"); +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb; +INSERT INTO t1 VALUES(0); +SET SESSION debug_dbug='+d,fail_binlog_write_1'; +INSERT INTO t1 VALUES(1); +ERROR HY000: Error writing file 'master-bin' (errno: 28 "No space left on device") +INSERT INTO t1 VALUES(2); +ERROR HY000: Error writing file 'master-bin' (errno: 28 "No space left on device") +SET SESSION debug_dbug=''; +INSERT INTO t1 VALUES(3); +SELECT * FROM t1; +a +0 +3 +SHOW BINLOG EVENTS; +Log_name Pos Event_type Server_id End_log_pos Info +BINLOG POS Format_desc 1 ENDPOS Server ver: #, Binlog ver: # +BINLOG POS Start_encryption 1 ENDPOS +BINLOG POS Gtid_list 1 ENDPOS [] +BINLOG POS Binlog_checkpoint 1 ENDPOS master-bin.000001 +BINLOG POS Gtid 1 ENDPOS GTID 0-1-1 +BINLOG POS Query 1 ENDPOS use `test`; CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb +BINLOG POS Gtid 1 ENDPOS BEGIN GTID 0-1-2 +BINLOG POS Query 1 ENDPOS use `test`; INSERT INTO t1 VALUES(0) +BINLOG POS Xid 1 ENDPOS COMMIT /* XID */ +BINLOG POS Gtid 1 ENDPOS BEGIN GTID 0-1-3 +BINLOG POS Gtid 1 ENDPOS BEGIN GTID 0-1-4 +BINLOG POS Gtid 1 ENDPOS BEGIN GTID 0-1-5 +BINLOG POS Query 1 ENDPOS use `test`; INSERT INTO t1 VALUES(3) +BINLOG POS Xid 1 ENDPOS COMMIT /* XID */ +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_encryption/binlog_ioerr.test b/mysql-test/suite/binlog_encryption/binlog_ioerr.test new file mode 100644 index 0000000000000..3155e14e6b0aa --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_ioerr.test @@ -0,0 +1 @@ +--source extra/binlog_tests/binlog_ioerr.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932-master.opt b/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932-master.opt new file mode 100644 index 0000000000000..bb0cda4519ad8 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932-master.opt @@ -0,0 +1 @@ +--max-binlog-size=8192 diff --git a/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.result b/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.result new file mode 100644 index 0000000000000..cbf6159516a76 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.result @@ -0,0 +1,19 @@ +RESET MASTER; +create table t3 (f text character set utf8); +create table t4 (f text character set cp932); +flush logs; +rename table t3 to t03, t4 to t04; +select HEX(f) from t03; +HEX(f) +E382BD +select HEX(f) from t3; +HEX(f) +E382BD +select HEX(f) from t04; +HEX(f) +835C +select HEX(f) from t4; +HEX(f) +835C +drop table t3, t4, t03, t04; +End of 5.0 tests diff --git a/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.test b/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.test new file mode 100644 index 0000000000000..3af0015a486aa --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_mysqlbinlog-cp932.test @@ -0,0 +1,2 @@ +--let $use_remote_mysqlbinlog= 1 +--source extra/binlog_tests/binlog_mysqlbinlog-cp932.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_row_annotate-master.opt b/mysql-test/suite/binlog_encryption/binlog_row_annotate-master.opt new file mode 100644 index 0000000000000..344a4ffc01475 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_row_annotate-master.opt @@ -0,0 +1 @@ +--timezone=GMT-3 --binlog-do-db=test1 --binlog-do-db=test2 --binlog-do-db=test3 --binlog-checksum=NONE diff --git a/mysql-test/suite/binlog_encryption/binlog_row_annotate.result b/mysql-test/suite/binlog_encryption/binlog_row_annotate.result new file mode 100644 index 0000000000000..d32b80b12497f --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_row_annotate.result @@ -0,0 +1,724 @@ +##################################################################################### +# The following Annotate_rows events should appear below: +# - INSERT INTO test2.t2 VALUES (1), (2), (3) +# - INSERT INTO test3.t3 VALUES (1), (2), (3) +# - DELETE test1.t1, test2.t2 FROM <...> +# - INSERT INTO test2.t2 VALUES (1), (2), (3) +# - DELETE xtest1.xt1, test2.t2 FROM <...> +##################################################################################### +show binlog events in 'master-bin.000001' from ; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Start_encryption 1 # +master-bin.000001 # Gtid_list 1 # [] +master-bin.000001 # Binlog_checkpoint 1 # master-bin.000001 +master-bin.000001 # Gtid 1 # GTID 0-1-1 +master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test1 +master-bin.000001 # Gtid 1 # GTID 0-1-2 +master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test2 +master-bin.000001 # Gtid 1 # GTID 0-1-3 +master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test3 +master-bin.000001 # Gtid 1 # GTID 0-1-4 +master-bin.000001 # Query 1 # CREATE DATABASE test1 +master-bin.000001 # Gtid 1 # GTID 0-1-5 +master-bin.000001 # Query 1 # CREATE DATABASE test2 +master-bin.000001 # Gtid 1 # GTID 0-1-6 +master-bin.000001 # Query 1 # CREATE DATABASE test3 +master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-7 +master-bin.000001 # Table_map 1 # table_id: # (test1.t1) +master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F +master-bin.000001 # Query 1 # COMMIT +master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-8 +master-bin.000001 # Annotate_rows 1 # INSERT INTO test2.t2 VALUES (1), (2), (3) +master-bin.000001 # Table_map 1 # table_id: # (test2.t2) +master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F +master-bin.000001 # Query 1 # COMMIT +master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-9 +master-bin.000001 # Annotate_rows 1 # INSERT INTO test3.t3 VALUES (1), (2), (3) +master-bin.000001 # Table_map 1 # table_id: # (test3.t3) +master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F +master-bin.000001 # Query 1 # COMMIT +master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-10 +master-bin.000001 # Annotate_rows 1 # DELETE test1.t1, test2.t2 +FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 +WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a +master-bin.000001 # Table_map 1 # table_id: # (test1.t1) +master-bin.000001 # Table_map 1 # table_id: # (test2.t2) +master-bin.000001 # Delete_rows_v1 1 # table_id: # +master-bin.000001 # Delete_rows_v1 1 # table_id: # flags: STMT_END_F +master-bin.000001 # Query 1 # COMMIT +master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-11 +master-bin.000001 # Annotate_rows 1 # INSERT INTO test2.v2 VALUES (1), (2), (3) +master-bin.000001 # Table_map 1 # table_id: # (test2.t2) +master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F +master-bin.000001 # Query 1 # COMMIT +master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-12 +master-bin.000001 # Annotate_rows 1 # DELETE xtest1.xt1, test2.t2 +FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 +WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a +master-bin.000001 # Table_map 1 # table_id: # (test2.t2) +master-bin.000001 # Delete_rows_v1 1 # table_id: # flags: STMT_END_F +master-bin.000001 # Query 1 # COMMIT +master-bin.000001 # Rotate 1 # master-bin.000002;pos=4 +# +##################################################################################### +# mysqlbinlog --read-from-remote-server +# The following Annotates should appear in this output: +# - INSERT INTO test2.t2 VALUES (1), (2), (3) +# - INSERT INTO test3.t3 VALUES (1), (2), (3) +# - DELETE test1.t1, test2.t2 FROM <...> (with two subsequent Table maps) +# - INSERT INTO test2.t2 VALUES (1), (2), (3) +# - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) +##################################################################################### +/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; +/*!40019 SET @@session.max_insert_delayed_threads=0*/; +/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; +DELIMITER /*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Start: binlog v 4, server v #.##.## created 010909 4:46:40 at startup +ROLLBACK/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Gtid list [] +# at # +#010909 4:46:40 server id # end_log_pos # Binlog checkpoint master-bin.000001 +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-1 ddl +/*!100101 SET @@session.skip_parallel_replication=0*//*!*/; +/*!100001 SET @@session.gtid_domain_id=0*//*!*/; +/*!100001 SET @@session.server_id=1*//*!*/; +/*!100001 SET @@session.gtid_seq_no=1*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +SET @@session.pseudo_thread_id=#/*!*/; +SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; +SET @@session.sql_mode=0/*!*/; +SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; +/*!\C latin1 *//*!*/; +SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; +SET @@session.lc_time_names=0/*!*/; +SET @@session.collation_database=DEFAULT/*!*/; +DROP DATABASE IF EXISTS test1 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl +/*!100001 SET @@session.gtid_seq_no=2*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +DROP DATABASE IF EXISTS test2 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl +/*!100001 SET @@session.gtid_seq_no=3*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +DROP DATABASE IF EXISTS test3 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test1 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test2 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test3 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test1`.`t1` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test1`.`t1` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test1`.`t1` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test1`.`t1` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +BEGIN +/*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Annotate_rows: +#Q> INSERT INTO test2.t2 VALUES (1), (2), (3) +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test2`.`t2` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +BEGIN +/*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Annotate_rows: +#Q> INSERT INTO test3.t3 VALUES (1), (2), (3) +#010909 4:46:40 server id # end_log_pos # Table_map: `test3`.`t3` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test3`.`t3` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test3`.`t3` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test3`.`t3` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 +/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +BEGIN +/*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Annotate_rows: +#Q> DELETE test1.t1, test2.t2 +#Q> FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 +#Q> WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3 +#010909 4:46:40 server id # end_log_pos # Table_map: `test1`.`t1` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # flags: STMT_END_F +### DELETE FROM `test1`.`t1` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test1`.`t1` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test1`.`t1` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 +/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +BEGIN +/*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Annotate_rows: +#Q> INSERT INTO test2.v2 VALUES (1), (2), (3) +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test2`.`t2` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 +/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +BEGIN +/*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Annotate_rows: +#Q> DELETE xtest1.xt1, test2.t2 +#Q> FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 +#Q> WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3 +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # flags: STMT_END_F +### DELETE FROM `test2`.`t2` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Rotate to master-bin.000002 pos: 4 +DELIMITER ; +# End of log file +ROLLBACK /* added by mysqlbinlog */; +/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; +/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; +# +##################################################################################### +# mysqlbinlog --read-from-remote-server --database=test1 +# The following Annotate should appear in this output: +# - DELETE test1.t1, test2.t2 FROM <...> +##################################################################################### +/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; +/*!40019 SET @@session.max_insert_delayed_threads=0*/; +/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; +DELIMITER /*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Start: binlog v 4, server v #.##.## created 010909 4:46:40 at startup +ROLLBACK/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Gtid list [] +# at # +#010909 4:46:40 server id # end_log_pos # Binlog checkpoint master-bin.000001 +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-1 ddl +/*!100101 SET @@session.skip_parallel_replication=0*//*!*/; +/*!100001 SET @@session.gtid_domain_id=0*//*!*/; +/*!100001 SET @@session.server_id=1*//*!*/; +/*!100001 SET @@session.gtid_seq_no=1*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +SET @@session.pseudo_thread_id=#/*!*/; +SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; +SET @@session.sql_mode=0/*!*/; +SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; +/*!\C latin1 *//*!*/; +SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; +SET @@session.lc_time_names=0/*!*/; +SET @@session.collation_database=DEFAULT/*!*/; +DROP DATABASE IF EXISTS test1 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl +/*!100001 SET @@session.gtid_seq_no=2*//*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl +/*!100001 SET @@session.gtid_seq_no=3*//*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test1 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test1`.`t1` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test1`.`t1` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test1`.`t1` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test1`.`t1` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +BEGIN +/*!*/; +# at # +# at # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +BEGIN +/*!*/; +# at # +# at # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 +/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +BEGIN +/*!*/; +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Annotate_rows: +#Q> DELETE test1.t1, test2.t2 +#Q> FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 +#Q> WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3 +#010909 4:46:40 server id # end_log_pos # Table_map: `test1`.`t1` mapped to number # +# at # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # +### DELETE FROM `test1`.`t1` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test1`.`t1` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test1`.`t1` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +'/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 +/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +BEGIN +/*!*/; +# at # +# at # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 +/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +BEGIN +/*!*/; +# at # +# at # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Rotate to master-bin.000002 pos: 4 +DELIMITER ; +# End of log file +ROLLBACK /* added by mysqlbinlog */; +/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; +/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; +# +##################################################################################### +# mysqlbinlog --read-from-remote-server --skip-annotate-row-events +# No Annotates should appear in this output +##################################################################################### +/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; +/*!40019 SET @@session.max_insert_delayed_threads=0*/; +/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; +DELIMITER /*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Start: binlog v 4, server v #.##.## created 010909 4:46:40 at startup +ROLLBACK/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Gtid list [] +# at # +#010909 4:46:40 server id # end_log_pos # Binlog checkpoint master-bin.000001 +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-1 ddl +/*!100101 SET @@session.skip_parallel_replication=0*//*!*/; +/*!100001 SET @@session.gtid_domain_id=0*//*!*/; +/*!100001 SET @@session.server_id=1*//*!*/; +/*!100001 SET @@session.gtid_seq_no=1*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +SET @@session.pseudo_thread_id=#/*!*/; +SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; +SET @@session.sql_mode=0/*!*/; +SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; +/*!\C latin1 *//*!*/; +SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; +SET @@session.lc_time_names=0/*!*/; +SET @@session.collation_database=DEFAULT/*!*/; +DROP DATABASE IF EXISTS test1 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl +/*!100001 SET @@session.gtid_seq_no=2*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +DROP DATABASE IF EXISTS test2 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl +/*!100001 SET @@session.gtid_seq_no=3*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +DROP DATABASE IF EXISTS test3 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test1 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test2 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +CREATE DATABASE test3 +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test1`.`t1` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test1`.`t1` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test1`.`t1` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test1`.`t1` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test2`.`t2` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test3`.`t3` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test3`.`t3` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test3`.`t3` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test3`.`t3` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 +/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test1`.`t1` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +# at # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # flags: STMT_END_F +### DELETE FROM `test1`.`t1` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test1`.`t1` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test1`.`t1` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 +/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Write_rows: table id # flags: STMT_END_F +### INSERT INTO `test2`.`t2` +### SET +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### INSERT INTO `test2`.`t2` +### SET +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 +/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +BEGIN +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Table_map: `test2`.`t2` mapped to number # +# at # +#010909 4:46:40 server id # end_log_pos # Delete_rows: table id # flags: STMT_END_F +### DELETE FROM `test2`.`t2` +### WHERE +### @1=3 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=2 /* INT meta=0 nullable=1 is_null=0 */ +### DELETE FROM `test2`.`t2` +### WHERE +### @1=1 /* INT meta=0 nullable=1 is_null=0 */ +# at # +#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 +SET TIMESTAMP=1000000000/*!*/; +COMMIT +/*!*/; +# at # +#010909 4:46:40 server id # end_log_pos # Rotate to master-bin.000002 pos: 4 +DELIMITER ; +# End of log file +ROLLBACK /* added by mysqlbinlog */; +/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; +/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; diff --git a/mysql-test/suite/binlog_encryption/binlog_row_annotate.test b/mysql-test/suite/binlog_encryption/binlog_row_annotate.test new file mode 100644 index 0000000000000..40aa0dbc6e35b --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_row_annotate.test @@ -0,0 +1,2 @@ +--let $use_remote_mysqlbinlog= 1 +--source extra/binlog_tests/binlog_row_annotate.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_write_error.result b/mysql-test/suite/binlog_encryption/binlog_write_error.result new file mode 100644 index 0000000000000..28cffb3a8e552 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_write_error.result @@ -0,0 +1,108 @@ +# +# Initialization +# +DROP TABLE IF EXISTS t1, t2; +DROP FUNCTION IF EXISTS f1; +DROP FUNCTION IF EXISTS f2; +DROP PROCEDURE IF EXISTS p1; +DROP PROCEDURE IF EXISTS p2; +DROP TRIGGER IF EXISTS tr1; +DROP TRIGGER IF EXISTS tr2; +DROP VIEW IF EXISTS v1, v2; +# +# Test injecting binlog write error when executing queries +# +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +CREATE TABLE t1 (a INT); +CREATE TABLE t1 (a INT); +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +INSERT INTO t1 VALUES (1),(2),(3); +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +INSERT INTO t1 VALUES (4),(5),(6); +INSERT INTO t1 VALUES (4),(5),(6); +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +UPDATE t1 set a=a+1; +UPDATE t1 set a=a+1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DELETE FROM t1; +DELETE FROM t1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +CREATE TRIGGER tr1 AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t1 VALUES (new.a + 100); +CREATE TRIGGER tr1 AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t1 VALUES (new.a + 100); +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DROP TRIGGER tr1; +DROP TRIGGER tr1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +ALTER TABLE t1 ADD (b INT); +ALTER TABLE t1 ADD (b INT); +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +CREATE VIEW v1 AS SELECT a FROM t1; +CREATE VIEW v1 AS SELECT a FROM t1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DROP VIEW v1; +DROP VIEW v1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +CREATE PROCEDURE p1(OUT rows INT) SELECT count(*) INTO rows FROM t1; +CREATE PROCEDURE p1(OUT rows INT) SELECT count(*) INTO rows FROM t1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DROP PROCEDURE p1; +DROP PROCEDURE p1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DROP TABLE t1; +DROP TABLE t1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +CREATE FUNCTION f1() RETURNS INT return 1; +CREATE FUNCTION f1() RETURNS INT return 1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DROP FUNCTION f1; +DROP FUNCTION f1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +CREATE USER user1; +CREATE USER user1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +SET GLOBAL debug_dbug='d,injecting_fault_writing'; +DROP USER user1; +DROP USER user1; +ERROR HY000: Error writing file 'master-bin' ((errno: #) +SET GLOBAL debug_dbug=''; +# +# Cleanup +# +DROP TABLE IF EXISTS t1, t2; +DROP FUNCTION IF EXISTS f1; +DROP PROCEDURE IF EXISTS p1; +DROP TRIGGER IF EXISTS tr1; +DROP VIEW IF EXISTS v1, v2; diff --git a/mysql-test/suite/binlog_encryption/binlog_write_error.test b/mysql-test/suite/binlog_encryption/binlog_write_error.test new file mode 100644 index 0000000000000..05f8eff6c3a38 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_write_error.test @@ -0,0 +1 @@ +--source extra/binlog_tests/binlog_write_error.inc diff --git a/mysql-test/suite/binlog_encryption/binlog_xa_recover-master.opt b/mysql-test/suite/binlog_encryption/binlog_xa_recover-master.opt new file mode 100644 index 0000000000000..3c44f9fad10ae --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_xa_recover-master.opt @@ -0,0 +1 @@ +--skip-stack-trace --skip-core-file --loose-debug-dbug=+d,xa_recover_expect_master_bin_000004 diff --git a/mysql-test/suite/binlog_encryption/binlog_xa_recover.result b/mysql-test/suite/binlog_encryption/binlog_xa_recover.result new file mode 100644 index 0000000000000..b78fd47dae6f6 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_xa_recover.result @@ -0,0 +1,216 @@ +SET GLOBAL max_binlog_size= 4096; +SET GLOBAL innodb_flush_log_at_trx_commit= 1; +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb; +INSERT INTO t1 VALUES (100, REPEAT("x", 4100)); +INSERT INTO t1 VALUES (101, REPEAT("x", 4100)); +INSERT INTO t1 VALUES (102, REPEAT("x", 4100)); +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con1_wait WAIT_FOR con1_cont"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con1_ready WAIT_FOR _ever"; +INSERT INTO t1 VALUES (1, REPEAT("x", 4100)); +SET DEBUG_SYNC= "now WAIT_FOR con1_wait"; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con2_wait WAIT_FOR con2_cont"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con2_ready WAIT_FOR _ever"; +INSERT INTO t1 VALUES (2, NULL); +SET DEBUG_SYNC= "now WAIT_FOR con2_wait"; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con3_wait WAIT_FOR con3_cont"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con3_ready WAIT_FOR _ever"; +INSERT INTO t1 VALUES (3, REPEAT("x", 4100)); +SET DEBUG_SYNC= "now WAIT_FOR con3_wait"; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL con4_wait WAIT_FOR con4_cont"; +SET SESSION debug_dbug="+d,crash_commit_after_log"; +INSERT INTO t1 VALUES (4, NULL); +SET DEBUG_SYNC= "now WAIT_FOR con4_wait"; +SET DEBUG_SYNC= "now SIGNAL con1_cont"; +SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; +SET DEBUG_SYNC= "now SIGNAL con2_cont"; +SET DEBUG_SYNC= "now WAIT_FOR con2_ready"; +SET DEBUG_SYNC= "now SIGNAL con3_cont"; +SET DEBUG_SYNC= "now WAIT_FOR con3_ready"; +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +master-bin.000003 # +master-bin.000004 # +master-bin.000005 # +master-bin.000006 # +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000003 # Start_encryption # # +master-bin.000003 # Gtid_list # # [#-#-#] +master-bin.000003 # Binlog_checkpoint # # master-bin.000002 +master-bin.000003 # Binlog_checkpoint # # master-bin.000003 +master-bin.000003 # Gtid # # BEGIN GTID #-#-# +master-bin.000003 # Table_map # # table_id: # (test.t1) +master-bin.000003 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000003 # Xid # # COMMIT /* XID */ +master-bin.000003 # Rotate # # master-bin.000004;pos=POS +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000004 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000004 # Start_encryption # # +master-bin.000004 # Gtid_list # # [#-#-#] +master-bin.000004 # Binlog_checkpoint # # master-bin.000003 +master-bin.000004 # Binlog_checkpoint # # master-bin.000004 +master-bin.000004 # Gtid # # BEGIN GTID #-#-# +master-bin.000004 # Table_map # # table_id: # (test.t1) +master-bin.000004 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000004 # Xid # # COMMIT /* XID */ +master-bin.000004 # Rotate # # master-bin.000005;pos=POS +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000005 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000005 # Start_encryption # # +master-bin.000005 # Gtid_list # # [#-#-#] +master-bin.000005 # Binlog_checkpoint # # master-bin.000004 +master-bin.000005 # Gtid # # BEGIN GTID #-#-# +master-bin.000005 # Table_map # # table_id: # (test.t1) +master-bin.000005 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000005 # Xid # # COMMIT /* XID */ +master-bin.000005 # Gtid # # BEGIN GTID #-#-# +master-bin.000005 # Table_map # # table_id: # (test.t1) +master-bin.000005 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000005 # Xid # # COMMIT /* XID */ +master-bin.000005 # Rotate # # master-bin.000006;pos=POS +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000006 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000006 # Start_encryption # # +master-bin.000006 # Gtid_list # # [#-#-#] +master-bin.000006 # Binlog_checkpoint # # master-bin.000004 +PURGE BINARY LOGS TO "master-bin.000006"; +show binary logs; +Log_name File_size +master-bin.000004 # +master-bin.000005 # +master-bin.000006 # +SET DEBUG_SYNC= "now SIGNAL con4_cont"; +Got one of the listed errors +SELECT a FROM t1 ORDER BY a; +a +1 +2 +3 +4 +100 +101 +102 +Test that with multiple binlog checkpoints, recovery starts from the last one. +SET GLOBAL max_binlog_size= 4096; +SET GLOBAL innodb_flush_log_at_trx_commit= 1; +RESET MASTER; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con10_ready WAIT_FOR con10_cont"; +INSERT INTO t1 VALUES (10, REPEAT("x", 4100)); +SET DEBUG_SYNC= "now WAIT_FOR con10_ready"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con11_ready WAIT_FOR con11_cont"; +INSERT INTO t1 VALUES (11, REPEAT("x", 4100)); +SET DEBUG_SYNC= "now WAIT_FOR con11_ready"; +SET DEBUG_SYNC= "commit_after_group_release_commit_ordered SIGNAL con12_ready WAIT_FOR con12_cont"; +INSERT INTO t1 VALUES (12, REPEAT("x", 4100)); +SET DEBUG_SYNC= "now WAIT_FOR con12_ready"; +INSERT INTO t1 VALUES (13, NULL); +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +master-bin.000003 # +master-bin.000004 # +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000004 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000004 # Start_encryption # # +master-bin.000004 # Gtid_list # # [#-#-#] +master-bin.000004 # Binlog_checkpoint # # master-bin.000001 +master-bin.000004 # Gtid # # BEGIN GTID #-#-# +master-bin.000004 # Table_map # # table_id: # (test.t1) +master-bin.000004 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000004 # Xid # # COMMIT /* XID */ +SET DEBUG_SYNC= "now SIGNAL con10_cont"; +SET @old_dbug= @@global.DEBUG_DBUG; +SET GLOBAL debug_dbug="+d,binlog_background_checkpoint_processed"; +SET DEBUG_SYNC= "now SIGNAL con12_cont"; +SET DEBUG_SYNC= "now WAIT_FOR binlog_background_checkpoint_processed"; +SET GLOBAL debug_dbug= @old_dbug; +SET DEBUG_SYNC= "now SIGNAL con11_cont"; +Checking that master-bin.000004 is the last binlog checkpoint +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000004 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000004 # Start_encryption # # +master-bin.000004 # Gtid_list # # [#-#-#] +master-bin.000004 # Binlog_checkpoint # # master-bin.000001 +master-bin.000004 # Gtid # # BEGIN GTID #-#-# +master-bin.000004 # Table_map # # table_id: # (test.t1) +master-bin.000004 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000004 # Xid # # COMMIT /* XID */ +master-bin.000004 # Binlog_checkpoint # # master-bin.000002 +master-bin.000004 # Binlog_checkpoint # # master-bin.000004 +Now crash the server +SET SESSION debug_dbug="+d,crash_commit_after_log"; +INSERT INTO t1 VALUES (14, NULL); +Got one of the listed errors +SELECT a FROM t1 ORDER BY a; +a +1 +2 +3 +4 +10 +11 +12 +13 +14 +100 +101 +102 +*** Check that recovery works if we crashed early during rotate, before +*** binlog checkpoint event could be written. +SET GLOBAL max_binlog_size= 4096; +SET GLOBAL innodb_flush_log_at_trx_commit= 1; +RESET MASTER; +INSERT INTO t1 VALUES (21, REPEAT("x", 4100)); +INSERT INTO t1 VALUES (22, REPEAT("x", 4100)); +INSERT INTO t1 VALUES (23, REPEAT("x", 4100)); +SET SESSION debug_dbug="+d,crash_before_write_checkpoint_event"; +INSERT INTO t1 VALUES (24, REPEAT("x", 4100)); +Got one of the listed errors +SELECT a FROM t1 ORDER BY a; +a +1 +2 +3 +4 +10 +11 +12 +13 +14 +21 +22 +23 +24 +100 +101 +102 +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +master-bin.000003 # +master-bin.000004 # +master-bin.000005 # +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000004 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000004 # Start_encryption # # +master-bin.000004 # Gtid_list # # [#-#-#] +master-bin.000004 # Binlog_checkpoint # # master-bin.000003 +master-bin.000004 # Binlog_checkpoint # # master-bin.000004 +master-bin.000004 # Gtid # # BEGIN GTID #-#-# +master-bin.000004 # Table_map # # table_id: # (test.t1) +master-bin.000004 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000004 # Xid # # COMMIT /* XID */ +master-bin.000004 # Rotate # # master-bin.000005;pos=POS +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_encryption/binlog_xa_recover.test b/mysql-test/suite/binlog_encryption/binlog_xa_recover.test new file mode 100644 index 0000000000000..0e0b80433ff43 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/binlog_xa_recover.test @@ -0,0 +1 @@ +--source extra/binlog_tests/binlog_xa_recover.inc diff --git a/mysql-test/suite/binlog_encryption/disabled.def b/mysql-test/suite/binlog_encryption/disabled.def new file mode 100644 index 0000000000000..4fe25e7112586 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/disabled.def @@ -0,0 +1,4 @@ +encrypted_master_switch_to_unencrypted : MDEV-11288 - server crash +binlog_incident : MDEV-11319 - mysqlbinlog crash or failure +encrypted_master_lost_key : MDEV-11323 - unspecified behavior for IO thread +rpl_checksum_cache : MDEV-11486 - sporadic failure in IO thread diff --git a/mysql-test/suite/binlog_encryption/encrypted_master.result b/mysql-test/suite/binlog_encryption/encrypted_master.result new file mode 100644 index 0000000000000..a61b56cf35538 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master.result @@ -0,0 +1,622 @@ +################# +# Initialization +################# +include/rpl_init.inc [topology=1->2] +connection server_2; +include/stop_slave_sql.inc +connection server_1; +SET @binlog_annotate_row_events.save= @@global.binlog_annotate_row_events; +SET @binlog_checksum.save= @@global.binlog_checksum; +SET @master_verify_checksum.save= @@global.master_verify_checksum; +SET @binlog_row_image.save= @@global.binlog_row_image; +#################################################### +# Test 1: simple binlog, no checksum, no annotation +#################################################### +connection server_1; +SET binlog_annotate_row_events= 0; +SET GLOBAL binlog_annotate_row_events= 0; +SET GLOBAL binlog_checksum= NONE; +SET GLOBAL master_verify_checksum= 0; +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CREATE DATABASE database_name_to_encrypt; +USE database_name_to_encrypt; +CREATE USER user_name_to_encrypt; +GRANT ALL ON database_name_to_encrypt.* TO user_name_to_encrypt; +SET PASSWORD FOR user_name_to_encrypt = PASSWORD('password_to_encrypt'); +CREATE TABLE innodb_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB, +virt_column_name_to_encrypt INT AS (int_column_name_to_encrypt % 10) VIRTUAL, +pers_column_name_to_encrypt INT AS (int_column_name_to_encrypt) PERSISTENT, +INDEX `index_name_to_encrypt`(`timestamp_column_name_to_encrypt`) +) ENGINE=InnoDB +PARTITION BY RANGE (int_column_name_to_encrypt) +SUBPARTITION BY KEY (int_column_name_to_encrypt) +SUBPARTITIONS 2 ( +PARTITION partition0_name_to_encrypt VALUES LESS THAN (100), +PARTITION partition1_name_to_encrypt VALUES LESS THAN (MAXVALUE) +) +; +CREATE TABLE myisam_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +char_column_name_to_encrypt VARCHAR(255), +datetime_column_name_to_encrypt DATETIME, +text_column_name_to_encrypt TEXT +) ENGINE=MyISAM; +CREATE TABLE aria_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +varchar_column_name_to_encrypt VARCHAR(1024), +enum_column_name_to_encrypt ENUM( +'enum_value1_to_encrypt', +'enum_value2_to_encrypt' + ), +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB +) ENGINE=Aria; +CREATE TRIGGER trigger_name_to_encrypt +AFTER INSERT ON myisam_table_name_to_encrypt FOR EACH ROW +INSERT INTO aria_table_name_to_encrypt (varchar_column_name_to_encrypt) +VALUES (NEW.char_column_name_to_encrypt); +CREATE DEFINER=user_name_to_encrypt VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +CREATE FUNCTION func_name_to_encrypt (func_parameter_to_encrypt INT) +RETURNS VARCHAR(64) +RETURN 'func_result_to_encrypt'; +CREATE PROCEDURE proc_name_to_encrypt ( +IN proc_in_parameter_to_encrypt CHAR(32), +OUT proc_out_parameter_to_encrypt INT +) +BEGIN +DECLARE procvar_name_to_encrypt CHAR(64) DEFAULT 'procvar_val_to_encrypt'; +DECLARE cursor_name_to_encrypt CURSOR FOR +SELECT virt_column_name_to_encrypt FROM innodb_table_name_to_encrypt; +DECLARE EXIT HANDLER FOR NOT FOUND +BEGIN +SET @stmt_var_to_encrypt = CONCAT( +"SELECT + IF (RAND()>0.5,'enum_value2_to_encrypt','enum_value1_to_encrypt') + FROM innodb_table_name_to_encrypt + INTO OUTFILE '", proc_in_parameter_to_encrypt, "'"); +PREPARE stmt_to_encrypt FROM @stmt_var_to_encrypt; +EXECUTE stmt_to_encrypt; +DEALLOCATE PREPARE stmt_to_encrypt; +END; +OPEN cursor_name_to_encrypt; +proc_label_to_encrypt: LOOP +FETCH cursor_name_to_encrypt INTO procvar_name_to_encrypt; +END LOOP; +CLOSE cursor_name_to_encrypt; +END $$ +CREATE SERVER server_name_to_encrypt +FOREIGN DATA WRAPPER mysql +OPTIONS (HOST 'host_name_to_encrypt'); +connect con1,localhost,user_name_to_encrypt,password_to_encrypt,database_name_to_encrypt; +CREATE TEMPORARY TABLE tmp_table_name_to_encrypt ( +float_column_name_to_encrypt FLOAT, +binary_column_name_to_encrypt BINARY(64) +); +disconnect con1; +connection server_1; +CREATE INDEX index_name_to_encrypt +ON myisam_table_name_to_encrypt (datetime_column_name_to_encrypt); +ALTER DATABASE database_name_to_encrypt CHARACTER SET utf8; +ALTER TABLE innodb_table_name_to_encrypt +MODIFY timestamp_column_name_to_encrypt TIMESTAMP NOT NULL +DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +; +ALTER ALGORITHM=MERGE VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +RENAME TABLE innodb_table_name_to_encrypt TO new_table_name_to_encrypt; +ALTER TABLE new_table_name_to_encrypt RENAME TO innodb_table_name_to_encrypt; +set @user_var1_to_encrypt= 'dyncol1_val_to_encrypt'; +set @user_var2_to_encrypt= 'dyncol2_name_to_encrypt'; +INSERT INTO view_name_to_encrypt VALUES +(1, NOW(6), COLUMN_CREATE('dyncol1_name_to_encrypt',@user_var1_to_encrypt), NULL, NULL), +(2, NOW(6), COLUMN_CREATE(@user_var2_to_encrypt,'dyncol2_val_to_encrypt'), NULL, NULL) +; +BEGIN NOT ATOMIC +DECLARE counter_name_to_encrypt INT DEFAULT 0; +START TRANSACTION; +WHILE counter_name_to_encrypt<12 DO +INSERT INTO innodb_table_name_to_encrypt +SELECT NULL, NOW(6), blob_column_name_to_encrypt, NULL, NULL +FROM innodb_table_name_to_encrypt +ORDER BY int_column_name_to_encrypt; +SET counter_name_to_encrypt = counter_name_to_encrypt+1; +END WHILE; +COMMIT; +END +$$ +INSERT INTO myisam_table_name_to_encrypt +SELECT NULL, 'char_literal_to_encrypt', NULL, 'text_to_encrypt'; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +CALL proc_name_to_encrypt('file_name_to_encrypt',@useless_var_to_encrypt); +TRUNCATE TABLE aria_table_name_to_encrypt; +LOAD DATA INFILE 'file_name_to_encrypt' INTO TABLE aria_table_name_to_encrypt +(enum_column_name_to_encrypt); +LOAD DATA LOCAL INFILE '/database_name_to_encrypt/file_name_to_encrypt' +INTO TABLE aria_table_name_to_encrypt (enum_column_name_to_encrypt); +UPDATE view_name_to_encrypt SET blob_column_name_to_encrypt = +COLUMN_CREATE('dyncol1_name_to_encrypt',func_name_to_encrypt(0)) +; +DELETE FROM aria_table_name_to_encrypt ORDER BY int_column_name_to_encrypt LIMIT 10; +ANALYZE TABLE myisam_table_name_to_encrypt; +CHECK TABLE aria_table_name_to_encrypt; +CHECKSUM TABLE innodb_table_name_to_encrypt, myisam_table_name_to_encrypt; +RENAME USER user_name_to_encrypt to new_user_name_to_encrypt; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM new_user_name_to_encrypt; +DROP DATABASE database_name_to_encrypt; +DROP USER new_user_name_to_encrypt; +DROP SERVER server_name_to_encrypt; +#################################################### +# Test 2: binlog with checksum, no annotated events +#################################################### +connection server_1; +SET binlog_annotate_row_events= 0; +SET GLOBAL binlog_annotate_row_events= 0; +SET GLOBAL binlog_checksum= CRC32; +SET GLOBAL master_verify_checksum= 1; +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CREATE DATABASE database_name_to_encrypt; +USE database_name_to_encrypt; +CREATE USER user_name_to_encrypt; +GRANT ALL ON database_name_to_encrypt.* TO user_name_to_encrypt; +SET PASSWORD FOR user_name_to_encrypt = PASSWORD('password_to_encrypt'); +CREATE TABLE innodb_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB, +virt_column_name_to_encrypt INT AS (int_column_name_to_encrypt % 10) VIRTUAL, +pers_column_name_to_encrypt INT AS (int_column_name_to_encrypt) PERSISTENT, +INDEX `index_name_to_encrypt`(`timestamp_column_name_to_encrypt`) +) ENGINE=InnoDB +PARTITION BY RANGE (int_column_name_to_encrypt) +SUBPARTITION BY KEY (int_column_name_to_encrypt) +SUBPARTITIONS 2 ( +PARTITION partition0_name_to_encrypt VALUES LESS THAN (100), +PARTITION partition1_name_to_encrypt VALUES LESS THAN (MAXVALUE) +) +; +CREATE TABLE myisam_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +char_column_name_to_encrypt VARCHAR(255), +datetime_column_name_to_encrypt DATETIME, +text_column_name_to_encrypt TEXT +) ENGINE=MyISAM; +CREATE TABLE aria_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +varchar_column_name_to_encrypt VARCHAR(1024), +enum_column_name_to_encrypt ENUM( +'enum_value1_to_encrypt', +'enum_value2_to_encrypt' + ), +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB +) ENGINE=Aria; +CREATE TRIGGER trigger_name_to_encrypt +AFTER INSERT ON myisam_table_name_to_encrypt FOR EACH ROW +INSERT INTO aria_table_name_to_encrypt (varchar_column_name_to_encrypt) +VALUES (NEW.char_column_name_to_encrypt); +CREATE DEFINER=user_name_to_encrypt VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +CREATE FUNCTION func_name_to_encrypt (func_parameter_to_encrypt INT) +RETURNS VARCHAR(64) +RETURN 'func_result_to_encrypt'; +CREATE PROCEDURE proc_name_to_encrypt ( +IN proc_in_parameter_to_encrypt CHAR(32), +OUT proc_out_parameter_to_encrypt INT +) +BEGIN +DECLARE procvar_name_to_encrypt CHAR(64) DEFAULT 'procvar_val_to_encrypt'; +DECLARE cursor_name_to_encrypt CURSOR FOR +SELECT virt_column_name_to_encrypt FROM innodb_table_name_to_encrypt; +DECLARE EXIT HANDLER FOR NOT FOUND +BEGIN +SET @stmt_var_to_encrypt = CONCAT( +"SELECT + IF (RAND()>0.5,'enum_value2_to_encrypt','enum_value1_to_encrypt') + FROM innodb_table_name_to_encrypt + INTO OUTFILE '", proc_in_parameter_to_encrypt, "'"); +PREPARE stmt_to_encrypt FROM @stmt_var_to_encrypt; +EXECUTE stmt_to_encrypt; +DEALLOCATE PREPARE stmt_to_encrypt; +END; +OPEN cursor_name_to_encrypt; +proc_label_to_encrypt: LOOP +FETCH cursor_name_to_encrypt INTO procvar_name_to_encrypt; +END LOOP; +CLOSE cursor_name_to_encrypt; +END $$ +CREATE SERVER server_name_to_encrypt +FOREIGN DATA WRAPPER mysql +OPTIONS (HOST 'host_name_to_encrypt'); +connect con1,localhost,user_name_to_encrypt,password_to_encrypt,database_name_to_encrypt; +CREATE TEMPORARY TABLE tmp_table_name_to_encrypt ( +float_column_name_to_encrypt FLOAT, +binary_column_name_to_encrypt BINARY(64) +); +disconnect con1; +connection server_1; +CREATE INDEX index_name_to_encrypt +ON myisam_table_name_to_encrypt (datetime_column_name_to_encrypt); +ALTER DATABASE database_name_to_encrypt CHARACTER SET utf8; +ALTER TABLE innodb_table_name_to_encrypt +MODIFY timestamp_column_name_to_encrypt TIMESTAMP NOT NULL +DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +; +ALTER ALGORITHM=MERGE VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +RENAME TABLE innodb_table_name_to_encrypt TO new_table_name_to_encrypt; +ALTER TABLE new_table_name_to_encrypt RENAME TO innodb_table_name_to_encrypt; +set @user_var1_to_encrypt= 'dyncol1_val_to_encrypt'; +set @user_var2_to_encrypt= 'dyncol2_name_to_encrypt'; +INSERT INTO view_name_to_encrypt VALUES +(1, NOW(6), COLUMN_CREATE('dyncol1_name_to_encrypt',@user_var1_to_encrypt), NULL, NULL), +(2, NOW(6), COLUMN_CREATE(@user_var2_to_encrypt,'dyncol2_val_to_encrypt'), NULL, NULL) +; +BEGIN NOT ATOMIC +DECLARE counter_name_to_encrypt INT DEFAULT 0; +START TRANSACTION; +WHILE counter_name_to_encrypt<12 DO +INSERT INTO innodb_table_name_to_encrypt +SELECT NULL, NOW(6), blob_column_name_to_encrypt, NULL, NULL +FROM innodb_table_name_to_encrypt +ORDER BY int_column_name_to_encrypt; +SET counter_name_to_encrypt = counter_name_to_encrypt+1; +END WHILE; +COMMIT; +END +$$ +INSERT INTO myisam_table_name_to_encrypt +SELECT NULL, 'char_literal_to_encrypt', NULL, 'text_to_encrypt'; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +CALL proc_name_to_encrypt('file_name_to_encrypt',@useless_var_to_encrypt); +TRUNCATE TABLE aria_table_name_to_encrypt; +LOAD DATA INFILE 'file_name_to_encrypt' INTO TABLE aria_table_name_to_encrypt +(enum_column_name_to_encrypt); +LOAD DATA LOCAL INFILE '/database_name_to_encrypt/file_name_to_encrypt' +INTO TABLE aria_table_name_to_encrypt (enum_column_name_to_encrypt); +UPDATE view_name_to_encrypt SET blob_column_name_to_encrypt = +COLUMN_CREATE('dyncol1_name_to_encrypt',func_name_to_encrypt(0)) +; +DELETE FROM aria_table_name_to_encrypt ORDER BY int_column_name_to_encrypt LIMIT 10; +ANALYZE TABLE myisam_table_name_to_encrypt; +CHECK TABLE aria_table_name_to_encrypt; +CHECKSUM TABLE innodb_table_name_to_encrypt, myisam_table_name_to_encrypt; +RENAME USER user_name_to_encrypt to new_user_name_to_encrypt; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM new_user_name_to_encrypt; +DROP DATABASE database_name_to_encrypt; +DROP USER new_user_name_to_encrypt; +DROP SERVER server_name_to_encrypt; +#################################################### +# Test 3: binlog with checksum and annotated events +#################################################### +connection server_1; +SET binlog_annotate_row_events= 1; +SET GLOBAL binlog_annotate_row_events= 1; +SET GLOBAL binlog_checksum= CRC32; +SET GLOBAL master_verify_checksum= 1; +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CREATE DATABASE database_name_to_encrypt; +USE database_name_to_encrypt; +CREATE USER user_name_to_encrypt; +GRANT ALL ON database_name_to_encrypt.* TO user_name_to_encrypt; +SET PASSWORD FOR user_name_to_encrypt = PASSWORD('password_to_encrypt'); +CREATE TABLE innodb_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB, +virt_column_name_to_encrypt INT AS (int_column_name_to_encrypt % 10) VIRTUAL, +pers_column_name_to_encrypt INT AS (int_column_name_to_encrypt) PERSISTENT, +INDEX `index_name_to_encrypt`(`timestamp_column_name_to_encrypt`) +) ENGINE=InnoDB +PARTITION BY RANGE (int_column_name_to_encrypt) +SUBPARTITION BY KEY (int_column_name_to_encrypt) +SUBPARTITIONS 2 ( +PARTITION partition0_name_to_encrypt VALUES LESS THAN (100), +PARTITION partition1_name_to_encrypt VALUES LESS THAN (MAXVALUE) +) +; +CREATE TABLE myisam_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +char_column_name_to_encrypt VARCHAR(255), +datetime_column_name_to_encrypt DATETIME, +text_column_name_to_encrypt TEXT +) ENGINE=MyISAM; +CREATE TABLE aria_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +varchar_column_name_to_encrypt VARCHAR(1024), +enum_column_name_to_encrypt ENUM( +'enum_value1_to_encrypt', +'enum_value2_to_encrypt' + ), +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB +) ENGINE=Aria; +CREATE TRIGGER trigger_name_to_encrypt +AFTER INSERT ON myisam_table_name_to_encrypt FOR EACH ROW +INSERT INTO aria_table_name_to_encrypt (varchar_column_name_to_encrypt) +VALUES (NEW.char_column_name_to_encrypt); +CREATE DEFINER=user_name_to_encrypt VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +CREATE FUNCTION func_name_to_encrypt (func_parameter_to_encrypt INT) +RETURNS VARCHAR(64) +RETURN 'func_result_to_encrypt'; +CREATE PROCEDURE proc_name_to_encrypt ( +IN proc_in_parameter_to_encrypt CHAR(32), +OUT proc_out_parameter_to_encrypt INT +) +BEGIN +DECLARE procvar_name_to_encrypt CHAR(64) DEFAULT 'procvar_val_to_encrypt'; +DECLARE cursor_name_to_encrypt CURSOR FOR +SELECT virt_column_name_to_encrypt FROM innodb_table_name_to_encrypt; +DECLARE EXIT HANDLER FOR NOT FOUND +BEGIN +SET @stmt_var_to_encrypt = CONCAT( +"SELECT + IF (RAND()>0.5,'enum_value2_to_encrypt','enum_value1_to_encrypt') + FROM innodb_table_name_to_encrypt + INTO OUTFILE '", proc_in_parameter_to_encrypt, "'"); +PREPARE stmt_to_encrypt FROM @stmt_var_to_encrypt; +EXECUTE stmt_to_encrypt; +DEALLOCATE PREPARE stmt_to_encrypt; +END; +OPEN cursor_name_to_encrypt; +proc_label_to_encrypt: LOOP +FETCH cursor_name_to_encrypt INTO procvar_name_to_encrypt; +END LOOP; +CLOSE cursor_name_to_encrypt; +END $$ +CREATE SERVER server_name_to_encrypt +FOREIGN DATA WRAPPER mysql +OPTIONS (HOST 'host_name_to_encrypt'); +connect con1,localhost,user_name_to_encrypt,password_to_encrypt,database_name_to_encrypt; +CREATE TEMPORARY TABLE tmp_table_name_to_encrypt ( +float_column_name_to_encrypt FLOAT, +binary_column_name_to_encrypt BINARY(64) +); +disconnect con1; +connection server_1; +CREATE INDEX index_name_to_encrypt +ON myisam_table_name_to_encrypt (datetime_column_name_to_encrypt); +ALTER DATABASE database_name_to_encrypt CHARACTER SET utf8; +ALTER TABLE innodb_table_name_to_encrypt +MODIFY timestamp_column_name_to_encrypt TIMESTAMP NOT NULL +DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +; +ALTER ALGORITHM=MERGE VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +RENAME TABLE innodb_table_name_to_encrypt TO new_table_name_to_encrypt; +ALTER TABLE new_table_name_to_encrypt RENAME TO innodb_table_name_to_encrypt; +set @user_var1_to_encrypt= 'dyncol1_val_to_encrypt'; +set @user_var2_to_encrypt= 'dyncol2_name_to_encrypt'; +INSERT INTO view_name_to_encrypt VALUES +(1, NOW(6), COLUMN_CREATE('dyncol1_name_to_encrypt',@user_var1_to_encrypt), NULL, NULL), +(2, NOW(6), COLUMN_CREATE(@user_var2_to_encrypt,'dyncol2_val_to_encrypt'), NULL, NULL) +; +BEGIN NOT ATOMIC +DECLARE counter_name_to_encrypt INT DEFAULT 0; +START TRANSACTION; +WHILE counter_name_to_encrypt<12 DO +INSERT INTO innodb_table_name_to_encrypt +SELECT NULL, NOW(6), blob_column_name_to_encrypt, NULL, NULL +FROM innodb_table_name_to_encrypt +ORDER BY int_column_name_to_encrypt; +SET counter_name_to_encrypt = counter_name_to_encrypt+1; +END WHILE; +COMMIT; +END +$$ +INSERT INTO myisam_table_name_to_encrypt +SELECT NULL, 'char_literal_to_encrypt', NULL, 'text_to_encrypt'; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +CALL proc_name_to_encrypt('file_name_to_encrypt',@useless_var_to_encrypt); +TRUNCATE TABLE aria_table_name_to_encrypt; +LOAD DATA INFILE 'file_name_to_encrypt' INTO TABLE aria_table_name_to_encrypt +(enum_column_name_to_encrypt); +LOAD DATA LOCAL INFILE '/database_name_to_encrypt/file_name_to_encrypt' +INTO TABLE aria_table_name_to_encrypt (enum_column_name_to_encrypt); +UPDATE view_name_to_encrypt SET blob_column_name_to_encrypt = +COLUMN_CREATE('dyncol1_name_to_encrypt',func_name_to_encrypt(0)) +; +DELETE FROM aria_table_name_to_encrypt ORDER BY int_column_name_to_encrypt LIMIT 10; +ANALYZE TABLE myisam_table_name_to_encrypt; +CHECK TABLE aria_table_name_to_encrypt; +CHECKSUM TABLE innodb_table_name_to_encrypt, myisam_table_name_to_encrypt; +RENAME USER user_name_to_encrypt to new_user_name_to_encrypt; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM new_user_name_to_encrypt; +DROP DATABASE database_name_to_encrypt; +DROP USER new_user_name_to_encrypt; +DROP SERVER server_name_to_encrypt; +#################################################### +# Test 4: binlog with annotated events and binlog_row_image=minimal +#################################################### +connection server_1; +SET binlog_annotate_row_events= 1; +SET GLOBAL binlog_annotate_row_events= 1; +SET GLOBAL binlog_checksum= NONE; +SET GLOBAL master_verify_checksum= 0; +SET GLOBAL binlog_row_image= MINIMAL; +SET binlog_row_image= MINIMAL; +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CREATE DATABASE database_name_to_encrypt; +USE database_name_to_encrypt; +CREATE USER user_name_to_encrypt; +GRANT ALL ON database_name_to_encrypt.* TO user_name_to_encrypt; +SET PASSWORD FOR user_name_to_encrypt = PASSWORD('password_to_encrypt'); +CREATE TABLE innodb_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB, +virt_column_name_to_encrypt INT AS (int_column_name_to_encrypt % 10) VIRTUAL, +pers_column_name_to_encrypt INT AS (int_column_name_to_encrypt) PERSISTENT, +INDEX `index_name_to_encrypt`(`timestamp_column_name_to_encrypt`) +) ENGINE=InnoDB +PARTITION BY RANGE (int_column_name_to_encrypt) +SUBPARTITION BY KEY (int_column_name_to_encrypt) +SUBPARTITIONS 2 ( +PARTITION partition0_name_to_encrypt VALUES LESS THAN (100), +PARTITION partition1_name_to_encrypt VALUES LESS THAN (MAXVALUE) +) +; +CREATE TABLE myisam_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +char_column_name_to_encrypt VARCHAR(255), +datetime_column_name_to_encrypt DATETIME, +text_column_name_to_encrypt TEXT +) ENGINE=MyISAM; +CREATE TABLE aria_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +varchar_column_name_to_encrypt VARCHAR(1024), +enum_column_name_to_encrypt ENUM( +'enum_value1_to_encrypt', +'enum_value2_to_encrypt' + ), +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB +) ENGINE=Aria; +CREATE TRIGGER trigger_name_to_encrypt +AFTER INSERT ON myisam_table_name_to_encrypt FOR EACH ROW +INSERT INTO aria_table_name_to_encrypt (varchar_column_name_to_encrypt) +VALUES (NEW.char_column_name_to_encrypt); +CREATE DEFINER=user_name_to_encrypt VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +CREATE FUNCTION func_name_to_encrypt (func_parameter_to_encrypt INT) +RETURNS VARCHAR(64) +RETURN 'func_result_to_encrypt'; +CREATE PROCEDURE proc_name_to_encrypt ( +IN proc_in_parameter_to_encrypt CHAR(32), +OUT proc_out_parameter_to_encrypt INT +) +BEGIN +DECLARE procvar_name_to_encrypt CHAR(64) DEFAULT 'procvar_val_to_encrypt'; +DECLARE cursor_name_to_encrypt CURSOR FOR +SELECT virt_column_name_to_encrypt FROM innodb_table_name_to_encrypt; +DECLARE EXIT HANDLER FOR NOT FOUND +BEGIN +SET @stmt_var_to_encrypt = CONCAT( +"SELECT + IF (RAND()>0.5,'enum_value2_to_encrypt','enum_value1_to_encrypt') + FROM innodb_table_name_to_encrypt + INTO OUTFILE '", proc_in_parameter_to_encrypt, "'"); +PREPARE stmt_to_encrypt FROM @stmt_var_to_encrypt; +EXECUTE stmt_to_encrypt; +DEALLOCATE PREPARE stmt_to_encrypt; +END; +OPEN cursor_name_to_encrypt; +proc_label_to_encrypt: LOOP +FETCH cursor_name_to_encrypt INTO procvar_name_to_encrypt; +END LOOP; +CLOSE cursor_name_to_encrypt; +END $$ +CREATE SERVER server_name_to_encrypt +FOREIGN DATA WRAPPER mysql +OPTIONS (HOST 'host_name_to_encrypt'); +connect con1,localhost,user_name_to_encrypt,password_to_encrypt,database_name_to_encrypt; +CREATE TEMPORARY TABLE tmp_table_name_to_encrypt ( +float_column_name_to_encrypt FLOAT, +binary_column_name_to_encrypt BINARY(64) +); +disconnect con1; +connection server_1; +CREATE INDEX index_name_to_encrypt +ON myisam_table_name_to_encrypt (datetime_column_name_to_encrypt); +ALTER DATABASE database_name_to_encrypt CHARACTER SET utf8; +ALTER TABLE innodb_table_name_to_encrypt +MODIFY timestamp_column_name_to_encrypt TIMESTAMP NOT NULL +DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +; +ALTER ALGORITHM=MERGE VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +RENAME TABLE innodb_table_name_to_encrypt TO new_table_name_to_encrypt; +ALTER TABLE new_table_name_to_encrypt RENAME TO innodb_table_name_to_encrypt; +set @user_var1_to_encrypt= 'dyncol1_val_to_encrypt'; +set @user_var2_to_encrypt= 'dyncol2_name_to_encrypt'; +INSERT INTO view_name_to_encrypt VALUES +(1, NOW(6), COLUMN_CREATE('dyncol1_name_to_encrypt',@user_var1_to_encrypt), NULL, NULL), +(2, NOW(6), COLUMN_CREATE(@user_var2_to_encrypt,'dyncol2_val_to_encrypt'), NULL, NULL) +; +BEGIN NOT ATOMIC +DECLARE counter_name_to_encrypt INT DEFAULT 0; +START TRANSACTION; +WHILE counter_name_to_encrypt<12 DO +INSERT INTO innodb_table_name_to_encrypt +SELECT NULL, NOW(6), blob_column_name_to_encrypt, NULL, NULL +FROM innodb_table_name_to_encrypt +ORDER BY int_column_name_to_encrypt; +SET counter_name_to_encrypt = counter_name_to_encrypt+1; +END WHILE; +COMMIT; +END +$$ +INSERT INTO myisam_table_name_to_encrypt +SELECT NULL, 'char_literal_to_encrypt', NULL, 'text_to_encrypt'; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +CALL proc_name_to_encrypt('file_name_to_encrypt',@useless_var_to_encrypt); +TRUNCATE TABLE aria_table_name_to_encrypt; +LOAD DATA INFILE 'file_name_to_encrypt' INTO TABLE aria_table_name_to_encrypt +(enum_column_name_to_encrypt); +LOAD DATA LOCAL INFILE '/database_name_to_encrypt/file_name_to_encrypt' +INTO TABLE aria_table_name_to_encrypt (enum_column_name_to_encrypt); +UPDATE view_name_to_encrypt SET blob_column_name_to_encrypt = +COLUMN_CREATE('dyncol1_name_to_encrypt',func_name_to_encrypt(0)) +; +DELETE FROM aria_table_name_to_encrypt ORDER BY int_column_name_to_encrypt LIMIT 10; +ANALYZE TABLE myisam_table_name_to_encrypt; +CHECK TABLE aria_table_name_to_encrypt; +CHECKSUM TABLE innodb_table_name_to_encrypt, myisam_table_name_to_encrypt; +RENAME USER user_name_to_encrypt to new_user_name_to_encrypt; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM new_user_name_to_encrypt; +DROP DATABASE database_name_to_encrypt; +DROP USER new_user_name_to_encrypt; +DROP SERVER server_name_to_encrypt; +############################# +# Final checks for the master +############################# +NOT FOUND /_to_encrypt/ in master-bin.0* +NOT FOUND /COMMIT/ in master-bin.0* +NOT FOUND /TIMESTAMP/ in master-bin.0* +include/save_master_pos.inc +############################# +# Final checks for the slave +############################# +connection server_2; +include/sync_io_with_master.inc +FOUND /_to_encrypt/ in slave-relay-bin.0* +FOUND /COMMIT/ in slave-relay-bin.0* +FOUND /TIMESTAMP/ in slave-relay-bin.0* +include/start_slave.inc +include/sync_slave_sql_with_io.inc +FOUND /_to_encrypt/ in slave-bin.0* +FOUND /COMMIT/ in slave-bin.0* +FOUND /TIMESTAMP/ in slave-bin.0* +########## +# Cleanup +########## +connection server_1; +SET GLOBAL binlog_annotate_row_events= @binlog_annotate_row_events.save; +SET GLOBAL binlog_checksum= @binlog_checksum.save; +SET GLOBAL master_verify_checksum= @master_verify_checksum.save; +SET GLOBAL binlog_row_image= @binlog_row_image.save; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_master.test b/mysql-test/suite/binlog_encryption/encrypted_master.test new file mode 100644 index 0000000000000..5eb0345342d5d --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master.test @@ -0,0 +1,183 @@ +# +# The test checks that basic DDL and DML events are encrypted +# in the binary log on master. +# The test is to be run with all binlog formats +# (combinations for rpl_init.inc take care of that). +# +# +# The test runs with the encrypted master and non-encrypted slave. +# It generates a sequence of events on master, and checks that +# - all events are encrypted on master; +# - slave is able to replicate from the master; +# - relay logs and binary logs are not encrypted on slave. +# +# The same exercise is repeated +# - without annotated binlog events and without binlog checksums; +# - with binlog checksums; +# - with annotated events and binlog checksums; +# - with annotated events, default checksums and minimal binlog row image +# + +--source encryption_algorithms.inc +--source include/have_innodb.inc +--enable_connect_log + +--echo ################# +--echo # Initialization +--echo ################# + +--disable_connect_log +--let $rpl_topology= 1->2 +--source include/rpl_init.inc +--enable_connect_log + +# We stop SQL thread because we want to have +# all relay logs at the end of the test flow + +--connection server_2 +--disable_connect_log +--source include/stop_slave_sql.inc +--enable_connect_log + +--connection server_1 + +SET @binlog_annotate_row_events.save= @@global.binlog_annotate_row_events; +SET @binlog_checksum.save= @@global.binlog_checksum; +SET @master_verify_checksum.save= @@global.master_verify_checksum; +SET @binlog_row_image.save= @@global.binlog_row_image; + +--echo #################################################### +--echo # Test 1: simple binlog, no checksum, no annotation +--echo #################################################### + +--connection server_1 + +SET binlog_annotate_row_events= 0; +SET GLOBAL binlog_annotate_row_events= 0; +SET GLOBAL binlog_checksum= NONE; +SET GLOBAL master_verify_checksum= 0; + +--source testdata.inc + +--echo #################################################### +--echo # Test 2: binlog with checksum, no annotated events +--echo #################################################### + +--connection server_1 + +SET binlog_annotate_row_events= 0; +SET GLOBAL binlog_annotate_row_events= 0; +SET GLOBAL binlog_checksum= CRC32; +SET GLOBAL master_verify_checksum= 1; + +--source testdata.inc + +--echo #################################################### +--echo # Test 3: binlog with checksum and annotated events +--echo #################################################### + +--connection server_1 + +SET binlog_annotate_row_events= 1; +SET GLOBAL binlog_annotate_row_events= 1; +SET GLOBAL binlog_checksum= CRC32; +SET GLOBAL master_verify_checksum= 1; + +--source testdata.inc + +--echo #################################################### +--echo # Test 4: binlog with annotated events and binlog_row_image=minimal +--echo #################################################### + +--connection server_1 + +SET binlog_annotate_row_events= 1; +SET GLOBAL binlog_annotate_row_events= 1; +SET GLOBAL binlog_checksum= NONE; +SET GLOBAL master_verify_checksum= 0; +SET GLOBAL binlog_row_image= MINIMAL; +SET binlog_row_image= MINIMAL; + +--source testdata.inc + +--echo ############################# +--echo # Final checks for the master +--echo ############################# + +--let $master_datadir= `SELECT @@datadir` + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= _to_encrypt +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= COMMIT +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= TIMESTAMP +--source include/search_pattern_in_file.inc + +--disable_connect_log +--source include/save_master_pos.inc +--enable_connect_log + +--echo ############################# +--echo # Final checks for the slave +--echo ############################# + +# Wait for the IO thread to write everything to relay logs + +--connection server_2 + +--let $slave_datadir= `SELECT @@datadir` + +--disable_connect_log +--source include/sync_io_with_master.inc + +# Check that relay logs are unencrypted + +--let SEARCH_FILE= $slave_datadir/slave-relay-bin.0* +--let SEARCH_PATTERN= _to_encrypt +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-relay-bin.0* +--let SEARCH_PATTERN= COMMIT +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-relay-bin.0* +--let SEARCH_PATTERN= TIMESTAMP +--source include/search_pattern_in_file.inc + + +# Re-enable SQL thread, let it catch up with IO thread +# and check slave binary logs + +--source include/start_slave.inc +--source include/sync_slave_sql_with_io.inc +--enable_connect_log + +--let SEARCH_FILE= $slave_datadir/slave-bin.0* +--let SEARCH_PATTERN= _to_encrypt +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-bin.0* +--let SEARCH_PATTERN= COMMIT +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-bin.0* +--let SEARCH_PATTERN= TIMESTAMP +--source include/search_pattern_in_file.inc + +--echo ########## +--echo # Cleanup +--echo ########## + +--connection server_1 +SET GLOBAL binlog_annotate_row_events= @binlog_annotate_row_events.save; +SET GLOBAL binlog_checksum= @binlog_checksum.save; +SET GLOBAL master_verify_checksum= @master_verify_checksum.save; +SET GLOBAL binlog_row_image= @binlog_row_image.save; + +--disable_connect_log +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_lost_key.result b/mysql-test/suite/binlog_encryption/encrypted_master_lost_key.result new file mode 100644 index 0000000000000..5c934af15e409 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master_lost_key.result @@ -0,0 +1,111 @@ +################# +# Initialization +################# +include/rpl_init.inc [topology=1->2] +connection server_2; +include/stop_slave.inc +##################################################### +# Pre-test 1: Initial key value +##################################################### +connection server_1; +CREATE TABLE table1_to_encrypt ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table1_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table1_to_encrypt SELECT NULL,NOW(),b FROM table1_to_encrypt; +SET binlog_format=ROW; +INSERT INTO table1_to_encrypt SELECT NULL,NOW(),b FROM table1_to_encrypt; +INSERT INTO table1_to_encrypt SELECT NULL,NOW(),b FROM table1_to_encrypt; +NOT FOUND /table1_to_encrypt/ in master-bin.0* +####################################################### +# Pre-test 2: restart master with a different key value +####################################################### +connection default; +connection server_1; +CREATE TABLE table2_to_encrypt ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +SET binlog_format=ROW; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +NOT FOUND /table2_to_encrypt/ in master-bin.0* +##################################################### +# Pre-test 3: restart master again with the right key +##################################################### +connection default; +connection server_1; +CREATE TABLE table3_to_encrypt ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table3_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table3_to_encrypt SELECT NULL,NOW(),b FROM table3_to_encrypt; +INSERT INTO table3_to_encrypt SELECT NULL,NOW(),b FROM table3_to_encrypt; +FLUSH BINARY LOGS; +INSERT INTO table3_to_encrypt SELECT NULL,NOW(),b FROM table3_to_encrypt; +##################################################### +# Test 1: Check that if master has an encrypted +# binary log which it cannot decrypt, it +# still feeds events to the slave, and SQL +# thread produces an expected error upon +# receiving these unreadable events . +# This behavior is confirmed in MDEV-11323 +##################################################### +connection server_2; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE SQL_THREAD; +include/wait_for_slave_sql_error.inc [errno=1594] +SHOW TABLES; +Tables_in_test +table1_to_encrypt +SELECT COUNT(*) FROM table1_to_encrypt; +COUNT(*) +8 +##################################################### +# Test 2: check that replication works if it starts +# from a good binary log +##################################################### +connection server_2; +include/stop_slave.inc +RESET SLAVE ALL; +DROP DATABASE test; +CREATE DATABASE test; +USE test; +CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=, MASTER_USER='root', MASTER_LOG_FILE='master-bin.000003'; +include/start_slave.inc +SHOW TABLES; +Tables_in_test +table3_to_encrypt +##################################################### +# Test 3: check that replication works if we purge +# master logs up to the good one +##################################################### +connection server_2; +connection server_1; +PURGE BINARY LOGS TO 'master-bin.000003'; +connection server_2; +include/stop_slave.inc +RESET SLAVE ALL; +DROP DATABASE test; +CREATE DATABASE test; +USE test; +CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=, MASTER_USER='root'; +include/start_slave.inc +SHOW TABLES; +Tables_in_test +table3_to_encrypt +########## +# Cleanup +########## +connection server_1; +DROP TABLE table1_to_encrypt, table2_to_encrypt, table3_to_encrypt; +connection server_2; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_lost_key.test b/mysql-test/suite/binlog_encryption/encrypted_master_lost_key.test new file mode 100644 index 0000000000000..7e5fd7859f062 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master_lost_key.test @@ -0,0 +1,205 @@ +# +# The test checks effects and workarounds for the situation when +# the key used to encrypt previous binary logs on master has been lost, +# and master runs with a different one. +# +# The test starts with encrypted binlogs on master. +# It stops replication, generates a few statement and row events +# on the master, then restarts the server with encrypted binlog, +# but with a different value for key 1. +# +# Then it resumes replication and checks what happens when the master +# feed the encrypted logs to the slave (slave SQL thread should +# produce and error). +# +# Then the test resets the slave, configures it to start from a "good" +# master binlog log, for which the master has a key, starts replication +# and checks that it works. +# +# Then it resets the slave again, purges binary logs on master up +# to the "good" one, starts replication and checks that it works. +# + +--source include/have_binlog_format_mixed.inc + +--echo ################# +--echo # Initialization +--echo ################# + +--let $rpl_topology= 1->2 +--source include/rpl_init.inc + +--enable_connect_log + +# We stop replication because we want it to happen after the switch + +--connection server_2 +--disable_connect_log +--source include/stop_slave.inc +--enable_connect_log + +--echo ##################################################### +--echo # Pre-test 1: Initial key value +--echo ##################################################### + +--connection server_1 + +CREATE TABLE table1_to_encrypt ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table1_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table1_to_encrypt SELECT NULL,NOW(),b FROM table1_to_encrypt; +SET binlog_format=ROW; +INSERT INTO table1_to_encrypt SELECT NULL,NOW(),b FROM table1_to_encrypt; +INSERT INTO table1_to_encrypt SELECT NULL,NOW(),b FROM table1_to_encrypt; + +# Make sure that binary logs are encrypted + +--let SEARCH_FILE= master-bin.0* +--let SEARCH_PATTERN= table1_to_encrypt +--source include/search_pattern_in_file.inc + +--echo ####################################################### +--echo # Pre-test 2: restart master with a different key value +--echo ####################################################### + +--write_file $MYSQL_TMP_DIR/master_lose_key.key +1;00000AAAAAAAAAAAAAAAAAAAAAA00000 +EOF + +--let $rpl_server_parameters= --file-key-management-filename=$MYSQL_TMP_DIR/master_lose_key.key + +--let $rpl_server_number= 1 +--source restart_server.inc + +CREATE TABLE table2_to_encrypt ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +SET binlog_format=ROW; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; + +# Make sure that binary logs are encrypted + +--let SEARCH_FILE= master-bin.0* +--let SEARCH_PATTERN= table2_to_encrypt +--source include/search_pattern_in_file.inc + +--echo ##################################################### +--echo # Pre-test 3: restart master again with the right key +--echo ##################################################### + +--let $rpl_server_parameters= +--let $rpl_server_number= 1 +--source restart_server.inc + +--let $good_master_binlog= query_get_value(SHOW MASTER STATUS,File,1) + +CREATE TABLE table3_to_encrypt ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table3_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table3_to_encrypt SELECT NULL,NOW(),b FROM table3_to_encrypt; +INSERT INTO table3_to_encrypt SELECT NULL,NOW(),b FROM table3_to_encrypt; +FLUSH BINARY LOGS; +INSERT INTO table3_to_encrypt SELECT NULL,NOW(),b FROM table3_to_encrypt; + +--save_master_pos + +--echo ##################################################### +--echo # Test 1: Check that if master has an encrypted +--echo # binary log which it cannot decrypt, it +--echo # still feeds events to the slave, and SQL +--echo # thread produces an expected error upon +--echo # receiving these unreadable events . +--echo # This behavior is confirmed in MDEV-11323 +--echo ##################################################### +--connection server_2 + +--disable_connect_log +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +START SLAVE SQL_THREAD; +--let $slave_sql_errno= 1594 +--source include/wait_for_slave_sql_error.inc +--enable_connect_log + +# Here we should see only table1_to_encrypt and its contents, +# because it was logged with the initial key +--sorted_result +SHOW TABLES; +SELECT COUNT(*) FROM table1_to_encrypt; + +--echo ##################################################### +--echo # Test 2: check that replication works if it starts +--echo # from a good binary log +--echo ##################################################### +--connection server_2 + +--disable_connect_log +--source include/stop_slave.inc +RESET SLAVE ALL; +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--replace_result $SERVER_MYPORT_1 +eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=$SERVER_MYPORT_1, MASTER_USER='root', MASTER_LOG_FILE='$good_master_binlog'; +--source include/start_slave.inc +--enable_connect_log +--sync_with_master + +--sorted_result +SHOW TABLES; + +--echo ##################################################### +--echo # Test 3: check that replication works if we purge +--echo # master logs up to the good one +--echo ##################################################### +--connection server_2 + +--connection server_1 +eval PURGE BINARY LOGS TO '$good_master_binlog'; + +--connection server_2 +--disable_connect_log +--source include/stop_slave.inc +RESET SLAVE ALL; +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--replace_result $SERVER_MYPORT_1 +eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=$SERVER_MYPORT_1, MASTER_USER='root'; +--source include/start_slave.inc +--enable_connect_log +--sync_with_master + +--sorted_result +SHOW TABLES; + +--echo ########## +--echo # Cleanup +--echo ########## + +--connection server_1 + +DROP TABLE table1_to_encrypt, table2_to_encrypt, table3_to_encrypt; + +--save_master_pos + +--connection server_2 +--sync_with_master + +--disable_connect_log +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.cnf b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.cnf new file mode 100644 index 0000000000000..73c9ad655bfee --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.cnf @@ -0,0 +1,5 @@ +!include my.cnf + +[mysqld.1] +encrypt-binlog=0 +skip-file-key-management diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test new file mode 100644 index 0000000000000..70133e30b6953 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test @@ -0,0 +1,135 @@ +# +# TODO: write here what the test checks after MDEV-11288 is fixed +# +# The test starts with unencrypted master. +# It stops replication, generates a few statement and row events +# on the master, then restarts the server with encrypted binlog, +# generates some more events and restarts it back without encryption +# (no encryption plugin). +# Then it resumes replication and checks what happens when the server +# tries to feed the binary logs (included the encrypted ones) +# to the slave. +# + +--source include/have_binlog_format_mixed.inc + +--echo ################# +--echo # Initialization +--echo ################# + +--let $rpl_topology= 1->2 +--source include/rpl_init.inc + +--enable_connect_log + +# We stop replication because we want it to happen after the switch + +--connection server_2 +--disable_connect_log +--source include/stop_slave.inc +--enable_connect_log + +--echo ##################################################### +--echo # Part 1: unencrypted master +--echo ##################################################### + +--connection server_1 + +CREATE TABLE table1_no_encryption ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table1_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; + +# Make sure that binary logs are not encrypted + +--let SEARCH_FILE= master-bin.0* +--let SEARCH_PATTERN= table1_no_encryption +--source include/search_pattern_in_file.inc + +--echo ##################################################### +--echo # Part 2: restart master, now with binlog encryption +--echo ##################################################### + +--let $rpl_server_parameters= --encrypt-binlog=1 --plugin-load-add=$FILE_KEY_MANAGEMENT_SO --file-key-management --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt + +--let $rpl_server_number= 1 +--source restart_server.inc + +CREATE TABLE table2_to_encrypt ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; + +# Make sure that binary logs are encrypted + +--let SEARCH_FILE= master-bin.0* +--let SEARCH_PATTERN= table2_to_encrypt +--source include/search_pattern_in_file.inc + +--echo ##################################################### +--echo # Part 3: restart master again without encryption +--echo ##################################################### + +--let $rpl_server_parameters= --encrypt-binlog=0 +--let $rpl_server_number= 1 +--source restart_server.inc + +CREATE TABLE table3_no_encryption ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table3_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; + +--save_master_pos + +--echo ##################################################### +--echo # Check: resume replication and check how it goes +--echo ##################################################### +--connection server_2 + +--disable_connect_log +--source include/start_slave.inc +--enable_connect_log +--sync_with_master + +--sorted_result +SHOW TABLES; + +--echo ########## +--echo # Cleanup +--echo ########## + +--connection server_1 + +SELECT COUNT(*) FROM table1_no_encryption; +SELECT COUNT(*) FROM table2_to_encrypt; +SELECT COUNT(*) FROM table3_no_encryption; +DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption; + +--save_master_pos + +--connection server_2 +--sync_with_master + +--disable_connect_log +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_slave.cnf b/mysql-test/suite/binlog_encryption/encrypted_slave.cnf new file mode 100644 index 0000000000000..fac94db71df6f --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_slave.cnf @@ -0,0 +1,12 @@ +!include my.cnf + +[mysqld.1] +encrypt-binlog=0 + +[mysqld.2] +#log-slave-updates +encrypt-binlog +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +file-key-management +loose-file-key-management-filename= @ENV.MYSQL_TEST_DIR/std_data/keys.txt +binlog_checksum=NONE diff --git a/mysql-test/suite/binlog_encryption/encrypted_slave.result b/mysql-test/suite/binlog_encryption/encrypted_slave.result new file mode 100644 index 0000000000000..c78cd9d150796 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_slave.result @@ -0,0 +1,175 @@ +################# +# Initialization +################# +include/rpl_init.inc [topology=1->2] +connection server_2; +include/stop_slave_sql.inc +################# +# Test flow +################# +connection server_1; +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +CREATE DATABASE database_name_to_encrypt; +USE database_name_to_encrypt; +CREATE USER user_name_to_encrypt; +GRANT ALL ON database_name_to_encrypt.* TO user_name_to_encrypt; +SET PASSWORD FOR user_name_to_encrypt = PASSWORD('password_to_encrypt'); +CREATE TABLE innodb_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB, +virt_column_name_to_encrypt INT AS (int_column_name_to_encrypt % 10) VIRTUAL, +pers_column_name_to_encrypt INT AS (int_column_name_to_encrypt) PERSISTENT, +INDEX `index_name_to_encrypt`(`timestamp_column_name_to_encrypt`) +) ENGINE=InnoDB +PARTITION BY RANGE (int_column_name_to_encrypt) +SUBPARTITION BY KEY (int_column_name_to_encrypt) +SUBPARTITIONS 2 ( +PARTITION partition0_name_to_encrypt VALUES LESS THAN (100), +PARTITION partition1_name_to_encrypt VALUES LESS THAN (MAXVALUE) +) +; +CREATE TABLE myisam_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +char_column_name_to_encrypt VARCHAR(255), +datetime_column_name_to_encrypt DATETIME, +text_column_name_to_encrypt TEXT +) ENGINE=MyISAM; +CREATE TABLE aria_table_name_to_encrypt ( +int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, +varchar_column_name_to_encrypt VARCHAR(1024), +enum_column_name_to_encrypt ENUM( +'enum_value1_to_encrypt', +'enum_value2_to_encrypt' + ), +timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, +blob_column_name_to_encrypt BLOB +) ENGINE=Aria; +CREATE TRIGGER trigger_name_to_encrypt +AFTER INSERT ON myisam_table_name_to_encrypt FOR EACH ROW +INSERT INTO aria_table_name_to_encrypt (varchar_column_name_to_encrypt) +VALUES (NEW.char_column_name_to_encrypt); +CREATE DEFINER=user_name_to_encrypt VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +CREATE FUNCTION func_name_to_encrypt (func_parameter_to_encrypt INT) +RETURNS VARCHAR(64) +RETURN 'func_result_to_encrypt'; +CREATE PROCEDURE proc_name_to_encrypt ( +IN proc_in_parameter_to_encrypt CHAR(32), +OUT proc_out_parameter_to_encrypt INT +) +BEGIN +DECLARE procvar_name_to_encrypt CHAR(64) DEFAULT 'procvar_val_to_encrypt'; +DECLARE cursor_name_to_encrypt CURSOR FOR +SELECT virt_column_name_to_encrypt FROM innodb_table_name_to_encrypt; +DECLARE EXIT HANDLER FOR NOT FOUND +BEGIN +SET @stmt_var_to_encrypt = CONCAT( +"SELECT + IF (RAND()>0.5,'enum_value2_to_encrypt','enum_value1_to_encrypt') + FROM innodb_table_name_to_encrypt + INTO OUTFILE '", proc_in_parameter_to_encrypt, "'"); +PREPARE stmt_to_encrypt FROM @stmt_var_to_encrypt; +EXECUTE stmt_to_encrypt; +DEALLOCATE PREPARE stmt_to_encrypt; +END; +OPEN cursor_name_to_encrypt; +proc_label_to_encrypt: LOOP +FETCH cursor_name_to_encrypt INTO procvar_name_to_encrypt; +END LOOP; +CLOSE cursor_name_to_encrypt; +END $$ +CREATE SERVER server_name_to_encrypt +FOREIGN DATA WRAPPER mysql +OPTIONS (HOST 'host_name_to_encrypt'); +connect con1,localhost,user_name_to_encrypt,password_to_encrypt,database_name_to_encrypt; +CREATE TEMPORARY TABLE tmp_table_name_to_encrypt ( +float_column_name_to_encrypt FLOAT, +binary_column_name_to_encrypt BINARY(64) +); +disconnect con1; +connection server_1; +CREATE INDEX index_name_to_encrypt +ON myisam_table_name_to_encrypt (datetime_column_name_to_encrypt); +ALTER DATABASE database_name_to_encrypt CHARACTER SET utf8; +ALTER TABLE innodb_table_name_to_encrypt +MODIFY timestamp_column_name_to_encrypt TIMESTAMP NOT NULL +DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +; +ALTER ALGORITHM=MERGE VIEW view_name_to_encrypt +AS SELECT * FROM innodb_table_name_to_encrypt; +RENAME TABLE innodb_table_name_to_encrypt TO new_table_name_to_encrypt; +ALTER TABLE new_table_name_to_encrypt RENAME TO innodb_table_name_to_encrypt; +set @user_var1_to_encrypt= 'dyncol1_val_to_encrypt'; +set @user_var2_to_encrypt= 'dyncol2_name_to_encrypt'; +INSERT INTO view_name_to_encrypt VALUES +(1, NOW(6), COLUMN_CREATE('dyncol1_name_to_encrypt',@user_var1_to_encrypt), NULL, NULL), +(2, NOW(6), COLUMN_CREATE(@user_var2_to_encrypt,'dyncol2_val_to_encrypt'), NULL, NULL) +; +BEGIN NOT ATOMIC +DECLARE counter_name_to_encrypt INT DEFAULT 0; +START TRANSACTION; +WHILE counter_name_to_encrypt<12 DO +INSERT INTO innodb_table_name_to_encrypt +SELECT NULL, NOW(6), blob_column_name_to_encrypt, NULL, NULL +FROM innodb_table_name_to_encrypt +ORDER BY int_column_name_to_encrypt; +SET counter_name_to_encrypt = counter_name_to_encrypt+1; +END WHILE; +COMMIT; +END +$$ +INSERT INTO myisam_table_name_to_encrypt +SELECT NULL, 'char_literal_to_encrypt', NULL, 'text_to_encrypt'; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) +SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +CALL proc_name_to_encrypt('file_name_to_encrypt',@useless_var_to_encrypt); +TRUNCATE TABLE aria_table_name_to_encrypt; +LOAD DATA INFILE 'file_name_to_encrypt' INTO TABLE aria_table_name_to_encrypt +(enum_column_name_to_encrypt); +LOAD DATA LOCAL INFILE '/database_name_to_encrypt/file_name_to_encrypt' +INTO TABLE aria_table_name_to_encrypt (enum_column_name_to_encrypt); +UPDATE view_name_to_encrypt SET blob_column_name_to_encrypt = +COLUMN_CREATE('dyncol1_name_to_encrypt',func_name_to_encrypt(0)) +; +DELETE FROM aria_table_name_to_encrypt ORDER BY int_column_name_to_encrypt LIMIT 10; +ANALYZE TABLE myisam_table_name_to_encrypt; +CHECK TABLE aria_table_name_to_encrypt; +CHECKSUM TABLE innodb_table_name_to_encrypt, myisam_table_name_to_encrypt; +RENAME USER user_name_to_encrypt to new_user_name_to_encrypt; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM new_user_name_to_encrypt; +DROP DATABASE database_name_to_encrypt; +DROP USER new_user_name_to_encrypt; +DROP SERVER server_name_to_encrypt; +################# +# Master binlog checks +################# +FOUND /_to_encrypt/ in master-bin.0* +FOUND /COMMIT/ in master-bin.0* +FOUND /TIMESTAMP/ in master-bin.0* +include/save_master_pos.inc +################# +# Relay log checks +################# +connection server_2; +include/sync_io_with_master.inc +NOT FOUND /_to_encrypt/ in slave-relay-bin.0* +NOT FOUND /COMMIT/ in slave-relay-bin.0* +NOT FOUND /TIMESTAMP/ in slave-relay-bin.0* +################# +# Slave binlog checks +################# +include/start_slave.inc +include/sync_slave_sql_with_io.inc +include/sync_io_with_master.inc +NOT FOUND /_to_encrypt/ in slave-bin.0* +NOT FOUND /COMMIT/ in slave-bin.0* +NOT FOUND /TIMESTAMP/ in slave-bin.0* +########## +# Cleanup +########## +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_slave.test b/mysql-test/suite/binlog_encryption/encrypted_slave.test new file mode 100644 index 0000000000000..a69e78cd94074 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_slave.test @@ -0,0 +1,117 @@ +# +# The test checks that basic DDL and DML events are encrypted +# in the relay and binary logs on slave. +# The test is to be run with all binlog formats +# (combinations for rpl_init.inc take care of that). +# +# The test runs with the non-encrypted master and encrypted slave. +# It generates a sequence of events on master and checks that +# relay logs and binary logs are encrypted on slave. +# + +--source encryption_algorithms.inc +--source include/have_innodb.inc + +--echo ################# +--echo # Initialization +--echo ################# + +--let $rpl_topology= 1->2 +--source include/rpl_init.inc + +--enable_connect_log +--connection server_2 + +# We stop SQL thread because we want to have +# all relay logs at the end of the test flow + +--disable_connect_log +--source include/stop_slave_sql.inc +--enable_connect_log + +--echo ################# +--echo # Test flow +--echo ################# + +--connection server_1 +--source testdata.inc + +--echo ################# +--echo # Master binlog checks +--echo ################# + +--let $master_datadir= `SELECT @@datadir` + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= _to_encrypt +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= COMMIT +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= TIMESTAMP +--source include/search_pattern_in_file.inc + +--disable_connect_log +--source include/save_master_pos.inc +--enable_connect_log + +--echo ################# +--echo # Relay log checks +--echo ################# + +--connection server_2 +--disable_connect_log +--source include/sync_io_with_master.inc +--enable_connect_log + +--let $slave_datadir= `SELECT @@datadir` + +--let SEARCH_FILE= $slave_datadir/slave-relay-bin.0* +--let SEARCH_PATTERN= _to_encrypt +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-relay-bin.0* +--let SEARCH_PATTERN= COMMIT +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-relay-bin.0* +--let SEARCH_PATTERN= TIMESTAMP +--source include/search_pattern_in_file.inc + +--echo ################# +--echo # Slave binlog checks +--echo ################# + +# Re-enable SQL thread, let it catch up with IO thread +# and check slave binary logs + +--disable_connect_log +--source include/start_slave.inc +--source include/sync_slave_sql_with_io.inc +--enable_connect_log + +--disable_connect_log +--source include/sync_io_with_master.inc +--enable_connect_log + +--let SEARCH_FILE= $slave_datadir/slave-bin.0* +--let SEARCH_PATTERN= _to_encrypt +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-bin.0* +--let SEARCH_PATTERN= COMMIT +--source include/search_pattern_in_file.inc + +--let SEARCH_FILE= $slave_datadir/slave-bin.0* +--let SEARCH_PATTERN= TIMESTAMP +--source include/search_pattern_in_file.inc + +--echo ########## +--echo # Cleanup +--echo ########## + +--disable_connect_log +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encryption_algorithms.combinations b/mysql-test/suite/binlog_encryption/encryption_algorithms.combinations new file mode 100644 index 0000000000000..6bda5a6b35e23 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encryption_algorithms.combinations @@ -0,0 +1,5 @@ +[ctr] +loose-file-key-management-encryption-algorithm=aes_ctr + +[cbc] +loose-file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/binlog_encryption/encryption_algorithms.inc b/mysql-test/suite/binlog_encryption/encryption_algorithms.inc new file mode 100644 index 0000000000000..ca559622b2611 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encryption_algorithms.inc @@ -0,0 +1,2 @@ +# Empty include file just to enable encryption algorithm combinations +# for those tests which need them diff --git a/mysql-test/suite/binlog_encryption/encryption_combo.cnf b/mysql-test/suite/binlog_encryption/encryption_combo.cnf new file mode 100644 index 0000000000000..bc4ecbcb47a99 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encryption_combo.cnf @@ -0,0 +1,5 @@ +!include my.cnf + +[mysqld.1] +encrypt-binlog=0 + diff --git a/mysql-test/suite/binlog_encryption/encryption_combo.result b/mysql-test/suite/binlog_encryption/encryption_combo.result new file mode 100644 index 0000000000000..d921c73440dc9 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encryption_combo.result @@ -0,0 +1,78 @@ +################# +# Initialization +################# +include/rpl_init.inc [topology=1->2] +connection server_2; +include/stop_slave.inc +##################################################### +# Part 1: unencrypted master +##################################################### +connection server_1; +CREATE TABLE table1_no_encryption ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table1_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +FOUND /table1_no_encryption/ in master-bin.0* +##################################################### +# Part 2: restart master, now with binlog encryption +##################################################### +connection default; +connection server_1; +CREATE TABLE table2_to_encrypt ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +NOT FOUND /table2_to_encrypt/ in master-bin.0* +##################################################### +# Part 3: restart master again without encryption +##################################################### +connection default; +connection server_1; +CREATE TABLE table3_no_encryption ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table3_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; +##################################################### +# Check: resume replication and check that it works +##################################################### +connection server_2; +include/start_slave.inc +SHOW TABLES; +Tables_in_test +table1_no_encryption +table2_to_encrypt +table3_no_encryption +########## +# Cleanup +########## +connection server_1; +SELECT COUNT(*) FROM table1_no_encryption; +COUNT(*) +8 +SELECT COUNT(*) FROM table2_to_encrypt; +COUNT(*) +8 +SELECT COUNT(*) FROM table3_no_encryption; +COUNT(*) +4 +DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption; +connection server_2; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encryption_combo.test b/mysql-test/suite/binlog_encryption/encryption_combo.test new file mode 100644 index 0000000000000..a5cf117d4a8a4 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encryption_combo.test @@ -0,0 +1,136 @@ +# +# The test checks that master with decryption capabilities can switch +# between encrypted and unencrypted logs (with server restart), +# and can feed the mix of encrypted/unencrypted logs to a slave. +# +# The test starts with unencrypted master. +# It stops replication, generates a few statement and row events +# on the master, then restarts the server with encrypted binlog, +# generates some more events and restarts it back with unencrypted binlog. +# Then it resumes replication and checks that all events +# are replicated successfully. +# + +--source include/have_binlog_format_mixed.inc + +--echo ################# +--echo # Initialization +--echo ################# + +--let $rpl_topology= 1->2 +--source include/rpl_init.inc + +--enable_connect_log + +# We stop replication because we want it to happen after the switch + +--connection server_2 +--disable_connect_log +--source include/stop_slave.inc +--enable_connect_log + +--echo ##################################################### +--echo # Part 1: unencrypted master +--echo ##################################################### + +--connection server_1 + +CREATE TABLE table1_no_encryption ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table1_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; + +# Make sure that binary logs are not encrypted + +--let $master_datadir= `SELECT @@datadir` + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= table1_no_encryption +--source include/search_pattern_in_file.inc + +--echo ##################################################### +--echo # Part 2: restart master, now with binlog encryption +--echo ##################################################### + +--let $rpl_server_parameters= --encrypt-binlog=1 +--let $rpl_server_number= 1 +--source restart_server.inc + +CREATE TABLE table2_to_encrypt ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; + +# Make sure that binary logs are encrypted + +--let SEARCH_FILE= $master_datadir/master-bin.0* +--let SEARCH_PATTERN= table2_to_encrypt +--source include/search_pattern_in_file.inc + +--echo ##################################################### +--echo # Part 3: restart master again without encryption +--echo ##################################################### + +--let $rpl_server_parameters= --encrypt-binlog=0 +--let $rpl_server_number= 1 +--source restart_server.inc + +CREATE TABLE table3_no_encryption ( + pk INT AUTO_INCREMENT PRIMARY KEY, + ts TIMESTAMP NULL, + b BLOB +) ENGINE=MyISAM; + +INSERT INTO table3_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; + +--save_master_pos + +--echo ##################################################### +--echo # Check: resume replication and check that it works +--echo ##################################################### +--connection server_2 + +--disable_connect_log +--source include/start_slave.inc +--enable_connect_log +--sync_with_master + +--sorted_result +SHOW TABLES; + +--echo ########## +--echo # Cleanup +--echo ########## + +--connection server_1 + +SELECT COUNT(*) FROM table1_no_encryption; +SELECT COUNT(*) FROM table2_to_encrypt; +SELECT COUNT(*) FROM table3_no_encryption; +DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption; + +--save_master_pos + +--connection server_2 +--sync_with_master + +--disable_connect_log +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/multisource.cnf b/mysql-test/suite/binlog_encryption/multisource.cnf new file mode 100644 index 0000000000000..52db51d90860a --- /dev/null +++ b/mysql-test/suite/binlog_encryption/multisource.cnf @@ -0,0 +1,17 @@ +!include my.cnf + +[mysqld.1] +log-bin=master-bin + +[mysqld.2] +log-bin=master-bin + +[mysqld.3] +innodb +log-bin=slave-bin +server-id=3 +log-warnings=2 + +[ENV] +SERVER_MYPORT_3= @mysqld.3.port +SERVER_MYSOCK_3= @mysqld.3.socket diff --git a/mysql-test/suite/binlog_encryption/multisource.result b/mysql-test/suite/binlog_encryption/multisource.result new file mode 100644 index 0000000000000..227e88f6d71fb --- /dev/null +++ b/mysql-test/suite/binlog_encryption/multisource.result @@ -0,0 +1,205 @@ +change master 'abc' to relay_log_file=''; +ERROR HY000: Failed initializing relay log position: Could not find target log during relay log initialization +change master 'abc2' to master_host=''; +ERROR HY000: Incorrect arguments to MASTER_HOST +change master 'master1' to +master_port=MYPORT_1, +master_host='127.0.0.1', +master_user='root'; +start slave 'master1'; +set default_master_connection = 'master1'; +include/wait_for_slave_to_start.inc +# +# Checking SHOW SLAVE 'master1' STATUS +# +Master_Port = 'MYPORT_1' +Relay_Log_File = 'mysqld-relay-bin-master1.000002' +Slave_IO_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Last_Errno = '0' +Last_SQL_Errno = '0' +# +# Checking SHOW SLAVE STATUS +# +Master_Port = 'MYPORT_1' +Relay_Log_File = 'mysqld-relay-bin-master1.000002' +Slave_IO_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Last_Errno = '0' +Last_SQL_Errno = '0' +# +# Checking SHOW ALL SLAVES STATUS +# +Connection_name = 'master1' +Master_Port = 'MYPORT_1' +Relay_Log_File = 'mysqld-relay-bin-master1.000002' +Slave_IO_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Last_Errno = '0' +Last_SQL_Errno = '0' +Slave_heartbeat_period = '60.000' +# +drop database if exists db1; +create database db1; +use db1; +create table t1 (i int auto_increment, f1 varchar(16), primary key pk (i,f1)) engine=MyISAM; +insert into t1 (f1) values ('one'),('two'); +select * from db1.t1; +i f1 +1 one +2 two +# List of relay log files in the datadir +mysqld-relay-bin-master1.000001 +mysqld-relay-bin-master1.000002 +mysqld-relay-bin-master1.index +include/show_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +mysqld-relay-bin-master1.000001 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +mysqld-relay-bin-master1.000001 # Rotate # # mysqld-relay-bin-master1.000002;pos=4 +include/show_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +mysqld-relay-bin-master1.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +mysqld-relay-bin-master1.000002 # Rotate # # master-bin.000001;pos=POS +mysqld-relay-bin-master1.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +mysqld-relay-bin-master1.000002 # Gtid_list # # [] +mysqld-relay-bin-master1.000002 # Binlog_checkpoint # # master-bin.000001 +mysqld-relay-bin-master1.000002 # Gtid # # GTID #-#-# +mysqld-relay-bin-master1.000002 # Query # # drop database if exists db1 +mysqld-relay-bin-master1.000002 # Gtid # # GTID #-#-# +mysqld-relay-bin-master1.000002 # Query # # create database db1 +mysqld-relay-bin-master1.000002 # Gtid # # GTID #-#-# +mysqld-relay-bin-master1.000002 # Query # # use `db1`; create table t1 (i int auto_increment, f1 varchar(16), primary key pk (i,f1)) engine=MyISAM +mysqld-relay-bin-master1.000002 # Gtid # # BEGIN GTID #-#-# +mysqld-relay-bin-master1.000002 # Intvar # # INSERT_ID=1 +mysqld-relay-bin-master1.000002 # Query # # use `db1`; insert into t1 (f1) values ('one'),('two') +mysqld-relay-bin-master1.000002 # Query # # COMMIT +change master 'master1' to +master_port=MYPORT_2, +master_host='127.0.0.1', +master_user='root'; +ERROR HY000: This operation cannot be performed as you have a running slave 'master1'; run STOP SLAVE 'master1' first +change master to +master_port=MYPORT_2, +master_host='127.0.0.1', +master_user='root'; +ERROR HY000: This operation cannot be performed as you have a running slave 'master1'; run STOP SLAVE 'master1' first +change master 'master2' to +master_port=MYPORT_1, +master_host='127.0.0.1', +master_user='root'; +ERROR HY000: Connection 'master2' conflicts with existing connection 'master1' +set default_master_connection = ''; +change master to +master_port=MYPORT_2, +master_host='127.0.0.1', +master_user='root'; +start slave; +include/wait_for_slave_to_start.inc +# +# Checking SHOW ALL SLAVES STATUS +# +Connection_name = '' +Connection_name = 'master1' +Master_Port = 'MYPORT_2' +Master_Port = 'MYPORT_1' +Relay_Log_File = 'mysqld-relay-bin.000002' +Relay_Log_File = 'mysqld-relay-bin-master1.000002' +Slave_IO_Running = 'Yes' +Slave_IO_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Last_Errno = '0' +Last_Errno = '0' +Last_SQL_Errno = '0' +Last_SQL_Errno = '0' +Slave_heartbeat_period = '60.000' +Slave_heartbeat_period = '60.000' +# +insert into t1 (f1) values ('three'); +drop database if exists db2; +create database db2; +use db2; +create table t1 (pk int auto_increment primary key, f1 int) engine=InnoDB; +begin; +insert into t1 (f1) values (1),(2); +select * from db1.t1; +i f1 +1 one +2 two +3 three +select * from db2.t1; +pk f1 +commit; +select * from db2.t1; +pk f1 +1 1 +2 2 +flush logs; +purge binary logs to 'master-bin.000002'; +show binary logs; +Log_name File_size +master-bin.000002 filesize +insert into t1 (f1) values ('four'); +create table db1.t3 (f1 int) engine=InnoDB; +# +# Checking SHOW ALL SLAVES STATUS +# +Connection_name = '' +Connection_name = 'master1' +Master_Port = 'MYPORT_2' +Master_Port = 'MYPORT_1' +Relay_Log_File = 'mysqld-relay-bin.000002' +Relay_Log_File = 'mysqld-relay-bin-master1.000004' +Slave_IO_Running = 'Yes' +Slave_IO_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Slave_SQL_Running = 'Yes' +Last_Errno = '0' +Last_Errno = '0' +Last_SQL_Errno = '0' +Last_SQL_Errno = '0' +Slave_heartbeat_period = '60.000' +Slave_heartbeat_period = '60.000' +# +select * from db1.t1; +i f1 +1 one +2 two +3 three +4 four +include/show_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +mysqld-relay-bin.000001 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +mysqld-relay-bin.000001 # Rotate # # mysqld-relay-bin.000002;pos=4 +include/show_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +mysqld-relay-bin.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +mysqld-relay-bin.000002 # Rotate # # master-bin.000001;pos=POS +mysqld-relay-bin.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +mysqld-relay-bin.000002 # Gtid_list # # [] +mysqld-relay-bin.000002 # Binlog_checkpoint # # master-bin.000001 +mysqld-relay-bin.000002 # Gtid # # GTID #-#-# +mysqld-relay-bin.000002 # Query # # drop database if exists db2 +mysqld-relay-bin.000002 # Gtid # # GTID #-#-# +mysqld-relay-bin.000002 # Query # # create database db2 +mysqld-relay-bin.000002 # Gtid # # GTID #-#-# +mysqld-relay-bin.000002 # Query # # use `db2`; create table t1 (pk int auto_increment primary key, f1 int) engine=InnoDB +mysqld-relay-bin.000002 # Gtid # # BEGIN GTID #-#-# +mysqld-relay-bin.000002 # Intvar # # INSERT_ID=1 +mysqld-relay-bin.000002 # Query # # use `db2`; insert into t1 (f1) values (1),(2) +mysqld-relay-bin.000002 # Xid # # COMMIT /* XID */ +stop slave io_thread; +show status like 'Slave_running'; +Variable_name Value +Slave_running OFF +set default_master_connection = 'master1'; +show status like 'Slave_running'; +Variable_name Value +Slave_running ON +drop database db1; +drop database db2; +include/reset_master_slave.inc +drop database db1; +include/reset_master_slave.inc +drop database db2; +include/reset_master_slave.inc diff --git a/mysql-test/suite/binlog_encryption/multisource.test b/mysql-test/suite/binlog_encryption/multisource.test new file mode 100644 index 0000000000000..e37970d62346a --- /dev/null +++ b/mysql-test/suite/binlog_encryption/multisource.test @@ -0,0 +1,2 @@ +--let $binlog_extra_length= 36 +--source extra/rpl_tests/multisource.inc diff --git a/mysql-test/suite/binlog_encryption/my.cnf b/mysql-test/suite/binlog_encryption/my.cnf new file mode 100644 index 0000000000000..d787ebe1d4c95 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/my.cnf @@ -0,0 +1,27 @@ +!include include/default_mysqld.cnf +!include include/default_client.cnf + +[mysqld.1] +innodb +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename= @ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog +log-basename= master + +[mysqld.2] +#!use-slave-opt +innodb +log-slave-updates +log-basename= slave + +[ENV] + +# We will adopt tests with master-slave setup as well as rpl_init setup, +# so need both sets of variables +MASTER_MYPORT= @mysqld.1.port +SERVER_MYPORT_1= @mysqld.1.port +SERVER_MYSOCK_1= @mysqld.1.socket + +SLAVE_MYPORT= @mysqld.2.port +SERVER_MYPORT_2= @mysqld.2.port +SERVER_MYSOCK_2= @mysqld.2.socket diff --git a/mysql-test/suite/binlog_encryption/restart_server.inc b/mysql-test/suite/binlog_encryption/restart_server.inc new file mode 100644 index 0000000000000..6cd0788cf4390 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/restart_server.inc @@ -0,0 +1,35 @@ +# +# We can not use the common include/restart_mysqld.inc or include/rpl_restart_server.inc, +# because they have hardcoded connection names (master, master1) +# which are not initiated by rpl_init.inc. +# This is the relevant and simplified part of the same set of scripts. +# +# ==== Usage ==== +# +# --let $rpl_server_number= N +# Number to identify the server that needs to reconnect. +# 1 is the master server, 2 the slave server +# [--let $rpl_server_parameters= --flag1 --flag2 ...] +# --source restart_server.inc +# + +--let $_cur_con= $CURRENT_CONNECTION + +--connection default +--enable_reconnect + +--connection $_cur_con +--enable_reconnect +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.$rpl_server_number.expect + +shutdown_server 10; + +--source include/wait_until_disconnected.inc + +--let $_rpl_start_server_command= restart +if ($rpl_server_parameters) +{ + --let $_rpl_start_server_command= restart:$rpl_server_parameters +} +--exec echo "$_rpl_start_server_command" > $MYSQLTEST_VARDIR/tmp/mysqld.$rpl_server_number.expect +--source include/wait_until_connected_again.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_binlog_errors.cnf b/mysql-test/suite/binlog_encryption/rpl_binlog_errors.cnf new file mode 100644 index 0000000000000..2d3db66ebb5c1 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_binlog_errors.cnf @@ -0,0 +1,7 @@ +!include my.cnf + +[mysqld.1] +max_binlog_size=4096 + +[mysqld.2] +skip-slave-start diff --git a/mysql-test/suite/binlog_encryption/rpl_binlog_errors.result b/mysql-test/suite/binlog_encryption/rpl_binlog_errors.result new file mode 100644 index 0000000000000..9fcfa35204b40 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_binlog_errors.result @@ -0,0 +1,275 @@ +include/master-slave.inc +[connection master] +####################################################################### +####################### PART 1: MASTER TESTS ########################## +####################################################################### +include/stop_slave.inc +call mtr.add_suppression("Can't generate a unique log-filename"); +call mtr.add_suppression("Writing one row to the row-based binary log failed.*"); +call mtr.add_suppression("Error writing file .*"); +SET @old_debug= @@global.debug; +SELECT repeat('x',8192) INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data'; +SELECT repeat('x',10) INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug_46166-2.data'; +RESET MASTER; +###################### TEST #1 +FLUSH LOGS; +# assert: must show two binlogs +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +###################### TEST #2 +RESET MASTER; +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +FLUSH LOGS; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +# assert: must show one binlog +show binary logs; +Log_name File_size +master-bin.000001 # +SET GLOBAL debug_dbug=@old_debug; +RESET MASTER; +###################### TEST #3 +CREATE TABLE t1 (a INT); +CREATE TABLE t2 (a VARCHAR(16384)) Engine=InnoDB; +CREATE TABLE t4 (a VARCHAR(16384)); +INSERT INTO t1 VALUES (1); +RESET MASTER; +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t2; +# assert: must show two binlog +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; +###################### TEST #4 +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t2; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +# assert: must show one entry +SELECT count(*) FROM t2; +count(*) +1 +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; +###################### TEST #5 +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166-2.data' INTO TABLE t2; +# assert: must show one entry +SELECT count(*) FROM t2; +count(*) +1 +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; +###################### TEST #6 +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +SET AUTOCOMMIT=0; +INSERT INTO t2 VALUES ('muse'); +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t2; +INSERT INTO t2 VALUES ('muse'); +COMMIT; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +# assert: must show three entries +SELECT count(*) FROM t2; +count(*) +3 +SET AUTOCOMMIT= 1; +SET GLOBAL debug_dbug=@old_debug; +DELETE FROM t2; +RESET MASTER; +###################### TEST #7 +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +SELECT count(*) FROM t4; +count(*) +0 +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t4; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +# assert: must show 1 entry +SELECT count(*) FROM t4; +count(*) +1 +### check that the incident event is written to the current log +SET GLOBAL debug_dbug=@old_debug; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Incident # # #1 (LOST_EVENTS) +DELETE FROM t4; +RESET MASTER; +###################### TEST #8 +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +# must show 0 entries +SELECT count(*) FROM t4; +count(*) +0 +SELECT count(*) FROM t2; +count(*) +0 +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t4; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t2; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +INSERT INTO t2 VALUES ('aaa'), ('bbb'), ('ccc'); +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +# INFO: Count(*) Before Offending DELETEs +# assert: must show 1 entry +SELECT count(*) FROM t4; +count(*) +1 +# assert: must show 4 entries +SELECT count(*) FROM t2; +count(*) +4 +DELETE FROM t4; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +DELETE FROM t2; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +# INFO: Count(*) After Offending DELETEs +# assert: must show zero entries +SELECT count(*) FROM t4; +count(*) +0 +SELECT count(*) FROM t2; +count(*) +0 +SET GLOBAL debug_dbug=@old_debug; +###################### TEST #9 +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +SET SQL_LOG_BIN=0; +INSERT INTO t2 VALUES ('aaa'), ('bbb'), ('ccc'), ('ddd'); +INSERT INTO t4 VALUES ('eee'), ('fff'), ('ggg'), ('hhh'); +# assert: must show four entries +SELECT count(*) FROM t2; +count(*) +4 +SELECT count(*) FROM t4; +count(*) +4 +DELETE FROM t2; +DELETE FROM t4; +# assert: must show zero entries +SELECT count(*) FROM t2; +count(*) +0 +SELECT count(*) FROM t4; +count(*) +0 +SET SQL_LOG_BIN=1; +SET GLOBAL debug_dbug=@old_debug; +###################### TEST #10 +call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("Could not open .*"); +RESET MASTER; +SHOW WARNINGS; +Level Code Message +SET GLOBAL debug_dbug="+d,fault_injection_registering_index"; +FLUSH LOGS; +ERROR HY000: Can't open file: 'master-bin.000002' (errno: 1 "Operation not permitted") +SET GLOBAL debug_dbug="-d,fault_injection_registering_index"; +SHOW BINARY LOGS; +ERROR HY000: You are not using binary logging +CREATE TABLE t5 (a INT); +INSERT INTO t4 VALUES ('bbbbb'); +INSERT INTO t2 VALUES ('aaaaa'); +DELETE FROM t4; +DELETE FROM t2; +DROP TABLE t5; +###################### TEST #11 +include/rpl_restart_server.inc [server_number=1] +SET GLOBAL debug_dbug="+d,fault_injection_openning_index"; +FLUSH LOGS; +ERROR HY000: Can't open file: 'master-bin.index' (errno: 1 "Operation not permitted") +SET GLOBAL debug_dbug="-d,fault_injection_openning_index"; +RESET MASTER; +ERROR HY000: Binlog closed, cannot RESET MASTER +CREATE TABLE t5 (a INT); +INSERT INTO t4 VALUES ('bbbbb'); +INSERT INTO t2 VALUES ('aaaaa'); +DELETE FROM t4; +DELETE FROM t2; +DROP TABLE t5; +include/rpl_restart_server.inc [server_number=1] +###################### TEST #12 +SET GLOBAL debug_dbug="+d,fault_injection_new_file_rotate_event"; +FLUSH LOGS; +ERROR HY000: Can't open file: 'master-bin' (errno: 2 "No such file or directory") +SET GLOBAL debug_dbug="-d,fault_injection_new_file_rotate_event"; +RESET MASTER; +ERROR HY000: Binlog closed, cannot RESET MASTER +CREATE TABLE t5 (a INT); +INSERT INTO t4 VALUES ('bbbbb'); +INSERT INTO t2 VALUES ('aaaaa'); +DELETE FROM t4; +DELETE FROM t2; +DROP TABLE t5; +include/rpl_restart_server.inc [server_number=1] +DROP TABLE t1, t2, t4; +RESET MASTER; +include/start_slave.inc +####################################################################### +####################### PART 2: SLAVE TESTS ########################### +####################################################################### +include/rpl_reset.inc +call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); +call mtr.add_suppression("Error writing file .*"); +call mtr.add_suppression("Could not open .*"); +call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("Can't generate a unique log-filename .*"); +###################### TEST #13 +SET @old_debug=@@global.debug; +include/stop_slave.inc +SET GLOBAL debug_dbug="+d,error_unique_log_filename"; +START SLAVE io_thread; +include/wait_for_slave_io_error.inc [errno=1595] +Last_IO_Error = 'Relay log write failure: could not queue event from master' +SET GLOBAL debug_dbug="-d,error_unique_log_filename"; +SET GLOBAL debug_dbug=@old_debug; +include/rpl_restart_server.inc [server_number=2] +###################### TEST #14 +SET @old_debug=@@global.debug; +include/stop_slave.inc +SET GLOBAL debug_dbug="+d,fault_injection_new_file_rotate_event"; +START SLAVE io_thread; +include/wait_for_slave_io_error.inc [errno=1595] +Last_IO_Error = 'Relay log write failure: could not queue event from master' +SET GLOBAL debug_dbug="-d,fault_injection_new_file_rotate_event"; +SET GLOBAL debug_dbug=@old_debug; +include/rpl_restart_server.inc [server_number=2] +###################### TEST #15 +SET @old_debug=@@global.debug; +include/stop_slave.inc +SET GLOBAL debug_dbug="+d,fault_injection_registering_index"; +START SLAVE io_thread; +include/wait_for_slave_io_error.inc [errno=1595] +Last_IO_Error = 'Relay log write failure: could not queue event from master' +SET GLOBAL debug_dbug="-d,fault_injection_registering_index"; +SET GLOBAL debug_dbug=@old_debug; +include/rpl_restart_server.inc [server_number=2] +###################### TEST #16 +SET @old_debug=@@global.debug; +include/stop_slave.inc +SET GLOBAL debug_dbug="+d,fault_injection_openning_index"; +START SLAVE io_thread; +include/wait_for_slave_io_error.inc [errno=1595] +Last_IO_Error = 'Relay log write failure: could not queue event from master' +SET GLOBAL debug_dbug="-d,fault_injection_openning_index"; +SET GLOBAL debug_dbug=@old_debug; +include/rpl_restart_server.inc [server_number=2] +include/stop_slave_sql.inc +Warnings: +Note 1255 Slave already has been stopped +RESET SLAVE; +RESET MASTER; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_binlog_errors.test b/mysql-test/suite/binlog_encryption/rpl_binlog_errors.test new file mode 100644 index 0000000000000..a4611f515c4a9 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_binlog_errors.test @@ -0,0 +1,2 @@ +--let $binlog_limit= 5,1 +--source extra/rpl_tests/rpl_binlog_errors.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.result b/mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.result new file mode 100644 index 0000000000000..0cc1e4b505792 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.result @@ -0,0 +1,21 @@ +include/master-slave.inc +[connection master] +include/stop_slave.inc +call mtr.add_suppression("Error in Log_event::read_log_event()"); +include/rpl_stop_server.inc [server_number=1] +include/rpl_start_server.inc [server_number=1] +show binlog events; +ERROR HY000: Error when executing command SHOW BINLOG EVENTS: Wrong offset or I/O error +call mtr.add_suppression("Slave I/O: Got fatal error 1236 from master when reading data from binary log"); +reset slave; +start slave; +include/wait_for_slave_param.inc [Last_IO_Errno] +Last_IO_Errno = '1236' +Last_IO_Error = 'Got fatal error 1236 from master when reading data from binary log: 'binlog truncated in the middle of event; consider out of disk space on master; the first event '.' at XXX, the last event read from 'master-bin.000001' at XXX, the last byte read from 'master-bin.000001' at XXX.'' +reset master; +stop slave; +reset slave; +drop table if exists t; +reset master; +End of the tests +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.test b/mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.test new file mode 100644 index 0000000000000..6d222cba1151c --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_cant_read_event_incident.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_cant_read_event_incident.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_checksum.cnf b/mysql-test/suite/binlog_encryption/rpl_checksum.cnf new file mode 100644 index 0000000000000..9d7ada8c656fa --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_checksum.cnf @@ -0,0 +1,10 @@ +!include my.cnf + +[mysqld.1] +binlog-checksum=CRC32 + +[mysqld.2] +binlog-checksum=CRC32 +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename=@ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_checksum.result b/mysql-test/suite/binlog_encryption/rpl_checksum.result new file mode 100644 index 0000000000000..b4ea8d9b1aeea --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_checksum.result @@ -0,0 +1,161 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression('Slave can not handle replication events with the checksum that master is configured to log'); +call mtr.add_suppression('Replication event checksum verification failed'); +call mtr.add_suppression('Relay log write failure: could not queue event from master'); +call mtr.add_suppression('Master is configured to log replication events with checksum, but will not send such events to slaves that cannot process them'); +set @master_save_binlog_checksum= @@global.binlog_checksum; +set @save_master_verify_checksum = @@global.master_verify_checksum; +select @@global.binlog_checksum as 'must be CRC32 because of the command line option'; +must be CRC32 because of the command line option +CRC32 +select @@session.binlog_checksum as 'no session var'; +ERROR HY000: Variable 'binlog_checksum' is a GLOBAL variable +select @@global.master_verify_checksum as 'must be zero because of default'; +must be zero because of default +0 +select @@session.master_verify_checksum as 'no session var'; +ERROR HY000: Variable 'master_verify_checksum' is a GLOBAL variable +set @slave_save_binlog_checksum= @@global.binlog_checksum; +set @save_slave_sql_verify_checksum = @@global.slave_sql_verify_checksum; +select @@global.slave_sql_verify_checksum as 'must be one because of default'; +must be one because of default +1 +select @@session.slave_sql_verify_checksum as 'no session var'; +ERROR HY000: Variable 'slave_sql_verify_checksum' is a GLOBAL variable +show binary logs; +Log_name File_size +master-bin.000001 # +set @@global.binlog_checksum = NONE; +select @@global.binlog_checksum; +@@global.binlog_checksum +NONE +*** must be rotations seen *** +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +set @@global.binlog_checksum = default; +select @@global.binlog_checksum; +@@global.binlog_checksum +NONE +set @@global.binlog_checksum = CRC32; +select @@global.binlog_checksum; +@@global.binlog_checksum +CRC32 +set @@global.binlog_checksum = CRC32; +set @@global.master_verify_checksum = 0; +set @@global.master_verify_checksum = default; +set @@global.binlog_checksum = ADLER32; +ERROR 42000: Variable 'binlog_checksum' can't be set to the value of 'ADLER32' +set @@global.master_verify_checksum = 2; +ERROR 42000: Variable 'master_verify_checksum' can't be set to the value of '2' +set @@global.slave_sql_verify_checksum = 0; +set @@global.slave_sql_verify_checksum = default; +set @@global.slave_sql_verify_checksum = 2; +ERROR 42000: Variable 'slave_sql_verify_checksum' can't be set to the value of '2' +set @@global.binlog_checksum = NONE; +create table t1 (a int); +flush logs; +flush logs; +flush logs; +flush logs; +flush logs; +flush logs; +select count(*) as zero from t1; +zero +0 +include/stop_slave.inc +set @@global.binlog_checksum = CRC32; +insert into t1 values (1) /* will not be applied on slave due to simulation */; +set @@global.debug_dbug='d,simulate_slave_unaware_checksum'; +start slave; +include/wait_for_slave_io_error.inc [errno=1236] +Last_IO_Error = 'Got fatal error 1236 from master when reading data from binary log: 'Slave can not handle replication events with the checksum that master is configured to log; the first event 'master-bin.000009' at 404, the last event read from 'master-bin.000010' at 4, the last byte read from 'master-bin.000010' at 249.'' +select count(*) as zero from t1; +zero +0 +set @@global.debug_dbug=''; +include/start_slave.inc +set @@global.master_verify_checksum = 1; +set @@session.debug_dbug='d,simulate_checksum_test_failure'; +show binlog events; +ERROR HY000: Error when executing command SHOW BINLOG EVENTS: Wrong offset or I/O error +set @@session.debug_dbug=''; +set @@global.master_verify_checksum = default; +include/stop_slave.inc +create table t2 (a int); +set @@global.debug_dbug='d,simulate_checksum_test_failure'; +start slave io_thread; +include/wait_for_slave_io_error.inc [errno=1595,1913] +set @@global.debug_dbug=''; +start slave io_thread; +include/wait_for_slave_param.inc [Read_Master_Log_Pos] +set @@global.slave_sql_verify_checksum = 1; +set @@global.debug_dbug='d,simulate_checksum_test_failure'; +start slave sql_thread; +include/wait_for_slave_sql_error.inc [errno=1593] +Last_SQL_Error = 'Error initializing relay log position: I/O error reading event at position 4' +set @@global.debug_dbug=''; +include/start_slave.inc +select count(*) as 'must be zero' from t2; +must be zero +0 +stop slave; +reset slave; +set @@global.binlog_checksum= IF(floor((rand()*1000)%2), "CRC32", "NONE"); +flush logs; +set @@global.binlog_checksum= CRC32; +reset master; +flush logs; +create table t3 (a int, b char(5)); +include/start_slave.inc +select count(*) as 'must be zero' from t3; +must be zero +0 +include/stop_slave.inc +change master to master_host='127.0.0.1',master_port=MASTER_PORT, master_user='root'; +flush logs; +reset master; +insert into t3 value (1, @@global.binlog_checksum); +include/start_slave.inc +flush logs; +select count(*) as 'must be one' from t3; +must be one +1 +set @@global.binlog_checksum= IF(floor((rand()*1000)%2), "CRC32", "NONE"); +insert into t3 value (1, @@global.binlog_checksum); +drop table t1, t2, t3; +set @@global.binlog_checksum = @master_save_binlog_checksum; +set @@global.master_verify_checksum = @save_master_verify_checksum; +*** Bug#59123 / MDEV-5799: INCIDENT_EVENT checksum written to error log as garbage characters *** +CREATE TABLE t4 (a INT PRIMARY KEY); +INSERT INTO t4 VALUES (1); +SET sql_log_bin=0; +CALL mtr.add_suppression("\\[ERROR\\] Can't generate a unique log-filename"); +SET sql_log_bin=1; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET debug_dbug= '+d,binlog_inject_new_name_error'; +FLUSH LOGS; +ERROR HY000: Can't generate a unique log-filename master-bin.(1-999) + +SET debug_dbug= @old_dbug; +INSERT INTO t4 VALUES (2); +include/wait_for_slave_sql_error.inc [errno=1590] +Last_SQL_Error = 'The incident LOST_EVENTS occurred on the master. Message: error writing to the binary log' +FOUND /Slave SQL: The incident LOST_EVENTS occurred on the master\. Message: error writing to the binary log, Internal MariaDB error code: 1590/ in mysqld.2.err +SELECT * FROM t4 ORDER BY a; +a +1 +STOP SLAVE IO_THREAD; +SET sql_slave_skip_counter= 1; +include/start_slave.inc +SELECT * FROM t4 ORDER BY a; +a +1 +2 +set @@global.binlog_checksum = @slave_save_binlog_checksum; +set @@global.slave_sql_verify_checksum = @save_slave_sql_verify_checksum; +End of tests +DROP TABLE t4; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_checksum.test b/mysql-test/suite/binlog_encryption/rpl_checksum.test new file mode 100644 index 0000000000000..8e006b1b6a0c5 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_checksum.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_checksum.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_checksum_cache.result b/mysql-test/suite/binlog_encryption/rpl_checksum_cache.result new file mode 100644 index 0000000000000..9508e94e7f21c --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_checksum_cache.result @@ -0,0 +1,119 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. Statement: insert into t2 set data=repeat.*'a', @act_size.*"); +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. Statement: insert into t1 values.* NAME_CONST.*'n',.*, @data .*"); +set @save_binlog_cache_size = @@global.binlog_cache_size; +set @save_binlog_checksum = @@global.binlog_checksum; +set @save_master_verify_checksum = @@global.master_verify_checksum; +set @@global.binlog_cache_size = 4096; +set @@global.binlog_checksum = CRC32; +set @@global.master_verify_checksum = 1; +include/stop_slave.inc +include/start_slave.inc +flush status; +show status like "binlog_cache_use"; +Variable_name Value +Binlog_cache_use 0 +show status like "binlog_cache_disk_use"; +Variable_name Value +Binlog_cache_disk_use 0 +drop table if exists t1; +create table t1 (a int PRIMARY KEY, b CHAR(32)) engine=innodb; +create procedure test.p_init (n int, size int) +begin +while n > 0 do +select round(RAND() * size) into @act_size; +set @data = repeat('a', @act_size); +insert into t1 values(n, @data ); +set n= n-1; +end while; +end| +begin; +call test.p_init(4000, 32); +commit; +show status like "binlog_cache_use"; +Variable_name Value +Binlog_cache_use 1 +*** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; +Variable_name Value +Binlog_cache_disk_use 1 +include/diff_tables.inc [master:test.t1, slave:test.t1] +begin; +delete from t1; +commit; +flush status; +create table t2(a int auto_increment primary key, data VARCHAR(12288)) ENGINE=Innodb; +show status like "binlog_cache_use"; +Variable_name Value +Binlog_cache_use 1 +*** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; +Variable_name Value +Binlog_cache_disk_use 1 +include/diff_tables.inc [master:test.t2, slave:test.t2] +begin; +delete from t2; +commit; +flush status; +create table t3(a int auto_increment primary key, data VARCHAR(8192)) engine=innodb; +show status like "binlog_cache_use"; +Variable_name Value +Binlog_cache_use 1 +*** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; +Variable_name Value +Binlog_cache_disk_use 1 +include/diff_tables.inc [master:test.t3, slave:test.t3] +begin; +delete from t3; +commit; +flush status; +create procedure test.p1 (n int) +begin +while n > 0 do +case (select (round(rand()*100) % 3) + 1) +when 1 then +select round(RAND() * 32) into @act_size; +set @data = repeat('a', @act_size); +insert into t1 values(n, @data); +when 2 then +begin +select round(8192 + RAND() * 4096) into @act_size; +insert into t2 set data=repeat('a', @act_size); +end; +when 3 then +begin +select round(3686.4000 + RAND() * 819.2000) into @act_size; +insert into t3 set data= repeat('a', @act_size); +end; +end case; +set n= n-1; +end while; +end| +set autocommit= 0; +begin; +call test.p1(1000); +commit; +show status like "binlog_cache_use"; +Variable_name Value +Binlog_cache_use 1 +*** binlog_cache_disk_use must be non-zero *** +show status like "binlog_cache_disk_use"; +Variable_name Value +Binlog_cache_disk_use 1 +include/diff_tables.inc [master:test.t1, slave:test.t1] +include/diff_tables.inc [master:test.t2, slave:test.t2] +include/diff_tables.inc [master:test.t3, slave:test.t3] +begin; +delete from t1; +delete from t2; +delete from t3; +commit; +drop table t1, t2, t3; +set @@global.binlog_cache_size = @save_binlog_cache_size; +set @@global.binlog_checksum = @save_binlog_checksum; +set @@global.master_verify_checksum = @save_master_verify_checksum; +drop procedure test.p_init; +drop procedure test.p1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_checksum_cache.test b/mysql-test/suite/binlog_encryption/rpl_checksum_cache.test new file mode 100644 index 0000000000000..56c3e1e1cb5f1 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_checksum_cache.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_checksum_cache.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_corruption.cnf b/mysql-test/suite/binlog_encryption/rpl_corruption.cnf new file mode 100644 index 0000000000000..7f7d0eeb8ea5d --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_corruption.cnf @@ -0,0 +1,9 @@ +!include my.cnf + +[mysqld.1] +binlog-checksum=CRC32 +master-verify-checksum=1 + +[mysqld.2] +binlog-checksum=CRC32 +slave-sql-verify-checksum=1 diff --git a/mysql-test/suite/binlog_encryption/rpl_corruption.result b/mysql-test/suite/binlog_encryption/rpl_corruption.result new file mode 100644 index 0000000000000..51c2c6261b8db --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_corruption.result @@ -0,0 +1,51 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression('Found invalid event in binary log'); +call mtr.add_suppression('Slave I/O: Relay log write failure: could not queue event from master'); +call mtr.add_suppression('event read from binlog did not pass crc check'); +call mtr.add_suppression('Replication event checksum verification failed'); +call mtr.add_suppression('Event crc check failed! Most likely there is event corruption'); +call mtr.add_suppression('Slave SQL: Error initializing relay log position: I/O error reading event at position .*, error.* 1593'); +SET @old_master_verify_checksum = @@master_verify_checksum; +# 1. Creating test table/data and set corruption position for testing +* insert/update/delete rows in table t1 * +CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY, b VARCHAR(10), c VARCHAR(100)); +include/stop_slave.inc +# 2. Corruption in master binlog and SHOW BINLOG EVENTS +SET GLOBAL debug_dbug="+d,corrupt_read_log_event_char"; +SHOW BINLOG EVENTS; +ERROR HY000: Error when executing command SHOW BINLOG EVENTS: Wrong offset or I/O error +SET GLOBAL debug_dbug="-d,corrupt_read_log_event_char"; +# 3. Master read a corrupted event from binlog and send the error to slave +SET GLOBAL debug_dbug="+d,corrupt_read_log_event2_set"; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1236] +SET GLOBAL debug_dbug="-d,corrupt_read_log_event2_set"; +# 4. Master read a corrupted event from binlog and send it to slave +SET GLOBAL master_verify_checksum=0; +SET GLOBAL debug_dbug="+d,corrupt_read_log_event2_set"; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595,1913] +SET GLOBAL debug_dbug="-d,corrupt_read_log_event2_set"; +SET GLOBAL debug_dbug= ""; +SET GLOBAL master_verify_checksum=1; +# 5. Slave. Corruption in network +SET GLOBAL debug_dbug="+d,corrupt_queue_event"; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595,1913] +SET GLOBAL debug_dbug="-d,corrupt_queue_event"; +# 6. Slave. Corruption in relay log +SET GLOBAL debug_dbug="+d,corrupt_read_log_event_char"; +START SLAVE SQL_THREAD; +include/wait_for_slave_sql_error.inc [errno=1593] +SET GLOBAL debug_dbug="-d,corrupt_read_log_event_char"; +SET GLOBAL debug_dbug= ""; +# 7. Seek diff for tables on master and slave +include/start_slave.inc +include/diff_tables.inc [master:test.t1, slave:test.t1] +# 8. Clean up +SET GLOBAL debug_dbug= ""; +SET GLOBAL master_verify_checksum = @old_master_verify_checksum; +DROP TABLE t1; +SET GLOBAL debug_dbug= ""; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_corruption.test b/mysql-test/suite/binlog_encryption/rpl_corruption.test new file mode 100644 index 0000000000000..310b0cef8e8f3 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_corruption.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_corruption.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_gtid_basic.cnf b/mysql-test/suite/binlog_encryption/rpl_gtid_basic.cnf new file mode 100644 index 0000000000000..ae47ef7a1fcdc --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_gtid_basic.cnf @@ -0,0 +1,24 @@ +!include my.cnf + +[mysqld.1] +log-slave-updates + +[mysqld.2] +init-rpl-role= slave +master-retry-count= 10 +skip-slave-start + +[mysqld.3] +log-slave-updates +innodb + +[mysqld.4] +log-slave-updates +innodb + +[ENV] +SERVER_MYPORT_3= @mysqld.3.port +SERVER_MYSOCK_3= @mysqld.3.socket + +SERVER_MYPORT_4= @mysqld.4.port +SERVER_MYSOCK_4= @mysqld.4.socket diff --git a/mysql-test/suite/binlog_encryption/rpl_gtid_basic.result b/mysql-test/suite/binlog_encryption/rpl_gtid_basic.result new file mode 100644 index 0000000000000..8562818b623cd --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_gtid_basic.result @@ -0,0 +1,481 @@ +include/rpl_init.inc [topology=1->2->3->4] +*** GTID position should be empty here *** +SELECT BINLOG_GTID_POS('',); +BINLOG_GTID_POS('',) + +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=MyISAM; +CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, "m1"); +INSERT INTO t1 VALUES (2, "m2"), (3, "m3"), (4, "m4"); +INSERT INTO t2 VALUES (1, "i1"); +BEGIN; +INSERT INTO t2 VALUES (2, "i2"), (3, "i3"); +INSERT INTO t2 VALUES (4, "i4"); +COMMIT; +*** GTID position should be non-empty here *** +SELECT BINLOG_GTID_POS('',); +BINLOG_GTID_POS('',) + +*** GTID position should be the same as on server_1 *** +SELECT BINLOG_GTID_POS('',); +BINLOG_GTID_POS('',) + +SELECT * FROM t1 ORDER BY a; +a b +1 m1 +2 m2 +3 m3 +4 m4 +SELECT * FROM t2 ORDER BY a; +a b +1 i1 +2 i2 +3 i3 +4 i4 +SELECT * FROM t1 ORDER BY a; +a b +1 m1 +2 m2 +3 m3 +4 m4 +SELECT * FROM t2 ORDER BY a; +a b +1 i1 +2 i2 +3 i3 +4 i4 +SELECT * FROM t1 ORDER BY a; +a b +1 m1 +2 m2 +3 m3 +4 m4 +SELECT * FROM t2 ORDER BY a; +a b +1 i1 +2 i2 +3 i3 +4 i4 +*** Now take out D, let it fall behind a bit, and then test re-attaching it to A *** +include/stop_slave.inc +INSERT INTO t1 VALUES (5, "m1a"); +INSERT INTO t2 VALUES (5, "i1a"); +CHANGE MASTER TO master_host = '127.0.0.1', master_port = MASTER_PORT, +MASTER_USE_GTID=CURRENT_POS; +include/start_slave.inc +SELECT * FROM t1 ORDER BY a; +a b +1 m1 +2 m2 +3 m3 +4 m4 +5 m1a +SELECT * FROM t2 ORDER BY a; +a b +1 i1 +2 i2 +3 i3 +4 i4 +5 i1a +*** Now move B to D (C is still replicating from B) *** +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = SERVER_MYPORT_4, +MASTER_USE_GTID=CURRENT_POS; +include/start_slave.inc +UPDATE t2 SET b="j1a" WHERE a=5; +SELECT * FROM t1 ORDER BY a; +a b +1 m1 +2 m2 +3 m3 +4 m4 +5 m1a +SELECT * FROM t2 ORDER BY a; +a b +1 i1 +2 i2 +3 i3 +4 i4 +5 j1a +*** Now move C to D, after letting it fall a little behind *** +include/stop_slave.inc +INSERT INTO t2 VALUES (6, "i6b"); +INSERT INTO t2 VALUES (7, "i7b"); +include/save_master_gtid.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = SERVER_MYPORT_4, +MASTER_USE_GTID=CURRENT_POS; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t2 ORDER BY a; +a b +1 i1 +2 i2 +3 i3 +4 i4 +5 j1a +6 i6b +7 i7b +*** Now change everything back to what it was, to make rpl_end.inc happy +include/sync_with_master_gtid.inc +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = MASTER_MYPORT; +include/start_slave.inc +include/wait_for_slave_to_start.inc +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = SLAVE_MYPORT; +include/start_slave.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = SERVER_MYPORT_3; +include/start_slave.inc +DROP TABLE t1,t2; +include/save_master_gtid.inc +*** A few more checks for BINLOG_GTID_POS function *** +SELECT BINLOG_GTID_POS(); +ERROR 42000: Incorrect parameter count in the call to native function 'BINLOG_GTID_POS' +SELECT BINLOG_GTID_POS('a'); +ERROR 42000: Incorrect parameter count in the call to native function 'BINLOG_GTID_POS' +SELECT BINLOG_GTID_POS('a',1,NULL); +ERROR 42000: Incorrect parameter count in the call to native function 'BINLOG_GTID_POS' +SELECT BINLOG_GTID_POS(1,'a'); +BINLOG_GTID_POS(1,'a') +NULL +Warnings: +Warning 1292 Truncated incorrect INTEGER value: 'a' +SELECT BINLOG_GTID_POS(NULL,NULL); +BINLOG_GTID_POS(NULL,NULL) +NULL +SELECT BINLOG_GTID_POS('',1); +BINLOG_GTID_POS('',1) + +SELECT BINLOG_GTID_POS('a',1); +BINLOG_GTID_POS('a',1) +NULL +SELECT BINLOG_GTID_POS('master-bin.000001',-1); +BINLOG_GTID_POS('master-bin.000001',-1) +NULL +SELECT BINLOG_GTID_POS('master-bin.000001',0); +BINLOG_GTID_POS('master-bin.000001',0) + +SELECT BINLOG_GTID_POS('master-bin.000001',18446744073709551615); +BINLOG_GTID_POS('master-bin.000001',18446744073709551615) +NULL +SELECT BINLOG_GTID_POS('master-bin.000001',18446744073709551616); +BINLOG_GTID_POS('master-bin.000001',18446744073709551616) +NULL +Warnings: +Warning 1916 Got overflow when converting '18446744073709551616' to INT. Value truncated. +*** Some tests of @@GLOBAL.gtid_binlog_state *** +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET @old_state= @@GLOBAL.gtid_binlog_state; +SET GLOBAL gtid_binlog_state = ''; +ERROR HY000: This operation is not allowed if any GTID has been logged to the binary log. Run RESET MASTER first to erase the log +RESET MASTER; +SET GLOBAL gtid_binlog_state = ''; +FLUSH LOGS; +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +SET GLOBAL gtid_binlog_state = '0-1-10,1-2-20,0-3-30'; +show binary logs; +Log_name File_size +master-bin.000001 # +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000001 # Start_encryption # # +master-bin.000001 # Gtid_list # # [#-#-#] +master-bin.000001 # Binlog_checkpoint # # master-bin.000001 +SET GLOBAL gtid_binlog_state = @old_state; +ERROR HY000: This operation is not allowed if any GTID has been logged to the binary log. Run RESET MASTER first to erase the log +RESET MASTER; +SET GLOBAL gtid_binlog_state = @old_state; +CREATE TABLE t1 (a INT PRIMARY KEY); +SET gtid_seq_no=100; +INSERT INTO t1 VALUES (1); +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1; +a +1 +Gtid_IO_Pos = '0-1-100' +*** Test @@LAST_GTID and MASTER_GTID_WAIT() *** +DROP TABLE t1; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +include/stop_slave.inc +SELECT @@last_gtid; +@@last_gtid + +SET gtid_seq_no=110; +SELECT @@last_gtid; +@@last_gtid + +BEGIN; +SELECT @@last_gtid; +@@last_gtid + +INSERT INTO t1 VALUES (2); +SELECT @@last_gtid; +@@last_gtid + +COMMIT; +SELECT @@last_gtid; +@@last_gtid +0-1-110 +SET @pos= '0-1-110'; +SELECT master_gtid_wait(NULL); +master_gtid_wait(NULL) +NULL +SELECT master_gtid_wait('', NULL); +master_gtid_wait('', NULL) +0 +SHOW STATUS LIKE 'Master_gtid_wait_count'; +Variable_name Value +Master_gtid_wait_count 1 +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +Variable_name Value +Master_gtid_wait_timeouts 0 +SHOW STATUS LIKE 'Master_gtid_wait_time'; +Variable_name Value +Master_gtid_wait_time 0 +SELECT master_gtid_wait(@pos, 0.5); +master_gtid_wait(@pos, 0.5) +-1 +SELECT * FROM t1 ORDER BY a; +a +SELECT master_gtid_wait(@pos); +include/start_slave.inc +master_gtid_wait(@pos) +0 +SELECT * FROM t1 ORDER BY a; +a +2 +include/stop_slave.inc +SET gtid_domain_id= 1; +INSERT INTO t1 VALUES (3); +SET @pos= 'POS'; +SELECT master_gtid_wait(@pos, 0); +master_gtid_wait(@pos, 0) +-1 +SELECT * FROM t1 WHERE a >= 3; +a +SELECT master_gtid_wait(@pos, -1); +include/start_slave.inc +master_gtid_wait(@pos, -1) +0 +SELECT * FROM t1 WHERE a >= 3; +a +3 +SELECT master_gtid_wait('1-1-1', 0); +master_gtid_wait('1-1-1', 0) +0 +SELECT master_gtid_wait('2-1-1,1-1-4,0-1-110'); +SELECT master_gtid_wait('0-1-1000', 0.5); +SELECT master_gtid_wait('0-1-2000'); +SELECT master_gtid_wait('2-1-10'); +SELECT master_gtid_wait('2-1-6', 1); +SELECT master_gtid_wait('2-1-5'); +SELECT master_gtid_wait('2-1-10'); +SELECT master_gtid_wait('2-1-5,1-1-4,0-1-110'); +SELECT master_gtid_wait('2-1-2'); +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +Variable_name Value +Master_gtid_wait_timeouts 0 +SHOW STATUS LIKE 'Master_gtid_wait_count'; +Variable_name Value +Master_gtid_wait_count 3 +SELECT master_gtid_wait('1-1-1'); +master_gtid_wait('1-1-1') +0 +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +Variable_name Value +Master_gtid_wait_timeouts 0 +SHOW STATUS LIKE 'Master_gtid_wait_count'; +Variable_name Value +Master_gtid_wait_count 4 +SET @a= MASTER_GTID_WAIT_TIME; +SELECT IF(@a <= 100*1000*1000, "OK", CONCAT("Error: wait time ", @a, " is larger than expected")) +AS Master_gtid_wait_time_as_expected; +Master_gtid_wait_time_as_expected +OK +SELECT master_gtid_wait('0-1-109'); +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +Variable_name Value +Master_gtid_wait_timeouts 0 +SHOW STATUS LIKE 'Master_gtid_wait_count'; +Variable_name Value +Master_gtid_wait_count 4 +SELECT master_gtid_wait('2-1-2', 0.5); +master_gtid_wait('2-1-2', 0.5) +-1 +SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; +Variable_name Value +Master_gtid_wait_timeouts 1 +SHOW STATUS LIKE 'Master_gtid_wait_count'; +Variable_name Value +Master_gtid_wait_count 5 +SET @a= MASTER_GTID_WAIT_TIME; +SELECT IF(@a BETWEEN 0.4*1000*1000 AND 100*1000*1000, "OK", CONCAT("Error: wait time ", @a, " not as expected")) AS Master_gtid_wait_time_as_expected; +Master_gtid_wait_time_as_expected +OK +KILL QUERY KILL_ID; +ERROR 70100: Query execution was interrupted +SET gtid_domain_id=2; +SET gtid_seq_no=2; +INSERT INTO t1 VALUES (4); +master_gtid_wait('2-1-2') +0 +KILL CONNECTION KILL_ID; +Got one of the listed errors +SET gtid_domain_id=1; +SET gtid_seq_no=4; +INSERT INTO t1 VALUES (5); +SET gtid_domain_id=2; +SET gtid_seq_no=5; +INSERT INTO t1 VALUES (6); +master_gtid_wait('2-1-5,1-1-4,0-1-110') +0 +master_gtid_wait('2-1-1,1-1-4,0-1-110') +0 +master_gtid_wait('0-1-1000', 0.5) +-1 +master_gtid_wait('2-1-6', 1) +-1 +master_gtid_wait('0-1-109') +0 +SET gtid_domain_id=2; +SET gtid_seq_no=10; +INSERT INTO t1 VALUES (7); +master_gtid_wait('2-1-10') +0 +master_gtid_wait('2-1-10') +0 +*** Test gtid_slave_pos when used with GTID *** +include/stop_slave.inc +SET gtid_domain_id=2; +SET gtid_seq_no=1000; +INSERT INTO t1 VALUES (10); +INSERT INTO t1 VALUES (11); +SET sql_slave_skip_counter= 1; +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +a +11 +SELECT IF(LOCATE("2-1-1001", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1001 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +include/stop_slave.inc +SET gtid_domain_id=2; +SET gtid_seq_no=1010; +INSERT INTO t1 VALUES (12); +INSERT INTO t1 VALUES (13); +SET sql_slave_skip_counter= 2; +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +a +11 +13 +SELECT IF(LOCATE("2-1-1011", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1011 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +include/stop_slave.inc +SET gtid_domain_id=2; +SET gtid_seq_no=1020; +INSERT INTO t1 VALUES (14); +INSERT INTO t1 VALUES (15); +INSERT INTO t1 VALUES (16); +SET sql_slave_skip_counter= 3; +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +a +11 +13 +15 +16 +SELECT IF(LOCATE("2-1-1022", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1022 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +include/stop_slave.inc +SET gtid_domain_id=2; +SET gtid_seq_no=1030; +INSERT INTO t1 VALUES (17); +INSERT INTO t1 VALUES (18); +INSERT INTO t1 VALUES (19); +SET sql_slave_skip_counter= 5; +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 10 ORDER BY a; +a +11 +13 +15 +16 +19 +SELECT IF(LOCATE("2-1-1032", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1032 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +include/stop_slave.inc +SET gtid_domain_id=3; +SET gtid_seq_no=100; +CREATE TABLE t2 (a INT PRIMARY KEY); +DROP TABLE t2; +SET gtid_domain_id=2; +SET gtid_seq_no=1040; +INSERT INTO t1 VALUES (20); +SET @saved_mode= @@GLOBAL.slave_ddl_exec_mode; +SET GLOBAL slave_ddl_exec_mode=STRICT; +SET sql_slave_skip_counter=1; +START SLAVE UNTIL master_gtid_pos="3-1-100"; +include/sync_with_master_gtid.inc +include/wait_for_slave_to_stop.inc +SELECT * FROM t2; +ERROR 42S02: Table 'test.t2' doesn't exist +SELECT IF(LOCATE("3-1-100", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-100 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +SET sql_log_bin=0; +CALL mtr.add_suppression("Slave: Unknown table 'test\\.t2' Error_code: 1051"); +SET sql_log_bin=1; +START SLAVE; +include/wait_for_slave_sql_error.inc [errno=1051] +SELECT IF(LOCATE("3-1-100", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-100 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +STOP SLAVE IO_THREAD; +SET sql_slave_skip_counter=2; +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 20 ORDER BY a; +a +20 +SELECT IF(LOCATE("3-1-101", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-101 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +SELECT IF(LOCATE("2-1-1040", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1040 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; +status +Ok +SET GLOBAL slave_ddl_exec_mode= @saved_mode; +*** Test GTID-connecting to a master with out-of-order sequence numbers in the binlog. *** +SET gtid_domain_id= @@GLOBAL.gtid_domain_id; +INSERT INTO t1 VALUES (31); +SET gtid_domain_id= @@GLOBAL.gtid_domain_id; +INSERT INTO t1 VALUES (32); +INSERT INTO t1 VALUES (33); +include/stop_slave.inc +INSERT INTO t1 VALUES (34); +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; +a +31 +32 +33 +34 +SELECT * FROM t1 WHERE a >= 30 ORDER BY a; +a +31 +32 +33 +34 +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_gtid_basic.test b/mysql-test/suite/binlog_encryption/rpl_gtid_basic.test new file mode 100644 index 0000000000000..70be6fbff1fd2 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_gtid_basic.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_gtid_basic.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_incident.cnf b/mysql-test/suite/binlog_encryption/rpl_incident.cnf new file mode 100644 index 0000000000000..7294976f6e2c6 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_incident.cnf @@ -0,0 +1,7 @@ +!include my.cnf + +[mysqld.1] +binlog_checksum=NONE + +[mysqld.2] +binlog_checksum=NONE diff --git a/mysql-test/suite/binlog_encryption/rpl_incident.result b/mysql-test/suite/binlog_encryption/rpl_incident.result new file mode 100644 index 0000000000000..7cb8168c6a96f --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_incident.result @@ -0,0 +1,38 @@ +include/master-slave.inc +[connection master] +**** On Master **** +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +SELECT * FROM t1; +a +1 +2 +3 +SET GLOBAL debug_dbug= '+d,incident_database_resync_on_replace,*'; +REPLACE INTO t1 VALUES (4); +SELECT * FROM t1; +a +1 +2 +3 +4 +call mtr.add_suppression("Slave SQL.*The incident LOST_EVENTS occurred on the master.* 1590"); +include/wait_for_slave_sql_error.inc [errno=1590] +Last_SQL_Error = 'The incident LOST_EVENTS occurred on the master. Message: ' +**** On Slave **** +SELECT * FROM t1; +a +1 +2 +3 +SET GLOBAL SQL_SLAVE_SKIP_COUNTER=1; +START SLAVE; +SELECT * FROM t1; +a +1 +2 +3 +4 +include/check_slave_is_running.inc +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_incident.test b/mysql-test/suite/binlog_encryption/rpl_incident.test new file mode 100644 index 0000000000000..9be855e1a8b17 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_incident.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_incident.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_init_slave_errors.result b/mysql-test/suite/binlog_encryption/rpl_init_slave_errors.result new file mode 100644 index 0000000000000..57c0bb129fab7 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_init_slave_errors.result @@ -0,0 +1,19 @@ +include/master-slave.inc +[connection master] +stop slave; +reset slave; +SET GLOBAL debug_dbug= "d,simulate_io_slave_error_on_init,simulate_sql_slave_error_on_init"; +start slave; +include/wait_for_slave_sql_error.inc [errno=1593] +Last_SQL_Error = 'Failed during slave thread initialization' +call mtr.add_suppression("Failed during slave.* thread initialization"); +SET GLOBAL debug_dbug= ""; +reset slave; +SET GLOBAL init_slave= "garbage"; +start slave; +include/wait_for_slave_sql_error.inc [errno=1064] +Last_SQL_Error = 'Slave SQL thread aborted. Can't execute init_slave query' +SET GLOBAL init_slave= ""; +include/stop_slave_io.inc +RESET SLAVE; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_init_slave_errors.test b/mysql-test/suite/binlog_encryption/rpl_init_slave_errors.test new file mode 100644 index 0000000000000..6f515b9390a4a --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_init_slave_errors.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_init_slave_errors.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_loaddata_local.result b/mysql-test/suite/binlog_encryption/rpl_loaddata_local.result new file mode 100644 index 0000000000000..16fd3f19c4aff --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_loaddata_local.result @@ -0,0 +1,125 @@ +include/master-slave.inc +[connection master] +create table t1(a int); +select * into outfile 'MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; +truncate table t1; +load data local infile 'MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; +select a,count(*) from t1 group by a; +a count(*) +1 10000 +drop table t1; +create table t1(a int); +insert into t1 values (1), (2), (2), (3); +select * into outfile 'MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; +drop table t1; +create table t1(a int primary key); +load data local infile 'MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; +Warnings: +Warning 1062 Duplicate entry '2' for key 'PRIMARY' +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +drop table t1; +==== Bug22504 Initialize ==== +[on master] +SET sql_mode='ignore_space'; +CREATE TABLE t1(a int); +insert into t1 values (1), (2), (3), (4); +select * into outfile 'MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; +truncate table t1; +load data local infile 'MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +[on slave] +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +==== Clean up ==== +[on master] +DROP TABLE t1; +[on slave] + +Bug #43746: +"return wrong query string when parse 'load data infile' sql statement" + +[master] +SELECT @@SESSION.sql_mode INTO @old_mode; +SET sql_mode='ignore_space'; +CREATE TABLE t1(a int); +INSERT INTO t1 VALUES (1), (2), (3), (4); +SELECT * INTO OUTFILE 'MYSQLD_DATADIR/bug43746.sql' FROM t1; +TRUNCATE TABLE t1; +LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; +LOAD/* look mum, with comments in weird places! */DATA/* oh hai */LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql'/* we are */INTO/* from the internets */TABLE t1; +LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; +LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' /*!10000 INTO */ TABLE t1; +LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' /*!10000 INTO TABLE */ t1; +LOAD DATA /*!10000 LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO TABLE */ t1; +LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql'/*!10000 INTO*/TABLE t1; +LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql'/* empty */INTO TABLE t1; +LOAD DATA/*!10000 LOCAL */INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO/* empty */TABLE t1; +LOAD/*!999999 special comments that do not expand */DATA/*!999999 code from the future */LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql'/*!999999 have flux capacitor */INTO/*!999999 will travel */TABLE t1; +SET sql_mode='PIPES_AS_CONCAT,ANSI_QUOTES,NO_KEY_OPTIONS,NO_TABLE_OPTIONS,NO_FIELD_OPTIONS,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER'; +LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; +[slave] + +Bug #59267: +"LOAD DATA LOCAL INFILE not executed on slave with SBR" + +[master] +SELECT * INTO OUTFILE 'MYSQLD_DATADIR/bug59267.sql' FROM t1; +TRUNCATE TABLE t1; +LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug59267.sql' INTO TABLE t1; +SELECT 'Master', COUNT(*) FROM t1; +Master COUNT(*) +Master 44 +[slave] +SELECT 'Slave', COUNT(*) FROM t1; +Slave COUNT(*) +Slave 44 +[master] +DROP TABLE t1; +SET SESSION sql_mode=@old_mode; +[slave] + +Bug #60580/#11902767: +"statement improperly replicated crashes slave sql thread" + +[master] +CREATE TABLE t1(f1 INT, f2 INT); +CREATE TABLE t2(f1 INT, f2 TIMESTAMP); +INSERT INTO t2 VALUES(1, '2011-03-22 21:01:28'); +INSERT INTO t2 VALUES(2, '2011-03-21 21:01:28'); +INSERT INTO t2 VALUES(3, '2011-03-20 21:01:28'); +CREATE TABLE t3 AS SELECT * FROM t2; +CREATE VIEW v1 AS SELECT * FROM t2 +WHERE f1 IN (SELECT f1 FROM t3 WHERE (t3.f2 IS NULL)); +SELECT 1 INTO OUTFILE 'MYSQLD_DATADIR/bug60580.csv' FROM DUAL; +LOAD DATA LOCAL INFILE 'MYSQLD_DATADIR/bug60580.csv' INTO TABLE t1 (@f1) SET f2 = (SELECT f1 FROM v1 WHERE f1=@f1); +SELECT * FROM t1; +f1 f2 +NULL NULL +[slave] +SELECT * FROM t1; +f1 f2 +NULL NULL +[master] +DROP VIEW v1; +DROP TABLE t1, t2, t3; +[slave] +include/rpl_end.inc +# End of 5.1 tests diff --git a/mysql-test/suite/binlog_encryption/rpl_loaddata_local.test b/mysql-test/suite/binlog_encryption/rpl_loaddata_local.test new file mode 100644 index 0000000000000..8d90afaed27fa --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_loaddata_local.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_loaddata_local.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_loadfile.result b/mysql-test/suite/binlog_encryption/rpl_loadfile.result new file mode 100644 index 0000000000000..25d50660bdef3 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_loadfile.result @@ -0,0 +1,242 @@ +include/master-slave.inc +[connection master] +DROP PROCEDURE IF EXISTS test.p1; +DROP TABLE IF EXISTS test.t1; +CREATE TABLE test.t1 (a INT, blob_column LONGBLOB, PRIMARY KEY(a)); +INSERT INTO test.t1 VALUES(1,'test'); +UPDATE test.t1 SET blob_column=LOAD_FILE('../../std_data/words2.dat') WHERE a=1; +create procedure test.p1() +begin +INSERT INTO test.t1 VALUES(2,'test'); +UPDATE test.t1 SET blob_column=LOAD_FILE('../../std_data/words2.dat') WHERE a=2; +end| +CALL test.p1(); +SELECT * FROM test.t1 ORDER BY blob_column; +a blob_column +1 abase +abased +abasement +abasements +abases +abash +abashed +abashes +abashing +abasing +abate +abated +abatement +abatements +abater +abates +abating +Abba +abbe +abbey +abbeys +abbot +abbots +Abbott +abbreviate +abbreviated +abbreviates +abbreviating +abbreviation +abbreviations +Abby +abdomen +abdomens +abdominal +abduct +abducted +abduction +abductions +abductor +abductors +abducts +Abe +abed +Abel +Abelian +Abelson +Aberdeen +Abernathy +aberrant +aberration + +2 abase +abased +abasement +abasements +abases +abash +abashed +abashes +abashing +abasing +abate +abated +abatement +abatements +abater +abates +abating +Abba +abbe +abbey +abbeys +abbot +abbots +Abbott +abbreviate +abbreviated +abbreviates +abbreviating +abbreviation +abbreviations +Abby +abdomen +abdomens +abdominal +abduct +abducted +abduction +abductions +abductor +abductors +abducts +Abe +abed +Abel +Abelian +Abelson +Aberdeen +Abernathy +aberrant +aberration + +SELECT * FROM test.t1 ORDER BY blob_column; +a blob_column +1 abase +abased +abasement +abasements +abases +abash +abashed +abashes +abashing +abasing +abate +abated +abatement +abatements +abater +abates +abating +Abba +abbe +abbey +abbeys +abbot +abbots +Abbott +abbreviate +abbreviated +abbreviates +abbreviating +abbreviation +abbreviations +Abby +abdomen +abdomens +abdominal +abduct +abducted +abduction +abductions +abductor +abductors +abducts +Abe +abed +Abel +Abelian +Abelson +Aberdeen +Abernathy +aberrant +aberration + +2 abase +abased +abasement +abasements +abases +abash +abashed +abashes +abashing +abasing +abate +abated +abatement +abatements +abater +abates +abating +Abba +abbe +abbey +abbeys +abbot +abbots +Abbott +abbreviate +abbreviated +abbreviates +abbreviating +abbreviation +abbreviations +Abby +abdomen +abdomens +abdominal +abduct +abducted +abduction +abductions +abductor +abductors +abducts +Abe +abed +Abel +Abelian +Abelson +Aberdeen +Abernathy +aberrant +aberration + +DROP PROCEDURE IF EXISTS test.p1; +DROP TABLE test.t1; +include/rpl_reset.inc +SELECT repeat('x',20) INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug_39701.data'; +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 (t text); +CREATE PROCEDURE p(file varchar(4096)) +BEGIN +INSERT INTO t1 VALUES (LOAD_FILE(file)); +END| +include/stop_slave.inc +CALL p('MYSQLTEST_VARDIR/tmp/bug_39701.data'); +include/start_slave.inc +include/diff_tables.inc [master:t1, slave:t1] +DROP TABLE t1; +DROP PROCEDURE p; +include/rpl_end.inc +# +# Check that the loaded data is encrypted in the master binlog +# +NOT FOUND /xxxxxxxxxxx/ in master-bin.0* diff --git a/mysql-test/suite/binlog_encryption/rpl_loadfile.test b/mysql-test/suite/binlog_encryption/rpl_loadfile.test new file mode 100644 index 0000000000000..97886ca0f4838 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_loadfile.test @@ -0,0 +1,11 @@ +--source extra/rpl_tests/rpl_loadfile.inc + +--let $datadir= `SELECT @@datadir` + +--echo # +--echo # Check that the loaded data is encrypted in the master binlog +--echo # + +--let SEARCH_FILE=$datadir/master-bin.0* +--let SEARCH_PATTERN= xxxxxxxxxxx +--source include/search_pattern_in_file.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result b/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result new file mode 100644 index 0000000000000..80f7616947283 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.result @@ -0,0 +1,186 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); +SET GLOBAL max_binlog_cache_size = 4096; +SET GLOBAL binlog_cache_size = 4096; +SET GLOBAL max_binlog_stmt_cache_size = 4096; +SET GLOBAL binlog_stmt_cache_size = 4096; +CREATE TABLE t1(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=Innodb; +CREATE TABLE t2(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=MyIsam; +CREATE TABLE t3(a INT PRIMARY KEY, data VARCHAR(30000)) ENGINE=Innodb; +######################################################################################## +# 1 - SINGLE STATEMENT +######################################################################################## +*** Single statement on transactional table *** +Got one of the listed errors +*** Single statement on non-transactional table *** +Got one of the listed errors +include/wait_for_slave_sql_error_and_skip.inc [errno=1590] +*** Single statement on both transactional and non-transactional tables. *** +Got one of the listed errors +include/wait_for_slave_sql_error_and_skip.inc [errno=1590] +include/diff_tables.inc [master:t1,slave:t1] +######################################################################################## +# 2 - BEGIN - IMPLICIT COMMIT by DDL +######################################################################################## +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +set default_storage_engine=innodb; +BEGIN; +Got one of the listed errors +Got one of the listed errors +Got one of the listed errors +INSERT INTO t1 (a, data) VALUES (7, 's');; +INSERT INTO t2 (a, data) VALUES (8, 's');; +INSERT INTO t1 (a, data) VALUES (9, 's');; +ALTER TABLE t3 ADD COLUMN d int; +BEGIN; +Got one of the listed errors +Got one of the listed errors +INSERT INTO t1 (a, data) VALUES (19, 's');; +INSERT INTO t2 (a, data) VALUES (20, 's');; +INSERT INTO t1 (a, data) VALUES (21, 's');; +CREATE TABLE t4 SELECT * FROM t1; +BEGIN; +Got one of the listed errors +Got one of the listed errors +INSERT INTO t1 (a, data) VALUES (27, 's');; +INSERT INTO t2 (a, data) VALUES (28, 's');; +INSERT INTO t1 (a, data) VALUES (29, 's');; +CREATE TABLE t5 (a int); +include/diff_tables.inc [master:t1,slave:t1] +######################################################################################## +# 3 - BEGIN - COMMIT +######################################################################################## +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +BEGIN; +Got one of the listed errors +Got one of the listed errors +Got one of the listed errors +COMMIT; +include/diff_tables.inc [master:t1,slave:t1] +######################################################################################## +# 4 - BEGIN - ROLLBACK +######################################################################################## +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +BEGIN; +Got one of the listed errors +Got one of the listed errors +Got one of the listed errors +ROLLBACK; +Warnings: +Warning 1196 Some non-transactional changed tables couldn't be rolled back +include/diff_tables.inc [master:t1,slave:t1] +######################################################################################## +# 5 - PROCEDURE +######################################################################################## +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +CREATE PROCEDURE p1(pd VARCHAR(30000)) +BEGIN +INSERT INTO t1 (a, data) VALUES (1, pd); +INSERT INTO t1 (a, data) VALUES (2, pd); +INSERT INTO t1 (a, data) VALUES (3, pd); +INSERT INTO t1 (a, data) VALUES (4, pd); +INSERT INTO t1 (a, data) VALUES (5, 's'); +END// +TRUNCATE TABLE t1; +TRUNCATE TABLE t1; +BEGIN; +Got one of the listed errors +COMMIT; +TRUNCATE TABLE t1; +BEGIN; +Got one of the listed errors +ROLLBACK; +include/diff_tables.inc [master:t1,slave:t1] +######################################################################################## +# 6 - XID +######################################################################################## +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +BEGIN; +Got one of the listed errors +Got one of the listed errors +Got one of the listed errors +INSERT INTO t1 (a, data) VALUES (7, 's');; +INSERT INTO t2 (a, data) VALUES (8, 's');; +INSERT INTO t1 (a, data) VALUES (9, 's');; +ROLLBACK TO sv; +Warnings: +Warning 1196 Some non-transactional changed tables couldn't be rolled back +COMMIT; +include/diff_tables.inc [master:t1,slave:t1] +######################################################################################## +# 7 - NON-TRANS TABLE +######################################################################################## +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +BEGIN; +Got one of the listed errors +Got one of the listed errors +Got one of the listed errors +INSERT INTO t1 (a, data) VALUES (8, 's');; +INSERT INTO t1 (a, data) VALUES (9, 's');; +INSERT INTO t2 (a, data) VALUES (10, 's');; +INSERT INTO t1 (a, data) VALUES (11, 's');; +COMMIT; +BEGIN; +Got one of the listed errors +COMMIT; +include/diff_tables.inc [master:t1,slave:t1] +######################################################################## +# 8 - Bug#55375(Regression Bug) Transaction bigger than +# max_binlog_cache_size crashes slave +######################################################################## +# [ On Slave ] +SET GLOBAL max_binlog_cache_size = 4096; +SET GLOBAL binlog_cache_size = 4096; +SET GLOBAL max_binlog_stmt_cache_size = 4096; +SET GLOBAL binlog_stmt_cache_size = 4096; +include/stop_slave.inc +include/start_slave.inc +CALL mtr.add_suppression("Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage.*"); +CALL mtr.add_suppression("Multi-statement transaction required more than 'max_binlog_stmt_cache_size' bytes of storage.*"); +CALL mtr.add_suppression("Writing one row to the row-based binary log failed.*"); +CALL mtr.add_suppression("Slave SQL.*The incident LOST_EVENTS occurred on the master. Message: error writing to the binary log"); +TRUNCATE t1; +SET GLOBAL max_binlog_cache_size= ORIGINAL_VALUE; +SET GLOBAL binlog_cache_size= ORIGINAL_VALUE; +SET GLOBAL max_binlog_stmt_cache_size= ORIGINAL_VALUE; +SET GLOBAL binlog_stmt_cache_size= ORIGINAL_VALUE; +BEGIN; +Repeat statement 'INSERT INTO t1 VALUES($n, repeat("a", 32))' 128 times +COMMIT; +include/wait_for_slave_sql_error.inc [errno=1197] +SELECT count(*) FROM t1; +count(*) +0 +include/show_binlog_events.inc +SET GLOBAL max_binlog_cache_size= ORIGINAL_VALUE; +SET GLOBAL binlog_cache_size= ORIGINAL_VALUE; +SET GLOBAL max_binlog_stmt_cache_size= ORIGINAL_VALUE; +SET GLOBAL binlog_stmt_cache_size= ORIGINAL_VALUE; +include/stop_slave.inc +include/start_slave.inc +SELECT count(*) FROM t1; +count(*) +128 +######################################################################################## +# CLEAN +######################################################################################## +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE IF EXISTS t4; +DROP TABLE t5; +DROP PROCEDURE p1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.test b/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.test new file mode 100644 index 0000000000000..8fb7350b8157c --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_mixed_binlog_max_cache_size.test @@ -0,0 +1,7 @@ +--source include/master-slave.inc +--source include/not_embedded.inc +--source include/not_windows.inc +--source include/have_binlog_format_mixed.inc + +--source extra/rpl_tests/rpl_binlog_max_cache_size.test +--source include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_packet.cnf b/mysql-test/suite/binlog_encryption/rpl_packet.cnf new file mode 100644 index 0000000000000..0f01aec743763 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_packet.cnf @@ -0,0 +1,10 @@ +!include my.cnf + +[mysqld.1] +max_allowed_packet=1024 +net_buffer_length=1024 + +[mysqld.2] +max_allowed_packet=1024 +net_buffer_length=1024 +slave_max_allowed_packet=1024 diff --git a/mysql-test/suite/binlog_encryption/rpl_packet.result b/mysql-test/suite/binlog_encryption/rpl_packet.result new file mode 100644 index 0000000000000..0755805328722 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_packet.result @@ -0,0 +1,58 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Slave I/O: Got a packet bigger than 'slave_max_allowed_packet' bytes, .*error.* 1153"); +call mtr.add_suppression("Log entry on master is longer than slave_max_allowed_packet"); +drop database if exists DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________; +create database DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________; +SET @@global.max_allowed_packet=1024; +SET @@global.net_buffer_length=1024; +include/stop_slave.inc +include/start_slave.inc +select @@net_buffer_length, @@max_allowed_packet; +@@net_buffer_length @@max_allowed_packet +1024 1024 +create table `t1` (`f1` LONGTEXT) ENGINE=MyISAM; +INSERT INTO `t1`(`f1`) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1023'); +select count(*) from `DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________`.`t1` /* must be 1 */; +count(*) +1 +SHOW STATUS LIKE 'Slave_running'; +Variable_name Value +Slave_running ON +select * from information_schema.session_status where variable_name= 'SLAVE_RUNNING'; +VARIABLE_NAME VARIABLE_VALUE +SLAVE_RUNNING ON +drop database DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________; +SET @@global.max_allowed_packet=4096; +SET @@global.net_buffer_length=4096; +include/stop_slave.inc +include/start_slave.inc +CREATE TABLE `t1` (`f1` LONGTEXT) ENGINE=MyISAM; +INSERT INTO `t1`(`f1`) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2048'); +include/wait_for_slave_io_error.inc [errno=1153] +Last_IO_Error = 'Got a packet bigger than 'slave_max_allowed_packet' bytes' +include/stop_slave_sql.inc +include/rpl_reset.inc +DROP TABLE t1; +CREATE TABLE t1 (f1 int PRIMARY KEY, f2 LONGTEXT, f3 LONGTEXT) ENGINE=MyISAM; +INSERT INTO t1(f1, f2, f3) VALUES(1, REPEAT('a', @@global.max_allowed_packet), REPEAT('b', @@global.max_allowed_packet)); +include/wait_for_slave_io_error.inc [errno=1153] +Last_IO_Error = 'Got a packet bigger than 'slave_max_allowed_packet' bytes' +STOP SLAVE; +RESET SLAVE; +RESET MASTER; +SET @max_allowed_packet_0= @@session.max_allowed_packet; +SHOW BINLOG EVENTS; +SET @max_allowed_packet_1= @@session.max_allowed_packet; +SHOW BINLOG EVENTS; +SET @max_allowed_packet_2= @@session.max_allowed_packet; +==== clean up ==== +DROP TABLE t1; +SET @@global.max_allowed_packet= 1024; +Warnings: +Warning 1708 The value of 'max_allowed_packet' should be no less than the value of 'net_buffer_length' +SET @@global.net_buffer_length= 1024; +SET @@global.slave_max_allowed_packet= 1073741824; +DROP TABLE t1; +RESET SLAVE; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_packet.test b/mysql-test/suite/binlog_encryption/rpl_packet.test new file mode 100644 index 0000000000000..31357cb148e5e --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_packet.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_packet.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_parallel.result b/mysql-test/suite/binlog_encryption/rpl_parallel.result new file mode 100644 index 0000000000000..34d23e6bf66f8 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_parallel.result @@ -0,0 +1,1774 @@ +include/master-slave.inc +[connection master] +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=10; +ERROR HY000: This operation cannot be performed as you have a running slave ''; run STOP SLAVE '' first +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +SELECT IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; +IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) +OK +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +SELECT IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; +IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) +OK +include/stop_slave.inc +SELECT IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; +IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) +OK +include/start_slave.inc +SELECT IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; +IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) +OK +*** Test long-running query in domain 1 can run in parallel with short queries in domain 0 *** +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=MyISAM; +CREATE TABLE t2 (a int PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1); +LOCK TABLE t1 WRITE; +SET gtid_domain_id=1; +INSERT INTO t1 VALUES (2); +SET gtid_domain_id=0; +INSERT INTO t2 VALUES (2); +INSERT INTO t2 VALUES (3); +BEGIN; +INSERT INTO t2 VALUES (4); +INSERT INTO t2 VALUES (5); +COMMIT; +INSERT INTO t2 VALUES (6); +SELECT * FROM t2 ORDER by a; +a +1 +2 +3 +4 +5 +6 +SELECT * FROM t1; +a +1 +UNLOCK TABLES; +SELECT * FROM t1 ORDER BY a; +a +1 +2 +*** Test two transactions in different domains committed in opposite order on slave but in a single group commit. *** +include/stop_slave.inc +SET sql_log_bin=0; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +RETURN x; +END +|| +SET sql_log_bin=1; +SET @old_format= @@SESSION.binlog_format; +SET binlog_format='statement'; +SET gtid_domain_id=1; +INSERT INTO t2 VALUES (foo(10, +'commit_before_enqueue SIGNAL ready1 WAIT_FOR cont1', +'commit_after_release_LOCK_prepare_ordered SIGNAL ready2')); +FLUSH LOGS; +SET sql_log_bin=0; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +IF d1 != '' THEN +SET debug_sync = d1; +END IF; +IF d2 != '' THEN +SET debug_sync = d2; +END IF; +RETURN x; +END +|| +SET sql_log_bin=1; +SET @old_format=@@GLOBAL.binlog_format; +SET GLOBAL binlog_format=statement; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +SET debug_sync='now WAIT_FOR ready1'; +SET gtid_domain_id=2; +INSERT INTO t2 VALUES (foo(11, +'commit_before_enqueue SIGNAL ready3 WAIT_FOR cont3', +'commit_after_release_LOCK_prepare_ordered SIGNAL ready4 WAIT_FOR cont4')); +SET gtid_domain_id=0; +SELECT * FROM t2 WHERE a >= 10 ORDER BY a; +a +10 +11 +SET debug_sync='now WAIT_FOR ready3'; +SET debug_sync='now SIGNAL cont3'; +SET debug_sync='now WAIT_FOR ready4'; +SET debug_sync='now SIGNAL cont1'; +SET debug_sync='now WAIT_FOR ready2'; +SET debug_sync='now SIGNAL cont4'; +SELECT * FROM t2 WHERE a >= 10 ORDER BY a; +a +10 +11 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +slave-bin.000002 # Binlog_checkpoint # # slave-bin.000002 +slave-bin.000002 # Gtid # # BEGIN GTID #-#-# cid=# +slave-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (foo(11, +'commit_before_enqueue SIGNAL ready3 WAIT_FOR cont3', +'commit_after_release_LOCK_prepare_ordered SIGNAL ready4 WAIT_FOR cont4')) +slave-bin.000002 # Xid # # COMMIT /* XID */ +slave-bin.000002 # Gtid # # BEGIN GTID #-#-# cid=# +slave-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (foo(10, +'commit_before_enqueue SIGNAL ready1 WAIT_FOR cont1', +'commit_after_release_LOCK_prepare_ordered SIGNAL ready2')) +slave-bin.000002 # Xid # # COMMIT /* XID */ +FLUSH LOGS; +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET debug_sync='RESET'; +include/start_slave.inc +*** Test that group-committed transactions on the master can replicate in parallel on the slave. *** +SET debug_sync='RESET'; +FLUSH LOGS; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t3 VALUES (1,1), (3,3), (5,5), (7,7); +BEGIN; +INSERT INTO t3 VALUES (2,102); +BEGIN; +INSERT INTO t3 VALUES (4,104); +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (2, foo(12, +'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued1 WAIT_FOR slave_cont1', +'')); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (4, foo(14, +'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued2', +'')); +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (6, foo(16, +'group_commit_waiting_for_prior SIGNAL slave_queued3', +'')); +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; +SET debug_sync='RESET'; +SELECT * FROM t3 ORDER BY a; +a b +1 1 +2 12 +3 3 +4 14 +5 5 +6 16 +7 7 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Binlog_checkpoint # # master-bin.000001 +master-bin.000002 # Binlog_checkpoint # # master-bin.000002 +master-bin.000002 # Gtid # # GTID #-#-# +master-bin.000002 # Query # # use `test`; CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB +master-bin.000002 # Gtid # # BEGIN GTID #-#-# +master-bin.000002 # Query # # use `test`; INSERT INTO t3 VALUES (1,1), (3,3), (5,5), (7,7) +master-bin.000002 # Xid # # COMMIT /* XID */ +master-bin.000002 # Gtid # # BEGIN GTID #-#-# cid=# +master-bin.000002 # Query # # use `test`; INSERT INTO t3 VALUES (2, foo(12, +'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued1 WAIT_FOR slave_cont1', +'')) +master-bin.000002 # Xid # # COMMIT /* XID */ +master-bin.000002 # Gtid # # BEGIN GTID #-#-# cid=# +master-bin.000002 # Query # # use `test`; INSERT INTO t3 VALUES (4, foo(14, +'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued2', +'')) +master-bin.000002 # Xid # # COMMIT /* XID */ +master-bin.000002 # Gtid # # BEGIN GTID #-#-# cid=# +master-bin.000002 # Query # # use `test`; INSERT INTO t3 VALUES (6, foo(16, +'group_commit_waiting_for_prior SIGNAL slave_queued3', +'')) +master-bin.000002 # Xid # # COMMIT /* XID */ +SET debug_sync='now WAIT_FOR slave_queued3'; +ROLLBACK; +SET debug_sync='now WAIT_FOR slave_queued1'; +ROLLBACK; +SET debug_sync='now WAIT_FOR slave_queued2'; +SET debug_sync='now SIGNAL slave_cont1'; +SELECT * FROM t3 ORDER BY a; +a b +1 1 +2 12 +3 3 +4 14 +5 5 +6 16 +7 7 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +slave-bin.000003 # Binlog_checkpoint # # slave-bin.000003 +slave-bin.000003 # Gtid # # GTID #-#-# +slave-bin.000003 # Query # # use `test`; CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB +slave-bin.000003 # Gtid # # BEGIN GTID #-#-# +slave-bin.000003 # Query # # use `test`; INSERT INTO t3 VALUES (1,1), (3,3), (5,5), (7,7) +slave-bin.000003 # Xid # # COMMIT /* XID */ +slave-bin.000003 # Gtid # # BEGIN GTID #-#-# cid=# +slave-bin.000003 # Query # # use `test`; INSERT INTO t3 VALUES (2, foo(12, +'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued1 WAIT_FOR slave_cont1', +'')) +slave-bin.000003 # Xid # # COMMIT /* XID */ +slave-bin.000003 # Gtid # # BEGIN GTID #-#-# cid=# +slave-bin.000003 # Query # # use `test`; INSERT INTO t3 VALUES (4, foo(14, +'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued2', +'')) +slave-bin.000003 # Xid # # COMMIT /* XID */ +slave-bin.000003 # Gtid # # BEGIN GTID #-#-# cid=# +slave-bin.000003 # Query # # use `test`; INSERT INTO t3 VALUES (6, foo(16, +'group_commit_waiting_for_prior SIGNAL slave_queued3', +'')) +slave-bin.000003 # Xid # # COMMIT /* XID */ +*** Test STOP SLAVE in parallel mode *** +include/stop_slave.inc +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET binlog_direct_non_transactional_updates=0; +SET sql_log_bin=0; +CALL mtr.add_suppression("Statement is unsafe because it accesses a non-transactional table after accessing a transactional table within the same transaction"); +SET sql_log_bin=1; +BEGIN; +INSERT INTO t2 VALUES (20); +INSERT INTO t1 VALUES (20); +INSERT INTO t2 VALUES (21); +INSERT INTO t3 VALUES (20, 20); +COMMIT; +INSERT INTO t3 VALUES(21, 21); +INSERT INTO t3 VALUES(22, 22); +SET binlog_format=@old_format; +BEGIN; +INSERT INTO t2 VALUES (21); +START SLAVE; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; +STOP SLAVE; +SET debug_sync='now WAIT_FOR wait_for_done_waiting'; +ROLLBACK; +SET GLOBAL debug_dbug=@old_dbug; +SET debug_sync='RESET'; +include/wait_for_slave_to_stop.inc +SELECT * FROM t1 WHERE a >= 20 ORDER BY a; +a +20 +SELECT * FROM t2 WHERE a >= 20 ORDER BY a; +a +20 +21 +SELECT * FROM t3 WHERE a >= 20 ORDER BY a; +a b +20 20 +include/start_slave.inc +SELECT * FROM t1 WHERE a >= 20 ORDER BY a; +a +20 +SELECT * FROM t2 WHERE a >= 20 ORDER BY a; +a +20 +21 +SELECT * FROM t3 WHERE a >= 20 ORDER BY a; +a b +20 20 +21 21 +22 22 +include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** Test killing slave threads at various wait points *** +*** 1. Test killing transaction waiting in commit for previous transaction to commit *** +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (31, foo(31, +'commit_before_prepare_ordered WAIT_FOR t2_waiting', +'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +BEGIN; +INSERT INTO t3 VALUES (32, foo(32, +'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', +'')); +INSERT INTO t3 VALUES (33, foo(33, +'group_commit_waiting_for_prior SIGNAL t2_waiting', +'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); +COMMIT; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (34, foo(34, +'', +'')); +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; +SELECT * FROM t3 WHERE a >= 30 ORDER BY a; +a b +31 31 +32 32 +33 33 +34 34 +SET debug_sync='RESET'; +SET sql_log_bin=0; +CALL mtr.add_suppression("Query execution was interrupted"); +CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); +CALL mtr.add_suppression("Slave: Connection was killed"); +SET sql_log_bin=1; +SET debug_sync='now WAIT_FOR t2_query'; +SET debug_sync='now SIGNAL t2_cont'; +SET debug_sync='now WAIT_FOR t1_ready'; +KILL THD_ID; +SET debug_sync='now WAIT_FOR t2_killed'; +SET debug_sync='now SIGNAL t1_cont'; +include/wait_for_slave_sql_error.inc [errno=1317,1927,1964] +STOP SLAVE IO_THREAD; +SELECT * FROM t3 WHERE a >= 30 ORDER BY a; +a b +31 31 +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +RETURN x; +END +|| +SET sql_log_bin=1; +INSERT INTO t3 VALUES (39,0); +include/start_slave.inc +SELECT * FROM t3 WHERE a >= 30 ORDER BY a; +a b +31 31 +32 32 +33 33 +34 34 +39 0 +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +IF d1 != '' THEN +SET debug_sync = d1; +END IF; +IF d2 != '' THEN +SET debug_sync = d2; +END IF; +RETURN x; +END +|| +SET sql_log_bin=1; +include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** 2. Same as (1), but without restarting IO thread after kill of SQL threads *** +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (41, foo(41, +'commit_before_prepare_ordered WAIT_FOR t2_waiting', +'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +BEGIN; +INSERT INTO t3 VALUES (42, foo(42, +'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', +'')); +INSERT INTO t3 VALUES (43, foo(43, +'group_commit_waiting_for_prior SIGNAL t2_waiting', +'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); +COMMIT; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (44, foo(44, +'', +'')); +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; +SELECT * FROM t3 WHERE a >= 40 ORDER BY a; +a b +41 41 +42 42 +43 43 +44 44 +SET debug_sync='RESET'; +SET debug_sync='now WAIT_FOR t2_query'; +SET debug_sync='now SIGNAL t2_cont'; +SET debug_sync='now WAIT_FOR t1_ready'; +KILL THD_ID; +SET debug_sync='now WAIT_FOR t2_killed'; +SET debug_sync='now SIGNAL t1_cont'; +include/wait_for_slave_sql_error.inc [errno=1317,1927,1964] +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +RETURN x; +END +|| +SET sql_log_bin=1; +INSERT INTO t3 VALUES (49,0); +START SLAVE SQL_THREAD; +SELECT * FROM t3 WHERE a >= 40 ORDER BY a; +a b +41 41 +42 42 +43 43 +44 44 +49 0 +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +IF d1 != '' THEN +SET debug_sync = d1; +END IF; +IF d2 != '' THEN +SET debug_sync = d2; +END IF; +RETURN x; +END +|| +SET sql_log_bin=1; +include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** 3. Same as (2), but not using gtid mode *** +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=no; +include/start_slave.inc +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (51, foo(51, +'commit_before_prepare_ordered WAIT_FOR t2_waiting', +'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +SET binlog_format=statement; +BEGIN; +INSERT INTO t3 VALUES (52, foo(52, +'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', +'')); +INSERT INTO t3 VALUES (53, foo(53, +'group_commit_waiting_for_prior SIGNAL t2_waiting', +'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); +COMMIT; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (54, foo(54, +'', +'')); +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='now SIGNAL master_cont1'; +SELECT * FROM t3 WHERE a >= 50 ORDER BY a; +a b +51 51 +52 52 +53 53 +54 54 +SET debug_sync='RESET'; +SET debug_sync='now WAIT_FOR t2_query'; +SET debug_sync='now SIGNAL t2_cont'; +SET debug_sync='now WAIT_FOR t1_ready'; +KILL THD_ID; +SET debug_sync='now WAIT_FOR t2_killed'; +SET debug_sync='now SIGNAL t1_cont'; +include/wait_for_slave_sql_error.inc [errno=1317,1927,1964] +SELECT * FROM t3 WHERE a >= 50 ORDER BY a; +a b +51 51 +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +RETURN x; +END +|| +SET sql_log_bin=1; +INSERT INTO t3 VALUES (59,0); +START SLAVE SQL_THREAD; +SELECT * FROM t3 WHERE a >= 50 ORDER BY a; +a b +51 51 +52 52 +53 53 +54 54 +59 0 +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +IF d1 != '' THEN +SET debug_sync = d1; +END IF; +IF d2 != '' THEN +SET debug_sync = d2; +END IF; +RETURN x; +END +|| +SET sql_log_bin=1; +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=4; +include/start_slave.inc +*** 4. Test killing thread that is waiting to start transaction until previous transaction commits *** +SET binlog_format=statement; +SET gtid_domain_id=2; +BEGIN; +INSERT INTO t3 VALUES (70, foo(70, +'rpl_parallel_start_waiting_for_prior SIGNAL t4_waiting', '')); +INSERT INTO t3 VALUES (60, foo(60, +'ha_write_row_end SIGNAL d2_query WAIT_FOR d2_cont2', +'rpl_parallel_end_of_group SIGNAL d2_done WAIT_FOR d2_cont')); +COMMIT; +SET gtid_domain_id=0; +SET debug_sync='now WAIT_FOR d2_query'; +SET gtid_domain_id=1; +BEGIN; +INSERT INTO t3 VALUES (61, foo(61, +'rpl_parallel_start_waiting_for_prior SIGNAL t3_waiting', +'rpl_parallel_start_waiting_for_prior_killed SIGNAL t3_killed')); +INSERT INTO t3 VALUES (62, foo(62, +'ha_write_row_end SIGNAL d1_query WAIT_FOR d1_cont2', +'rpl_parallel_end_of_group SIGNAL d1_done WAIT_FOR d1_cont')); +COMMIT; +SET gtid_domain_id=0; +SET debug_sync='now WAIT_FOR d1_query'; +SET gtid_domain_id=0; +INSERT INTO t3 VALUES (63, foo(63, +'ha_write_row_end SIGNAL d0_query WAIT_FOR d0_cont2', +'rpl_parallel_end_of_group SIGNAL d0_done WAIT_FOR d0_cont')); +SET debug_sync='now WAIT_FOR d0_query'; +SET gtid_domain_id=3; +BEGIN; +INSERT INTO t3 VALUES (68, foo(68, +'rpl_parallel_start_waiting_for_prior SIGNAL t2_waiting', '')); +INSERT INTO t3 VALUES (69, foo(69, +'ha_write_row_end SIGNAL d3_query WAIT_FOR d3_cont2', +'rpl_parallel_end_of_group SIGNAL d3_done WAIT_FOR d3_cont')); +COMMIT; +SET gtid_domain_id=0; +SET debug_sync='now WAIT_FOR d3_query'; +SET debug_sync='now SIGNAL d2_cont2'; +SET debug_sync='now WAIT_FOR d2_done'; +SET debug_sync='now SIGNAL d1_cont2'; +SET debug_sync='now WAIT_FOR d1_done'; +SET debug_sync='now SIGNAL d0_cont2'; +SET debug_sync='now WAIT_FOR d0_done'; +SET debug_sync='now SIGNAL d3_cont2'; +SET debug_sync='now WAIT_FOR d3_done'; +SET binlog_format=statement; +INSERT INTO t3 VALUES (64, foo(64, +'rpl_parallel_before_mark_start_commit SIGNAL t1_waiting WAIT_FOR t1_cont', '')); +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2 WAIT_FOR master_cont2'; +INSERT INTO t3 VALUES (65, foo(65, '', '')); +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; +INSERT INTO t3 VALUES (66, foo(66, '', '')); +SET debug_sync='now WAIT_FOR master_queued3'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued4'; +INSERT INTO t3 VALUES (67, foo(67, '', '')); +SET debug_sync='now WAIT_FOR master_queued4'; +SET debug_sync='now SIGNAL master_cont2'; +SELECT * FROM t3 WHERE a >= 60 ORDER BY a; +a b +60 60 +61 61 +62 62 +63 63 +64 64 +65 65 +66 66 +67 67 +68 68 +69 69 +70 70 +SET debug_sync='RESET'; +SET debug_sync='now SIGNAL d0_cont'; +SET debug_sync='now WAIT_FOR t1_waiting'; +SET debug_sync='now SIGNAL d3_cont'; +SET debug_sync='now WAIT_FOR t2_waiting'; +SET debug_sync='now SIGNAL d1_cont'; +SET debug_sync='now WAIT_FOR t3_waiting'; +SET debug_sync='now SIGNAL d2_cont'; +SET debug_sync='now WAIT_FOR t4_waiting'; +KILL THD_ID; +SET debug_sync='now WAIT_FOR t3_killed'; +SET debug_sync='now SIGNAL t1_cont'; +include/wait_for_slave_sql_error.inc [errno=1317,1927,1964] +STOP SLAVE IO_THREAD; +SELECT * FROM t3 WHERE a >= 60 AND a != 65 ORDER BY a; +a b +60 60 +61 61 +62 62 +63 63 +64 64 +68 68 +69 69 +70 70 +SET debug_sync='RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +RETURN x; +END +|| +SET sql_log_bin=1; +UPDATE t3 SET b=b+1 WHERE a=60; +include/start_slave.inc +SELECT * FROM t3 WHERE a >= 60 ORDER BY a; +a b +60 61 +61 61 +62 62 +63 63 +64 64 +65 65 +66 66 +67 67 +68 68 +69 69 +70 70 +SET sql_log_bin=0; +DROP FUNCTION foo; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +IF d1 != '' THEN +SET debug_sync = d1; +END IF; +IF d2 != '' THEN +SET debug_sync = d2; +END IF; +RETURN x; +END +|| +SET sql_log_bin=1; +include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** 5. Test killing thread that is waiting for queue of max length to shorten *** +SET @old_max_queued= @@GLOBAL.slave_parallel_max_queued; +SET GLOBAL slave_parallel_max_queued=9000; +SET binlog_format=statement; +INSERT INTO t3 VALUES (80, foo(0, +'ha_write_row_end SIGNAL query_waiting WAIT_FOR query_cont', '')); +SET debug_sync='now WAIT_FOR query_waiting'; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_queue_max"; +SELECT * FROM t3 WHERE a >= 80 ORDER BY a; +a b +80 0 +81 10000 +SET debug_sync='now WAIT_FOR wait_queue_ready'; +KILL THD_ID; +SET debug_sync='now WAIT_FOR wait_queue_killed'; +SET debug_sync='now SIGNAL query_cont'; +include/wait_for_slave_sql_error.inc [errno=1317,1927,1964] +STOP SLAVE IO_THREAD; +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_max_queued= @old_max_queued; +INSERT INTO t3 VALUES (82,0); +SET binlog_format=@old_format; +SET debug_sync='RESET'; +include/start_slave.inc +SELECT * FROM t3 WHERE a >= 80 ORDER BY a; +a b +80 0 +81 10000 +82 0 +include/stop_slave.inc +SET GLOBAL binlog_format=@old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** MDEV-5788 Incorrect free of rgi->deferred_events in parallel replication *** +include/stop_slave.inc +SET GLOBAL replicate_ignore_table="test.t3"; +SET GLOBAL slave_parallel_threads=2; +include/start_slave.inc +INSERT INTO t3 VALUES (100, rand()); +INSERT INTO t3 VALUES (101, rand()); +INSERT INTO t3 VALUES (102, rand()); +INSERT INTO t3 VALUES (103, rand()); +INSERT INTO t3 VALUES (104, rand()); +INSERT INTO t3 VALUES (105, rand()); +include/stop_slave.inc +SET GLOBAL replicate_ignore_table=""; +include/start_slave.inc +INSERT INTO t3 VALUES (106, rand()); +INSERT INTO t3 VALUES (107, rand()); +SELECT * FROM t3 WHERE a >= 100 ORDER BY a; +a b +106 # +107 # +*** MDEV-5921: In parallel replication, an error is not correctly signalled to the next transaction *** +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +INSERT INTO t3 VALUES (110, 1); +SELECT * FROM t3 WHERE a >= 110 ORDER BY a; +a b +110 1 +SET sql_log_bin=0; +INSERT INTO t3 VALUES (111, 666); +SET sql_log_bin=1; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +INSERT INTO t3 VALUES (111, 2); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +INSERT INTO t3 VALUES (112, 3); +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; +SET debug_sync='RESET'; +include/wait_for_slave_sql_error.inc [errno=1062] +include/wait_for_slave_sql_to_stop.inc +SELECT * FROM t3 WHERE a >= 110 ORDER BY a; +a b +110 1 +111 666 +SET sql_log_bin=0; +DELETE FROM t3 WHERE a=111 AND b=666; +SET sql_log_bin=1; +START SLAVE SQL_THREAD; +SELECT * FROM t3 WHERE a >= 110 ORDER BY a; +a b +110 1 +111 2 +112 3 +***MDEV-5914: Parallel replication deadlock due to InnoDB lock conflicts *** +include/stop_slave.inc +CREATE TABLE t4 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB; +INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +UPDATE t4 SET b=NULL WHERE a=6; +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +DELETE FROM t4 WHERE b <= 3; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; +SET debug_sync='RESET'; +include/start_slave.inc +include/stop_slave.inc +SELECT * FROM t4 ORDER BY a; +a b +1 NULL +3 NULL +4 4 +5 NULL +6 NULL +DELETE FROM t4; +INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +INSERT INTO t4 VALUES (7, NULL); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +DELETE FROM t4 WHERE b <= 3; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; +SET debug_sync='RESET'; +include/start_slave.inc +include/stop_slave.inc +SELECT * FROM t4 ORDER BY a; +a b +1 NULL +3 NULL +4 4 +5 NULL +6 6 +7 NULL +DELETE FROM t4; +INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +UPDATE t4 SET b=NULL WHERE a=6; +SET debug_sync='now WAIT_FOR master_queued1'; +SET @old_format= @@SESSION.binlog_format; +SET binlog_format='statement'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +DELETE FROM t4 WHERE b <= 1; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; +SET @old_format=@@GLOBAL.binlog_format; +SET debug_sync='RESET'; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,disable_thd_need_ordering_with"; +include/start_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SELECT * FROM t4 ORDER BY a; +a b +1 NULL +2 2 +3 NULL +4 4 +5 NULL +6 NULL +SET @last_gtid= 'GTID'; +SELECT IF(@@gtid_slave_pos LIKE CONCAT('%',@last_gtid,'%'), "GTID found ok", +CONCAT("GTID ", @last_gtid, " not found in gtid_slave_pos=", @@gtid_slave_pos)) +AS result; +result +GTID found ok +SELECT "ROW FOUND" AS `Is the row found?` + FROM mysql.gtid_slave_pos +WHERE CONCAT(domain_id, "-", server_id, "-", seq_no) = @last_gtid; +Is the row found? +ROW FOUND +*** MDEV-5938: Exec_master_log_pos not updated at log rotate in parallel replication *** +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=1; +SET DEBUG_SYNC= 'RESET'; +include/start_slave.inc +CREATE TABLE t5 (a INT PRIMARY KEY, b INT); +INSERT INTO t5 VALUES (1,1); +INSERT INTO t5 VALUES (2,2), (3,8); +INSERT INTO t5 VALUES (4,16); +test_check +OK +test_check +OK +FLUSH LOGS; +test_check +OK +test_check +OK +*** MDEV_6435: Incorrect error handling when query binlogged partially on master with "killed" error *** +CREATE TABLE t6 (a INT) ENGINE=MyISAM; +CREATE TRIGGER tr AFTER INSERT ON t6 FOR EACH ROW SET @a = 1; +SET @old_format= @@binlog_format; +SET binlog_format= statement; +SET debug_sync='sp_head_execute_before_loop SIGNAL ready WAIT_FOR cont'; +INSERT INTO t6 VALUES (1), (2), (3); +SET debug_sync='now WAIT_FOR ready'; +KILL QUERY CONID; +SET debug_sync='now SIGNAL cont'; +ERROR 70100: Query execution was interrupted +SET binlog_format= @old_format; +SET debug_sync='RESET'; +SET debug_sync='RESET'; +include/wait_for_slave_sql_error.inc [errno=1317] +STOP SLAVE IO_THREAD; +SET GLOBAL gtid_slave_pos= 'AFTER_ERROR_GTID_POS'; +include/start_slave.inc +INSERT INTO t6 VALUES (4); +SELECT * FROM t6 ORDER BY a; +a +1 +4 +SELECT * FROM t6 ORDER BY a; +a +4 +*** MDEV-6551: Some replication errors are ignored if slave_parallel_threads > 0 *** +INSERT INTO t2 VALUES (31); +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET GLOBAL slave_parallel_threads= 0; +include/start_slave.inc +SET sql_log_bin= 0; +INSERT INTO t2 VALUES (32); +SET sql_log_bin= 1; +INSERT INTO t2 VALUES (32); +FLUSH LOGS; +INSERT INTO t2 VALUES (33); +INSERT INTO t2 VALUES (34); +SELECT * FROM t2 WHERE a >= 30 ORDER BY a; +a +31 +32 +33 +34 +include/save_master_gtid.inc +include/wait_for_slave_sql_error.inc [errno=1062] +include/stop_slave_io.inc +SET GLOBAL slave_parallel_threads=10; +START SLAVE; +include/wait_for_slave_sql_error.inc [errno=1062] +START SLAVE SQL_THREAD; +include/wait_for_slave_sql_error.inc [errno=1062] +SELECT * FROM t2 WHERE a >= 30 ORDER BY a; +a +31 +32 +SET sql_slave_skip_counter= 1; +ERROR HY000: When using parallel replication and GTID with multiple replication domains, @@sql_slave_skip_counter can not be used. Instead, setting @@gtid_slave_pos explicitly can be used to skip to after a given GTID position. +include/stop_slave_io.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 30 ORDER BY a; +a +31 +32 +33 +34 +*** MDEV-6775: Wrong binlog order in parallel replication *** +DELETE FROM t4; +INSERT INTO t4 VALUES (1,NULL), (3,NULL), (4,4), (5, NULL), (6, 6); +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,inject_binlog_commit_before_get_LOCK_log"; +SET @old_format=@@GLOBAL.binlog_format; +SET GLOBAL binlog_format=ROW; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +SET @old_format= @@binlog_format; +SET binlog_format= statement; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +UPDATE t4 SET b=NULL WHERE a=6; +SET debug_sync='now WAIT_FOR master_queued1'; +SET @old_format= @@binlog_format; +SET binlog_format= statement; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +DELETE FROM t4 WHERE b <= 3; +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; +SET binlog_format= @old_format; +SET binlog_format= @old_format; +SET debug_sync='RESET'; +SELECT * FROM t4 ORDER BY a; +a b +1 NULL +3 NULL +4 4 +5 NULL +6 NULL +include/start_slave.inc +SET debug_sync= 'now WAIT_FOR waiting'; +SELECT * FROM t4 ORDER BY a; +a b +1 NULL +3 NULL +4 4 +5 NULL +6 NULL +SET debug_sync= 'now SIGNAL cont'; +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL binlog_format= @old_format; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** MDEV-7237: Parallel replication: incorrect relaylog position after stop/start the slave *** +INSERT INTO t2 VALUES (40); +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=no; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_scheduled_gtid_0_x_100"; +SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +INSERT INTO t2 VALUES (41); +INSERT INTO t2 VALUES (42); +SET @old_format= @@binlog_format; +SET binlog_format= statement; +DELETE FROM t2 WHERE a=40; +SET binlog_format= @old_format; +INSERT INTO t2 VALUES (43); +INSERT INTO t2 VALUES (44); +FLUSH LOGS; +INSERT INTO t2 VALUES (45); +SET gtid_seq_no=100; +INSERT INTO t2 VALUES (46); +BEGIN; +SELECT * FROM t2 WHERE a=40 FOR UPDATE; +a +40 +include/start_slave.inc +SET debug_sync= 'now WAIT_FOR scheduled_gtid_0_x_100'; +STOP SLAVE; +SET debug_sync= 'now WAIT_FOR wait_for_done_waiting'; +ROLLBACK; +include/wait_for_slave_sql_to_stop.inc +SELECT * FROM t2 WHERE a >= 40 ORDER BY a; +a +41 +42 +include/start_slave.inc +SELECT * FROM t2 WHERE a >= 40 ORDER BY a; +a +41 +42 +43 +44 +45 +46 +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET DEBUG_SYNC= 'RESET'; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +*** MDEV-7326 Server deadlock in connection with parallel replication *** +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=3; +SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; +include/start_slave.inc +SET @old_format= @@SESSION.binlog_format; +SET binlog_format= STATEMENT; +INSERT INTO t1 VALUES (foo(50, +"rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", +"rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); +SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; +INSERT INTO t2 VALUES (foo(50, +"rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", +"rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); +SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; +INSERT INTO t1 VALUES (foo(51, +"rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", +"rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); +SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; +SET DEBUG_SYNC= "now SIGNAL t1_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; +INSERT INTO t1 VALUES (52); +SET BINLOG_FORMAT= @old_format; +SELECT * FROM t2 WHERE a>=50 ORDER BY a; +a +50 +SELECT * FROM t1 WHERE a>=50 ORDER BY a; +a +50 +51 +52 +SET DEBUG_SYNC= "now SIGNAL prep_cont"; +SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; +SET DEBUG_SYNC= "now SIGNAL t2_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; +SET DEBUG_SYNC= "now SIGNAL t1_cont2"; +SELECT * FROM t2 WHERE a>=50 ORDER BY a; +a +50 +SELECT * FROM t1 WHERE a>=50 ORDER BY a; +a +50 +51 +52 +SET DEBUG_SYNC="reset"; +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** MDEV-7326 Server deadlock in connection with parallel replication *** +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=3; +SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; +include/start_slave.inc +SET @old_format= @@SESSION.binlog_format; +SET binlog_format= STATEMENT; +INSERT INTO t1 VALUES (foo(60, +"rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", +"rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); +SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; +INSERT INTO t2 VALUES (foo(60, +"rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", +"rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); +SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; +SET binlog_format=statement; +INSERT INTO t1 VALUES (foo(61, +"rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", +"rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); +SET debug_sync='now WAIT_FOR master_queued1'; +SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; +INSERT INTO t6 VALUES (62); +SET debug_sync='now WAIT_FOR master_queued2'; +SET debug_sync='now SIGNAL master_cont1'; +SET debug_sync='RESET'; +SET BINLOG_FORMAT= @old_format; +SELECT * FROM t2 WHERE a>=60 ORDER BY a; +a +60 +SELECT * FROM t1 WHERE a>=60 ORDER BY a; +a +60 +61 +SELECT * FROM t6 WHERE a>=60 ORDER BY a; +a +62 +SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; +SET DEBUG_SYNC= "now SIGNAL t1_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; +SET DEBUG_SYNC= "now SIGNAL prep_cont"; +SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; +SET DEBUG_SYNC= "now SIGNAL t2_cont1"; +SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; +SET DEBUG_SYNC= "now SIGNAL t1_cont2"; +SELECT * FROM t2 WHERE a>=60 ORDER BY a; +a +60 +SELECT * FROM t1 WHERE a>=60 ORDER BY a; +a +60 +61 +SELECT * FROM t6 WHERE a>=60 ORDER BY a; +a +62 +SET DEBUG_SYNC="reset"; +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_threads=0; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** MDEV-7335: Potential parallel slave deadlock with specific binlog corruption *** +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=1; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,slave_discard_xid_for_gtid_0_x_1000"; +INSERT INTO t2 VALUES (101); +INSERT INTO t2 VALUES (102); +INSERT INTO t2 VALUES (103); +INSERT INTO t2 VALUES (104); +INSERT INTO t2 VALUES (105); +SET gtid_seq_no=1000; +INSERT INTO t2 VALUES (106); +INSERT INTO t2 VALUES (107); +INSERT INTO t2 VALUES (108); +INSERT INTO t2 VALUES (109); +INSERT INTO t2 VALUES (110); +INSERT INTO t2 VALUES (111); +INSERT INTO t2 VALUES (112); +INSERT INTO t2 VALUES (113); +INSERT INTO t2 VALUES (114); +INSERT INTO t2 VALUES (115); +INSERT INTO t2 VALUES (116); +INSERT INTO t2 VALUES (117); +INSERT INTO t2 VALUES (118); +INSERT INTO t2 VALUES (119); +INSERT INTO t2 VALUES (120); +INSERT INTO t2 VALUES (121); +INSERT INTO t2 VALUES (122); +INSERT INTO t2 VALUES (123); +INSERT INTO t2 VALUES (124); +INSERT INTO t2 VALUES (125); +INSERT INTO t2 VALUES (126); +INSERT INTO t2 VALUES (127); +INSERT INTO t2 VALUES (128); +INSERT INTO t2 VALUES (129); +INSERT INTO t2 VALUES (130); +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 100 ORDER BY a; +a +101 +102 +103 +104 +105 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** MDEV-6676 - test syntax of @@slave_parallel_mode *** +Parallel_Mode = 'conservative' +include/stop_slave.inc +SET GLOBAL slave_parallel_mode='aggressive'; +Parallel_Mode = 'aggressive' +SET GLOBAL slave_parallel_mode='conservative'; +Parallel_Mode = 'conservative' +*** MDEV-6676 - test that empty parallel_mode does not replicate in parallel *** +INSERT INTO t2 VALUES (1040); +include/save_master_gtid.inc +SET GLOBAL slave_parallel_mode='none'; +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,slave_crash_if_parallel_apply"; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 1040 ORDER BY a; +a +1040 +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +*** MDEV-6676 - test disabling domain-based parallel replication *** +SET gtid_domain_id = 1; +INSERT INTO t2 VALUES (1041); +INSERT INTO t2 VALUES (1042); +INSERT INTO t2 VALUES (1043); +INSERT INTO t2 VALUES (1044); +INSERT INTO t2 VALUES (1045); +INSERT INTO t2 VALUES (1046); +DELETE FROM t2 WHERE a >= 1041; +SET gtid_domain_id = 2; +INSERT INTO t2 VALUES (1041); +INSERT INTO t2 VALUES (1042); +INSERT INTO t2 VALUES (1043); +INSERT INTO t2 VALUES (1044); +INSERT INTO t2 VALUES (1045); +INSERT INTO t2 VALUES (1046); +SET gtid_domain_id = 0; +include/save_master_gtid.inc +SET GLOBAL slave_parallel_mode=minimal; +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t2 WHERE a >= 1040 ORDER BY a; +a +1040 +1041 +1042 +1043 +1044 +1045 +1046 +include/stop_slave.inc +SET GLOBAL slave_parallel_mode='conservative'; +include/start_slave.inc +*** MDEV-7847: "Slave worker thread retried transaction 10 time(s) in vain, giving up", followed by replication hanging *** +*** MDEV-7882: Excessive transaction retry in parallel replication *** +CREATE TABLE t7 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; +CREATE TABLE t8 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=40; +SELECT @old_retries:=@@GLOBAL.slave_transaction_retries; +@old_retries:=@@GLOBAL.slave_transaction_retries +10 +SET GLOBAL slave_transaction_retries= 5; +INSERT INTO t7 VALUES (1,1), (2,2), (3,3), (4,4), (5,5); +SET @old_dbug= @@SESSION.debug_dbug; +SET @commit_id= 42; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +INSERT INTO t8 VALUES (1,1); +INSERT INTO t8 VALUES (2,2); +INSERT INTO t8 VALUES (3,3); +INSERT INTO t8 VALUES (4,4); +INSERT INTO t8 VALUES (5,5); +INSERT INTO t8 VALUES (6,6); +INSERT INTO t8 VALUES (7,7); +INSERT INTO t8 VALUES (8,8); +UPDATE t7 SET b=9 WHERE a=3; +UPDATE t7 SET b=10 WHERE a=3; +UPDATE t7 SET b=11 WHERE a=3; +INSERT INTO t8 VALUES (12,12); +INSERT INTO t8 VALUES (13,13); +UPDATE t7 SET b=14 WHERE a=3; +UPDATE t7 SET b=15 WHERE a=3; +INSERT INTO t8 VALUES (16,16); +UPDATE t7 SET b=17 WHERE a=3; +INSERT INTO t8 VALUES (18,18); +INSERT INTO t8 VALUES (19,19); +UPDATE t7 SET b=20 WHERE a=3; +INSERT INTO t8 VALUES (21,21); +UPDATE t7 SET b=22 WHERE a=3; +INSERT INTO t8 VALUES (23,24); +INSERT INTO t8 VALUES (24,24); +UPDATE t7 SET b=25 WHERE a=3; +INSERT INTO t8 VALUES (26,26); +UPDATE t7 SET b=27 WHERE a=3; +BEGIN; +INSERT INTO t8 VALUES (28,28); +INSERT INTO t8 VALUES (29,28), (30,28); +INSERT INTO t8 VALUES (31,28); +INSERT INTO t8 VALUES (32,28); +INSERT INTO t8 VALUES (33,28); +INSERT INTO t8 VALUES (34,28); +INSERT INTO t8 VALUES (35,28); +INSERT INTO t8 VALUES (36,28); +INSERT INTO t8 VALUES (37,28); +INSERT INTO t8 VALUES (38,28); +INSERT INTO t8 VALUES (39,28); +INSERT INTO t8 VALUES (40,28); +INSERT INTO t8 VALUES (41,28); +INSERT INTO t8 VALUES (42,28); +COMMIT; +SET @commit_id=43; +INSERT INTO t8 VALUES (43,43); +INSERT INTO t8 VALUES (44,44); +UPDATE t7 SET b=45 WHERE a=3; +INSERT INTO t8 VALUES (46,46); +INSERT INTO t8 VALUES (47,47); +UPDATE t7 SET b=48 WHERE a=3; +INSERT INTO t8 VALUES (49,49); +INSERT INTO t8 VALUES (50,50); +SET @commit_id=44; +INSERT INTO t8 VALUES (51,51); +INSERT INTO t8 VALUES (52,52); +UPDATE t7 SET b=53 WHERE a=3; +INSERT INTO t8 VALUES (54,54); +INSERT INTO t8 VALUES (55,55); +UPDATE t7 SET b=56 WHERE a=3; +INSERT INTO t8 VALUES (57,57); +UPDATE t7 SET b=58 WHERE a=3; +INSERT INTO t8 VALUES (58,58); +INSERT INTO t8 VALUES (59,59); +INSERT INTO t8 VALUES (60,60); +INSERT INTO t8 VALUES (61,61); +UPDATE t7 SET b=62 WHERE a=3; +INSERT INTO t8 VALUES (63,63); +INSERT INTO t8 VALUES (64,64); +INSERT INTO t8 VALUES (65,65); +INSERT INTO t8 VALUES (66,66); +UPDATE t7 SET b=67 WHERE a=3; +INSERT INTO t8 VALUES (68,68); +UPDATE t7 SET b=69 WHERE a=3; +UPDATE t7 SET b=70 WHERE a=3; +UPDATE t7 SET b=71 WHERE a=3; +INSERT INTO t8 VALUES (72,72); +UPDATE t7 SET b=73 WHERE a=3; +UPDATE t7 SET b=74 WHERE a=3; +UPDATE t7 SET b=75 WHERE a=3; +UPDATE t7 SET b=76 WHERE a=3; +INSERT INTO t8 VALUES (77,77); +UPDATE t7 SET b=78 WHERE a=3; +INSERT INTO t8 VALUES (79,79); +UPDATE t7 SET b=80 WHERE a=3; +INSERT INTO t8 VALUES (81,81); +UPDATE t7 SET b=82 WHERE a=3; +INSERT INTO t8 VALUES (83,83); +UPDATE t7 SET b=84 WHERE a=3; +SET @commit_id=45; +INSERT INTO t8 VALUES (85,85); +UPDATE t7 SET b=86 WHERE a=3; +INSERT INTO t8 VALUES (87,87); +SET @commit_id=46; +INSERT INTO t8 VALUES (88,88); +INSERT INTO t8 VALUES (89,89); +INSERT INTO t8 VALUES (90,90); +SET SESSION debug_dbug=@old_dbug; +INSERT INTO t8 VALUES (91,91); +INSERT INTO t8 VALUES (92,92); +INSERT INTO t8 VALUES (93,93); +INSERT INTO t8 VALUES (94,94); +INSERT INTO t8 VALUES (95,95); +INSERT INTO t8 VALUES (96,96); +INSERT INTO t8 VALUES (97,97); +INSERT INTO t8 VALUES (98,98); +INSERT INTO t8 VALUES (99,99); +SELECT * FROM t7 ORDER BY a; +a b +1 1 +2 2 +3 86 +4 4 +5 5 +SELECT * FROM t8 ORDER BY a; +a b +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +12 12 +13 13 +16 16 +18 18 +19 19 +21 21 +23 24 +24 24 +26 26 +28 28 +29 28 +30 28 +31 28 +32 28 +33 28 +34 28 +35 28 +36 28 +37 28 +38 28 +39 28 +40 28 +41 28 +42 28 +43 43 +44 44 +46 46 +47 47 +49 49 +50 50 +51 51 +52 52 +54 54 +55 55 +57 57 +58 58 +59 59 +60 60 +61 61 +63 63 +64 64 +65 65 +66 66 +68 68 +72 72 +77 77 +79 79 +81 81 +83 83 +85 85 +87 87 +88 88 +89 89 +90 90 +91 91 +92 92 +93 93 +94 94 +95 95 +96 96 +97 97 +98 98 +99 99 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t7 ORDER BY a; +a b +1 1 +2 2 +3 86 +4 4 +5 5 +SELECT * FROM t8 ORDER BY a; +a b +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +12 12 +13 13 +16 16 +18 18 +19 19 +21 21 +23 24 +24 24 +26 26 +28 28 +29 28 +30 28 +31 28 +32 28 +33 28 +34 28 +35 28 +36 28 +37 28 +38 28 +39 28 +40 28 +41 28 +42 28 +43 43 +44 44 +46 46 +47 47 +49 49 +50 50 +51 51 +52 52 +54 54 +55 55 +57 57 +58 58 +59 59 +60 60 +61 61 +63 63 +64 64 +65 65 +66 66 +68 68 +72 72 +77 77 +79 79 +81 81 +83 83 +85 85 +87 87 +88 88 +89 89 +90 90 +91 91 +92 92 +93 93 +94 94 +95 95 +96 96 +97 97 +98 98 +99 99 +include/stop_slave.inc +SET GLOBAL slave_transaction_retries= @old_retries; +SET GLOBAL slave_parallel_threads=10; +include/start_slave.inc +*** MDEV-7888: ANALYZE TABLE does wakeup_subsequent_commits(), causing wrong binlog order and parallel replication hang *** +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_analyze_table_sleep'; +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +SET @commit_id= 10000; +ANALYZE TABLE t2; +Table Op Msg_type Msg_text +test.t2 analyze status OK +INSERT INTO t3 VALUES (120, 0); +SET @commit_id= 10001; +INSERT INTO t3 VALUES (121, 0); +SET SESSION debug_dbug=@old_dbug; +SELECT * FROM t3 WHERE a >= 120 ORDER BY a; +a b +120 0 +121 0 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t3 WHERE a >= 120 ORDER BY a; +a b +120 0 +121 0 +include/stop_slave.inc +SET GLOBAL debug_dbug= @old_dbug; +include/start_slave.inc +*** MDEV-7929: record_gtid() for non-transactional event group calls wakeup_subsequent_commits() too early, causing slave hang. *** +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_record_gtid_serverid_100_sleep'; +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +SET @old_server_id= @@SESSION.server_id; +SET SESSION server_id= 100; +SET @commit_id= 10010; +ALTER TABLE t1 COMMENT "Hulubulu!"; +SET SESSION server_id= @old_server_id; +INSERT INTO t3 VALUES (130, 0); +SET @commit_id= 10011; +INSERT INTO t3 VALUES (131, 0); +SET SESSION debug_dbug=@old_dbug; +SELECT * FROM t3 WHERE a >= 130 ORDER BY a; +a b +130 0 +131 0 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t3 WHERE a >= 130 ORDER BY a; +a b +130 0 +131 0 +include/stop_slave.inc +SET GLOBAL debug_dbug= @old_dbug; +include/start_slave.inc +*** MDEV-8031: Parallel replication stops on "connection killed" error (probably incorrectly handled deadlock kill) *** +INSERT INTO t3 VALUES (201,0), (202,0); +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_mdev8031'; +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +SET @commit_id= 10200; +INSERT INTO t3 VALUES (203, 1); +INSERT INTO t3 VALUES (204, 1); +INSERT INTO t3 VALUES (205, 1); +UPDATE t3 SET b=b+1 WHERE a=201; +UPDATE t3 SET b=b+1 WHERE a=201; +UPDATE t3 SET b=b+1 WHERE a=201; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=202; +UPDATE t3 SET b=b+1 WHERE a=203; +UPDATE t3 SET b=b+1 WHERE a=203; +UPDATE t3 SET b=b+1 WHERE a=204; +UPDATE t3 SET b=b+1 WHERE a=204; +UPDATE t3 SET b=b+1 WHERE a=204; +UPDATE t3 SET b=b+1 WHERE a=203; +UPDATE t3 SET b=b+1 WHERE a=205; +UPDATE t3 SET b=b+1 WHERE a=205; +SET SESSION debug_dbug=@old_dbug; +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +a b +201 3 +202 4 +203 4 +204 4 +205 3 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +a b +201 3 +202 4 +203 4 +204 4 +205 3 +include/stop_slave.inc +SET GLOBAL debug_dbug= @old_dbug; +include/start_slave.inc +*** Check getting deadlock killed inside open_binlog() during retry. *** +include/stop_slave.inc +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug= '+d,inject_retry_event_group_open_binlog_kill'; +SET @old_max= @@GLOBAL.max_relay_log_size; +SET GLOBAL max_relay_log_size= 4096; +SET @old_dbug= @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +SET @commit_id= 10210; +Omit long queries that cause relaylog rotations and transaction retries... +SET SESSION debug_dbug=@old_dbug; +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +a b +201 6 +202 8 +203 7 +204 7 +205 5 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t3 WHERE a>=200 ORDER BY a; +a b +201 6 +202 8 +203 7 +204 7 +205 5 +include/stop_slave.inc +SET GLOBAL debug_dbug= @old_debg; +SET GLOBAL max_relay_log_size= @old_max; +include/start_slave.inc +*** MDEV-8302: Duplicate key with parallel replication *** +include/stop_slave.inc +/* Inject a small sleep which makes the race easier to hit. */ +SET @old_dbug=@@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,inject_mdev8302"; +INSERT INTO t7 VALUES (100,1), (101,2), (102,3), (103,4), (104,5); +SET @old_dbug= @@SESSION.debug_dbug; +SET @commit_id= 20000; +SET SESSION debug_dbug="+d,binlog_force_commit_id"; +SET SESSION debug_dbug=@old_dbug; +SELECT * FROM t7 ORDER BY a; +a b +1 1 +2 2 +3 86 +4 4 +5 5 +100 5 +101 1 +102 2 +103 3 +104 4 +include/save_master_gtid.inc +include/start_slave.inc +include/sync_with_master_gtid.inc +SELECT * FROM t7 ORDER BY a; +a b +1 1 +2 2 +3 86 +4 4 +5 5 +100 5 +101 1 +102 2 +103 3 +104 4 +include/stop_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +include/start_slave.inc +*** MDEV-8725: Assertion on ROLLBACK statement in the binary log *** +BEGIN; +INSERT INTO t2 VALUES (2000); +INSERT INTO t1 VALUES (2000); +INSERT INTO t2 VALUES (2001); +ROLLBACK; +SELECT * FROM t1 WHERE a>=2000 ORDER BY a; +a +2000 +SELECT * FROM t2 WHERE a>=2000 ORDER BY a; +a +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 WHERE a>=2000 ORDER BY a; +a +2000 +SELECT * FROM t2 WHERE a>=2000 ORDER BY a; +a +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +SET DEBUG_SYNC= 'RESET'; +DROP function foo; +DROP TABLE t1,t2,t3,t4,t5,t6,t7,t8; +SET DEBUG_SYNC= 'RESET'; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_parallel.test b/mysql-test/suite/binlog_encryption/rpl_parallel.test new file mode 100644 index 0000000000000..b7c4bb429a493 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_parallel.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_parallel.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.cnf b/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.cnf new file mode 100644 index 0000000000000..b8e22e97ae9ea --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.cnf @@ -0,0 +1,6 @@ +!include my.cnf + +[mysqld.2] +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename=@ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.result b/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.result new file mode 100644 index 0000000000000..d454fa411119e --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.result @@ -0,0 +1,12 @@ +include/master-slave.inc +[connection master] +[connection slave] +SET DEBUG_SYNC= 'after_show_binlog_events SIGNAL on_show_binlog_events WAIT_FOR end'; +SHOW BINLOG EVENTS; +[connection slave1] +SET DEBUG_SYNC= 'now WAIT_FOR on_show_binlog_events'; +FLUSH LOGS; +SET DEBUG_SYNC= 'now SIGNAL end'; +[connection slave] +SET DEBUG_SYNC= 'RESET'; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.test b/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.test new file mode 100644 index 0000000000000..9e93b0b56e9bc --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_parallel_show_binlog_events_purge_logs.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_parallel_show_binlog_events_purge_logs.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_relayrotate-slave.opt b/mysql-test/suite/binlog_encryption/rpl_relayrotate-slave.opt new file mode 100644 index 0000000000000..1665aec291db8 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_relayrotate-slave.opt @@ -0,0 +1,5 @@ +--max_relay_log_size=16384 +--log-warnings +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management-filename=$MYSQLTEST_VARDIR/std_data/keys.txt +--encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_relayrotate.result b/mysql-test/suite/binlog_encryption/rpl_relayrotate.result new file mode 100644 index 0000000000000..ea00aee00853d --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_relayrotate.result @@ -0,0 +1,13 @@ +include/master-slave.inc +[connection master] +stop slave; +create table t1 (a int) engine=innodb; +reset slave; +start slave; +stop slave; +start slave; +select max(a) from t1; +max(a) +8000 +drop table t1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_relayrotate.test b/mysql-test/suite/binlog_encryption/rpl_relayrotate.test new file mode 100644 index 0000000000000..5e3bcdcd711f0 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_relayrotate.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_relayrotate.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_semi_sync.result b/mysql-test/suite/binlog_encryption/rpl_semi_sync.result new file mode 100644 index 0000000000000..06eb56a40c5a7 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_semi_sync.result @@ -0,0 +1,465 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Timeout waiting for reply of binlog"); +call mtr.add_suppression("Read semi-sync reply"); +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("Master server does not support semi-sync"); +call mtr.add_suppression("Semi-sync slave .* reply"); +call mtr.add_suppression("Slave SQL.*Request to stop slave SQL Thread received while applying a group that has non-transactional changes; waiting for completion of the group"); +# +# Uninstall semi-sync plugins on master and slave +# +include/stop_slave.inc +reset slave; +set global rpl_semi_sync_master_enabled= 0; +set global rpl_semi_sync_slave_enabled= 0; +reset master; +set global rpl_semi_sync_master_enabled= 0; +set global rpl_semi_sync_slave_enabled= 0; +# +# Main test of semi-sync replication start here +# +[ on master ] +set global rpl_semi_sync_master_timeout= 60000; +[ default state of semi-sync on master should be OFF ] +show variables like 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled OFF +[ enable semi-sync on master ] +set global rpl_semi_sync_master_enabled = 1; +show variables like 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled ON +[ status of semi-sync on master should be ON even without any semi-sync slaves ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 0 +# +# BUG#45672 Semisync repl: ActiveTranx:insert_tranx_node: transaction node allocation failed +# BUG#45673 Semisynch reports correct operation even if no slave is connected +# +[ status of semi-sync on master should be OFF ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status OFF +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 0 +reset master; +[ on slave ] +[ default state of semi-sync on slave should be OFF ] +show variables like 'rpl_semi_sync_slave_enabled'; +Variable_name Value +rpl_semi_sync_slave_enabled OFF +[ enable semi-sync on slave ] +set global rpl_semi_sync_slave_enabled = 1; +show variables like 'rpl_semi_sync_slave_enabled'; +Variable_name Value +rpl_semi_sync_slave_enabled ON +include/start_slave.inc +[ on master ] +[ initial master state after the semi-sync slave connected ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 0 +create table t1(a int) engine = ENGINE_TYPE; +[ master state after CREATE TABLE statement ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 1 +select CONNECTIONS_NORMAL_SLAVE - CONNECTIONS_NORMAL_SLAVE as 'Should be 0'; +Should be 0 +0 +[ insert records to table ] +insert t1 values (10); +insert t1 values (9); +insert t1 values (8); +insert t1 values (7); +insert t1 values (6); +insert t1 values (5); +insert t1 values (4); +insert t1 values (3); +insert t1 values (2); +insert t1 values (1); +[ master status after inserts ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 11 +[ on slave ] +[ slave status after replicated inserts ] +show status like 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status ON +select count(distinct a) from t1; +count(distinct a) +10 +select min(a) from t1; +min(a) +1 +select max(a) from t1; +max(a) +10 + +# BUG#50157 +# semi-sync replication crashes when replicating a transaction which +# include 'CREATE TEMPORARY TABLE `MyISAM_t` SELECT * FROM `Innodb_t` ; +[ on master ] +SET SESSION AUTOCOMMIT= 0; +CREATE TABLE t2(c1 INT) ENGINE=innodb; +BEGIN; + +# Even though it is in a transaction, this statement is binlogged into binlog +# file immediately. +CREATE TEMPORARY TABLE t3 SELECT c1 FROM t2 where 1=1; + +# These statements will not be binlogged until the transaction is committed +INSERT INTO t2 VALUES(11); +INSERT INTO t2 VALUES(22); +COMMIT; +DROP TABLE t2, t3; +SET SESSION AUTOCOMMIT= 1; +# +# Test semi-sync master will switch OFF after one transaction +# timeout waiting for slave reply. +# +include/stop_slave.inc +[ on master ] +set global rpl_semi_sync_master_timeout= 5000; +[ master status should be ON ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 14 +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +[ semi-sync replication of these transactions will fail ] +insert into t1 values (500); +[ master status should be OFF ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status OFF +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 1 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 14 +delete from t1 where a=10; +delete from t1 where a=9; +delete from t1 where a=8; +delete from t1 where a=7; +delete from t1 where a=6; +delete from t1 where a=5; +delete from t1 where a=4; +delete from t1 where a=3; +delete from t1 where a=2; +delete from t1 where a=1; +insert into t1 values (100); +[ master status should be OFF ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status OFF +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 12 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 14 +# +# Test semi-sync status on master will be ON again when slave catches up +# +[ on slave ] +[ slave status should be OFF ] +show status like 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status OFF +include/start_slave.inc +[ slave status should be ON ] +show status like 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status ON +select count(distinct a) from t1; +count(distinct a) +2 +select min(a) from t1; +min(a) +100 +select max(a) from t1; +max(a) +500 +[ on master ] +[ master status should be ON again after slave catches up ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 12 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 14 +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +# +# Test disable/enable master semi-sync on the fly. +# +drop table t1; +[ on slave ] +include/stop_slave.inc +# +# Flush status +# +[ Semi-sync master status variables before FLUSH STATUS ] +SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 12 +SHOW STATUS LIKE 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 15 +FLUSH NO_WRITE_TO_BINLOG STATUS; +[ Semi-sync master status variables after FLUSH STATUS ] +SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +SHOW STATUS LIKE 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 0 +[ on master ] +show master logs; +Log_name master-bin.000001 +File_size # +show variables like 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled ON +[ disable semi-sync on the fly ] +set global rpl_semi_sync_master_enabled=0; +show variables like 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled OFF +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status OFF +[ enable semi-sync on the fly ] +set global rpl_semi_sync_master_enabled=1; +show variables like 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled ON +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +# +# Test RESET MASTER/SLAVE +# +[ on slave ] +include/start_slave.inc +[ on master ] +create table t1 (a int) engine = ENGINE_TYPE; +drop table t1; +show status like 'Rpl_relay%'; +Variable_name Value +[ test reset master ] +[ on master] +reset master; +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 0 +[ on slave ] +include/stop_slave.inc +reset slave; +kill query _tid; +include/start_slave.inc +[ on master ] +create table t1 (a int) engine = ENGINE_TYPE; +insert into t1 values (1); +insert into t1 values (2), (3); +[ on slave ] +select * from t1; +a +1 +2 +3 +[ on master ] +[ master semi-sync status should be ON ] +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 3 +# +# Start semi-sync replication without SUPER privilege +# +include/stop_slave.inc +reset slave; +[ on master ] +reset master; +kill query _tid; +set sql_log_bin=0; +grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; +flush privileges; +set sql_log_bin=1; +[ on slave ] +grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; +flush privileges; +change master to master_user='rpl',master_password='rpl_password'; +include/start_slave.inc +show status like 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status ON +[ on master ] +[ master semi-sync should be ON ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 0 +insert into t1 values (4); +insert into t1 values (5); +[ master semi-sync should be ON ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_no_tx'; +Variable_name Value +Rpl_semi_sync_master_no_tx 0 +show status like 'Rpl_semi_sync_master_yes_tx'; +Variable_name Value +Rpl_semi_sync_master_yes_tx 2 +# +# Test semi-sync slave connect to non-semi-sync master +# +[ on slave ] +include/stop_slave.inc +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status OFF +[ on master ] +kill query _tid; +[ Semi-sync status on master should be ON ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +set global rpl_semi_sync_master_enabled= 0; +[ on slave ] +SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; +Variable_name Value +rpl_semi_sync_slave_enabled ON +include/start_slave.inc +[ on master ] +insert into t1 values (8); +[ master semi-sync clients should be 1, status should be OFF ] +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status OFF +[ on slave ] +show status like 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status ON +include/stop_slave.inc +[ on master ] +set global rpl_semi_sync_master_enabled= 0; +[ on slave ] +SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; +Variable_name Value +rpl_semi_sync_slave_enabled ON +include/start_slave.inc +[ on master ] +insert into t1 values (10); +# +# Test non-semi-sync slave connect to semi-sync master +# +set global rpl_semi_sync_master_timeout= 5000; +set global rpl_semi_sync_master_enabled= 1; +[ on slave ] +include/stop_slave.inc +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status OFF +[ uninstall semi-sync slave plugin ] +set global rpl_semi_sync_slave_enabled= 0; +[ reinstall semi-sync slave plugin and disable semi-sync ] +SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; +Variable_name Value +rpl_semi_sync_slave_enabled OFF +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status OFF +include/start_slave.inc +SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; +Variable_name Value +Rpl_semi_sync_slave_status OFF +# +# Clean up +# +include/stop_slave.inc +set global rpl_semi_sync_slave_enabled= 0; +set global rpl_semi_sync_master_enabled= 0; +change master to master_user='root',master_password=''; +include/start_slave.inc +drop table t1; +drop user rpl@127.0.0.1; +flush privileges; +set global rpl_semi_sync_master_timeout= default; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_semi_sync.test b/mysql-test/suite/binlog_encryption/rpl_semi_sync.test new file mode 100644 index 0000000000000..d5f80619aeb43 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_semi_sync.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_semi_sync.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_skip_replication.cnf b/mysql-test/suite/binlog_encryption/rpl_skip_replication.cnf new file mode 100644 index 0000000000000..b8e22e97ae9ea --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_skip_replication.cnf @@ -0,0 +1,6 @@ +!include my.cnf + +[mysqld.2] +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename=@ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_skip_replication.result b/mysql-test/suite/binlog_encryption/rpl_skip_replication.result new file mode 100644 index 0000000000000..586a104239a00 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_skip_replication.result @@ -0,0 +1,252 @@ +include/master-slave.inc +[connection master] +CREATE USER 'nonsuperuser'@'127.0.0.1'; +GRANT ALTER,CREATE,DELETE,DROP,EVENT,INSERT,PROCESS,REPLICATION SLAVE, +SELECT,UPDATE ON *.* TO 'nonsuperuser'@'127.0.0.1'; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +ERROR 42000: Access denied; you need (at least one of) the SUPER privilege(s) for this operation +DROP USER'nonsuperuser'@'127.0.0.1'; +SELECT @@global.replicate_events_marked_for_skip; +@@global.replicate_events_marked_for_skip +REPLICATE +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +ERROR HY000: This operation cannot be performed as you have a running slave ''; run STOP SLAVE '' first +SELECT @@global.replicate_events_marked_for_skip; +@@global.replicate_events_marked_for_skip +REPLICATE +STOP SLAVE; +SET SESSION replicate_events_marked_for_skip=FILTER_ON_MASTER; +ERROR HY000: Variable 'replicate_events_marked_for_skip' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@global.replicate_events_marked_for_skip; +@@global.replicate_events_marked_for_skip +REPLICATE +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +SELECT @@global.replicate_events_marked_for_skip; +@@global.replicate_events_marked_for_skip +FILTER_ON_MASTER +START SLAVE; +SELECT @@skip_replication; +@@skip_replication +0 +SET GLOBAL skip_replication=1; +ERROR HY000: Variable 'skip_replication' is a SESSION variable and can't be used with SET GLOBAL +SELECT @@skip_replication; +@@skip_replication +0 +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +CREATE TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=innodb; +INSERT INTO t1(a) VALUES (1); +INSERT INTO t2(a) VALUES (1); +SET skip_replication=1; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +INSERT INTO t1(a) VALUES (2); +INSERT INTO t2(a) VALUES (2); +FLUSH NO_WRITE_TO_BINLOG LOGS; +SHOW TABLES; +Tables_in_test +t1 +t2 +SELECT * FROM t1; +a b +1 NULL +SELECT * FROM t2; +a b +1 NULL +DROP TABLE t3; +FLUSH NO_WRITE_TO_BINLOG LOGS; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +START SLAVE; +SET skip_replication=1; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +INSERT INTO t1(a) VALUES (3); +INSERT INTO t2(a) VALUES (3); +FLUSH NO_WRITE_TO_BINLOG LOGS; +SHOW TABLES; +Tables_in_test +t1 +t2 +SELECT * FROM t1; +a b +1 NULL +SELECT * FROM t2; +a b +1 NULL +DROP TABLE t3; +FLUSH NO_WRITE_TO_BINLOG LOGS; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; +SET skip_replication=1; +CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; +INSERT INTO t3(a) VALUES(2); +SELECT * FROM t3; +a b +2 NULL +DROP TABLE t3; +TRUNCATE t1; +RESET MASTER; +SET skip_replication=0; +INSERT INTO t1 VALUES (1,0); +SET skip_replication=1; +INSERT INTO t1 VALUES (2,0); +SET skip_replication=0; +INSERT INTO t1 VALUES (3,0); +SELECT * FROM t1 ORDER by a; +a b +1 0 +2 0 +3 0 +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +TRUNCATE t1; +SELECT * FROM t1 ORDER by a; +a b +1 0 +2 0 +3 0 +START SLAVE; +SELECT * FROM t1 ORDER by a; +a b +1 0 +3 0 +TRUNCATE t1; +STOP SLAVE; +SET GLOBAL sql_slave_skip_counter=6; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +START SLAVE; +SET @old_binlog_format= @@binlog_format; +SET binlog_format= statement; +SET skip_replication=0; +INSERT INTO t1 VALUES (1,5); +SET skip_replication=1; +INSERT INTO t1 VALUES (2,5); +SET skip_replication=0; +INSERT INTO t1 VALUES (3,5); +INSERT INTO t1 VALUES (4,5); +SET binlog_format= @old_binlog_format; +SELECT * FROM t1; +a b +4 5 +include/stop_slave.inc +SET @old_slave_binlog_format= @@global.binlog_format; +SET GLOBAL binlog_format= row; +include/start_slave.inc +TRUNCATE t1; +SET @old_binlog_format= @@binlog_format; +SET binlog_format= row; +BINLOG 'wlZOTw8BAAAA8QAAAPUAAAAAAAQANS41LjIxLU1hcmlhREItZGVidWctbG9nAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAEzgNAAgAEgAEBAQEEgAA2QAEGggAAAAICAgCAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAA371saA=='; +BINLOG 'wlZOTxMBAAAAKgAAAGMBAAAAgCkAAAAAAAEABHRlc3QAAnQxAAIDAwAC +wlZOTxcBAAAAJgAAAIkBAAAAgCkAAAAAAAEAAv/8AQAAAAgAAAA='; +BINLOG 'wlZOTxMBAAAAKgAAADwCAAAAACkAAAAAAAEABHRlc3QAAnQxAAIDAwAC +wlZOTxcBAAAAJgAAAGICAAAAACkAAAAAAAEAAv/8AgAAAAgAAAA='; +SET binlog_format= @old_binlog_format; +SELECT * FROM t1 ORDER BY a; +a b +1 8 +2 8 +SELECT * FROM t1 ORDER by a; +a b +2 8 +include/stop_slave.inc +SET GLOBAL binlog_format= @old_slave_binlog_format; +include/start_slave.inc +SET skip_replication=0; +BEGIN; +SET skip_replication=0; +ERROR HY000: Cannot modify @@session.skip_replication inside a transaction +SET skip_replication=1; +ERROR HY000: Cannot modify @@session.skip_replication inside a transaction +ROLLBACK; +SET skip_replication=1; +BEGIN; +SET skip_replication=0; +ERROR HY000: Cannot modify @@session.skip_replication inside a transaction +SET skip_replication=1; +ERROR HY000: Cannot modify @@session.skip_replication inside a transaction +COMMIT; +SET autocommit=0; +INSERT INTO t2(a) VALUES(100); +SET skip_replication=1; +ERROR HY000: Cannot modify @@session.skip_replication inside a transaction +ROLLBACK; +SET autocommit=1; +SET skip_replication=1; +CREATE FUNCTION foo (x INT) RETURNS INT BEGIN SET SESSION skip_replication=x; RETURN x; END| +CREATE PROCEDURE bar(x INT) BEGIN SET SESSION skip_replication=x; END| +CREATE FUNCTION baz (x INT) RETURNS INT BEGIN CALL bar(x); RETURN x; END| +SELECT foo(0); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +SELECT baz(0); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +SET @a= foo(1); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +SET @a= baz(1); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +UPDATE t2 SET b=foo(0); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +UPDATE t2 SET b=baz(0); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +INSERT INTO t1 VALUES (101, foo(1)); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +INSERT INTO t1 VALUES (101, baz(0)); +ERROR HY000: Cannot modify @@session.skip_replication inside a stored function or trigger +SELECT @@skip_replication; +@@skip_replication +1 +CALL bar(0); +SELECT @@skip_replication; +@@skip_replication +0 +CALL bar(1); +SELECT @@skip_replication; +@@skip_replication +1 +DROP FUNCTION foo; +DROP PROCEDURE bar; +DROP FUNCTION baz; +SET skip_replication= 0; +TRUNCATE t1; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; +START SLAVE IO_THREAD; +SET skip_replication= 1; +INSERT INTO t1(a) VALUES (1); +SET skip_replication= 0; +INSERT INTO t1(a) VALUES (2); +include/save_master_pos.inc +include/sync_io_with_master.inc +STOP SLAVE IO_THREAD; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; +SELECT * FROM t1; +a b +2 NULL +SET skip_replication= 0; +TRUNCATE t1; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; +START SLAVE IO_THREAD; +SET skip_replication= 1; +INSERT INTO t1(a) VALUES (1); +SET skip_replication= 0; +INSERT INTO t1(a) VALUES (2); +include/save_master_pos.inc +include/sync_io_with_master.inc +STOP SLAVE IO_THREAD; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; +SELECT * FROM t1 ORDER BY a; +a b +1 NULL +2 NULL +SET skip_replication=0; +DROP TABLE t1,t2; +STOP SLAVE; +SET GLOBAL replicate_events_marked_for_skip=REPLICATE; +START SLAVE; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_skip_replication.test b/mysql-test/suite/binlog_encryption/rpl_skip_replication.test new file mode 100644 index 0000000000000..e7b52f2fadb41 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_skip_replication.test @@ -0,0 +1,2 @@ +--let $use_remote_mysqlbinlog= 1 +--source extra/rpl_tests/rpl_skip_replication.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_special_charset.opt b/mysql-test/suite/binlog_encryption/rpl_special_charset.opt new file mode 100644 index 0000000000000..b071fb20845f9 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_special_charset.opt @@ -0,0 +1 @@ +--character-set-server=utf16 diff --git a/mysql-test/suite/binlog_encryption/rpl_special_charset.result b/mysql-test/suite/binlog_encryption/rpl_special_charset.result new file mode 100644 index 0000000000000..99b8d3a21d10f --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_special_charset.result @@ -0,0 +1,8 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Cannot use utf16 as character_set_client"); +CREATE TABLE t1(i VARCHAR(20)); +INSERT INTO t1 VALUES (0xFFFF); +include/diff_tables.inc [master:t1, slave:t1] +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_special_charset.test b/mysql-test/suite/binlog_encryption/rpl_special_charset.test new file mode 100644 index 0000000000000..6f196005711b4 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_special_charset.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_special_charset.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_sporadic_master-master.opt b/mysql-test/suite/binlog_encryption/rpl_sporadic_master-master.opt new file mode 100644 index 0000000000000..5f038b69bb765 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sporadic_master-master.opt @@ -0,0 +1 @@ +--debug-sporadic-binlog-dump-fail --debug-max-binlog-dump-events=2 diff --git a/mysql-test/suite/binlog_encryption/rpl_sporadic_master.result b/mysql-test/suite/binlog_encryption/rpl_sporadic_master.result new file mode 100644 index 0000000000000..1ec94964acf16 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sporadic_master.result @@ -0,0 +1,23 @@ +include/master-slave.inc +[connection master] +create table t2(n int); +create table t1(n int not null auto_increment primary key); +insert into t1 values (NULL),(NULL); +truncate table t1; +insert into t1 values (4),(NULL); +include/stop_slave.inc +include/start_slave.inc +insert into t1 values (NULL),(NULL); +flush logs; +truncate table t1; +insert into t1 values (10),(NULL),(NULL),(NULL),(NULL),(NULL); +select * from t1 ORDER BY n; +n +10 +11 +12 +13 +14 +15 +drop table t1,t2; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_sporadic_master.test b/mysql-test/suite/binlog_encryption/rpl_sporadic_master.test new file mode 100644 index 0000000000000..0a75698204760 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sporadic_master.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_sporadic_master.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_ssl.result b/mysql-test/suite/binlog_encryption/rpl_ssl.result new file mode 100644 index 0000000000000..f02fa4e7b9fe2 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_ssl.result @@ -0,0 +1,47 @@ +include/master-slave.inc +[connection master] +create user replssl@localhost; +grant replication slave on *.* to replssl@localhost require ssl; +create table t1 (t int auto_increment, KEY(t)); +stop slave; +change master to +master_user='replssl', +master_password='', +master_ssl=1, +master_ssl_ca ='MYSQL_TEST_DIR/std_data/cacert.pem', +master_ssl_cert='MYSQL_TEST_DIR/std_data/client-cert.pem', +master_ssl_key='MYSQL_TEST_DIR/std_data/client-key.pem'; +start slave; +insert into t1 values(1); +select * from t1; +t +1 +Master_SSL_Allowed = 'Yes' +Master_SSL_CA_Path = '' +Master_SSL_CA_File = 'MYSQL_TEST_DIR/std_data/cacert.pem' +Master_SSL_Cert = 'MYSQL_TEST_DIR/std_data/client-cert.pem' +Master_SSL_Key = 'MYSQL_TEST_DIR/std_data/client-key.pem' +include/check_slave_is_running.inc +STOP SLAVE; +select * from t1; +t +1 +insert into t1 values (NULL); +include/wait_for_slave_to_start.inc +Master_SSL_Allowed = 'Yes' +Master_SSL_CA_Path = '' +Master_SSL_CA_File = 'MYSQL_TEST_DIR/std_data/cacert.pem' +Master_SSL_Cert = 'MYSQL_TEST_DIR/std_data/client-cert.pem' +Master_SSL_Key = 'MYSQL_TEST_DIR/std_data/client-key.pem' +include/check_slave_is_running.inc +drop user replssl@localhost; +drop table t1; +include/stop_slave.inc +CHANGE MASTER TO +master_user = 'root', +master_ssl = 0, +master_ssl_ca = '', +master_ssl_cert = '', +master_ssl_key = ''; +End of 5.0 tests +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_ssl.test b/mysql-test/suite/binlog_encryption/rpl_ssl.test new file mode 100644 index 0000000000000..883b367e9f26a --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_ssl.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_ssl.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space-slave.opt b/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space-slave.opt new file mode 100644 index 0000000000000..f780540aba854 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space-slave.opt @@ -0,0 +1 @@ +--relay-log-space-limit=8192 --relay-log-purge --max-relay-log-size=4096 diff --git a/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.result b/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.result new file mode 100644 index 0000000000000..7820f1ef97fbb --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.result @@ -0,0 +1,5 @@ +include/master-slave.inc +[connection master] +include/assert.inc [Assert that relay log space is close to the limit] +include/diff_tables.inc [master:test.t1,slave:test.t1] +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.test b/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.test new file mode 100644 index 0000000000000..f72300ee2ded1 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_stm_relay_ign_space.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_stm_relay_ign_space.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.result b/mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.result new file mode 100644 index 0000000000000..7fcb0601ebf0b --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.result @@ -0,0 +1,428 @@ +include/master-slave.inc +[connection master] +drop database if exists mysqltest1; +create database mysqltest1; +use mysqltest1; +set @my_binlog_format= @@global.binlog_format; +set session binlog_format=mixed; +show session variables like "binlog_format%"; +Variable_name Value +binlog_format MIXED +set session binlog_format=statement; +show session variables like "binlog_format%"; +Variable_name Value +binlog_format STATEMENT +set session binlog_format=row; +show session variables like "binlog_format%"; +Variable_name Value +binlog_format ROW +set global binlog_format=DEFAULT; +show global variables like "binlog_format%"; +Variable_name Value +binlog_format STATEMENT +set global binlog_format=MIXED; +show global variables like "binlog_format%"; +Variable_name Value +binlog_format MIXED +set global binlog_format=STATEMENT; +show global variables like "binlog_format%"; +Variable_name Value +binlog_format STATEMENT +set global binlog_format=ROW; +show global variables like "binlog_format%"; +Variable_name Value +binlog_format ROW +show session variables like "binlog_format%"; +Variable_name Value +binlog_format ROW +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +ROW ROW +CREATE TABLE t1 (a varchar(100)); +prepare stmt1 from 'insert into t1 select concat(UUID(),?)'; +set @string="emergency_1_"; +insert into t1 values("work_2_"); +execute stmt1 using @string; +deallocate prepare stmt1; +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values(concat(UUID(),"work_3_")); +execute stmt1 using @string; +deallocate prepare stmt1; +insert into t1 values(concat("for_4_",UUID())); +insert into t1 select "yesterday_5_"; +create temporary table tmp(a char(100)); +insert into tmp values("see_6_"); +set binlog_format=statement; +ERROR HY000: Cannot switch out of the row-based binary log format when the session has open temporary tables +insert into t1 select * from tmp; +drop temporary table tmp; +set binlog_format=statement; +show global variables like "binlog_format%"; +Variable_name Value +binlog_format ROW +show session variables like "binlog_format%"; +Variable_name Value +binlog_format STATEMENT +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +ROW STATEMENT +set global binlog_format=statement; +show global variables like "binlog_format%"; +Variable_name Value +binlog_format STATEMENT +show session variables like "binlog_format%"; +Variable_name Value +binlog_format STATEMENT +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +STATEMENT STATEMENT +prepare stmt1 from 'insert into t1 select ?'; +set @string="emergency_7_"; +insert into t1 values("work_8_"); +execute stmt1 using @string; +deallocate prepare stmt1; +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values("work_9_"); +execute stmt1 using @string; +deallocate prepare stmt1; +insert into t1 values("for_10_"); +insert into t1 select "yesterday_11_"; +set binlog_format=statement; +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +STATEMENT STATEMENT +set global binlog_format=statement; +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +STATEMENT STATEMENT +prepare stmt1 from 'insert into t1 select ?'; +set @string="emergency_12_"; +insert into t1 values("work_13_"); +execute stmt1 using @string; +deallocate prepare stmt1; +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values("work_14_"); +execute stmt1 using @string; +deallocate prepare stmt1; +insert into t1 values("for_15_"); +insert into t1 select "yesterday_16_"; +set global binlog_format=mixed; +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +MIXED STATEMENT +set binlog_format=default; +select @@global.binlog_format, @@session.binlog_format; +@@global.binlog_format @@session.binlog_format +MIXED MIXED +prepare stmt1 from 'insert into t1 select concat(UUID(),?)'; +set @string="emergency_17_"; +insert into t1 values("work_18_"); +execute stmt1 using @string; +deallocate prepare stmt1; +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values(concat(UUID(),"work_19_")); +execute stmt1 using @string; +deallocate prepare stmt1; +insert into t1 values(concat("for_20_",UUID())); +insert into t1 select "yesterday_21_"; +prepare stmt1 from 'insert into t1 select ?'; +insert into t1 values(concat(UUID(),"work_22_")); +execute stmt1 using @string; +deallocate prepare stmt1; +insert into t1 values(concat("for_23_",UUID())); +insert into t1 select "yesterday_24_"; +create table t2 ENGINE=MyISAM select rpad(UUID(),100,' '); +create table t3 select 1 union select UUID(); +create table t4 select * from t1 where 3 in (select 1 union select 2 union select UUID() union select 3); +create table t5 select * from t1 where 3 in (select 1 union select 2 union select curdate() union select 3); +Warnings: +Warning 1292 Incorrect datetime value: '3' +insert into t5 select UUID() from t1 where 3 in (select 1 union select 2 union select 3 union select * from t4); +create procedure foo() +begin +insert into t1 values("work_25_"); +insert into t1 values(concat("for_26_",UUID())); +insert into t1 select "yesterday_27_"; +end| +create procedure foo2() +begin +insert into t1 values(concat("emergency_28_",UUID())); +insert into t1 values("work_29_"); +insert into t1 values(concat("for_30_",UUID())); +set session binlog_format=row; # accepted for stored procs +insert into t1 values("more work_31_"); +set session binlog_format=mixed; +end| +create function foo3() returns bigint unsigned +begin +set session binlog_format=row; # rejected for stored funcs +insert into t1 values("alarm"); +return 100; +end| +create procedure foo4(x varchar(100)) +begin +insert into t1 values(concat("work_250_",x)); +insert into t1 select "yesterday_270_"; +end| +call foo(); +call foo2(); +call foo4("hello"); +call foo4(UUID()); +call foo4("world"); +select foo3(); +ERROR HY000: Cannot change the binary logging format inside a stored function or trigger +select * from t1 where a="alarm"; +a +drop function foo3; +create function foo3() returns bigint unsigned +begin +insert into t1 values("foo3_32_"); +call foo(); +return 100; +end| +insert into t2 select foo3(); +prepare stmt1 from 'insert into t2 select foo3()'; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; +create function foo4() returns bigint unsigned +begin +insert into t2 select foo3(); +return 100; +end| +select foo4(); +foo4() +100 +prepare stmt1 from 'select foo4()'; +execute stmt1; +foo4() +100 +execute stmt1; +foo4() +100 +deallocate prepare stmt1; +create function foo5() returns bigint unsigned +begin +insert into t2 select UUID(); +return 100; +end| +select foo5(); +foo5() +100 +prepare stmt1 from 'select foo5()'; +execute stmt1; +foo5() +100 +execute stmt1; +foo5() +100 +deallocate prepare stmt1; +create function foo6(x varchar(100)) returns bigint unsigned +begin +insert into t2 select x; +return 100; +end| +select foo6("foo6_1_"); +foo6("foo6_1_") +100 +select foo6(concat("foo6_2_",UUID())); +foo6(concat("foo6_2_",UUID())) +100 +prepare stmt1 from 'select foo6(concat("foo6_3_",UUID()))'; +execute stmt1; +foo6(concat("foo6_3_",UUID())) +100 +execute stmt1; +foo6(concat("foo6_3_",UUID())) +100 +deallocate prepare stmt1; +create view v1 as select uuid(); +create table t11 (data varchar(255)); +insert into t11 select * from v1; +insert into t11 select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='mysqltest1' and TABLE_NAME IN ('v1','t11'); +prepare stmt1 from "insert into t11 select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='mysqltest1' and TABLE_NAME IN ('v1','t11')"; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; +create trigger t11_bi before insert on t11 for each row +begin +set NEW.data = concat(NEW.data,UUID()); +end| +insert into t11 values("try_560_"); +insert delayed into t2 values("delay_1_"); +insert delayed into t2 values(concat("delay_2_",UUID())); +insert delayed into t2 values("delay_6_"); +insert delayed into t2 values(rand()); +set @a=2.345; +insert delayed into t2 values(@a); +create table t20 select * from t1; +create table t21 select * from t2; +create table t22 select * from t3; +drop table t1,t2,t3; +create table t1 (a int primary key auto_increment, b varchar(100)); +create table t2 (a int primary key auto_increment, b varchar(100)); +create table t3 (b varchar(100)); +create function f (x varchar(100)) returns int deterministic +begin +insert into t1 values(null,x); +insert into t2 values(null,x); +return 1; +end| +select f("try_41_"); +f("try_41_") +1 +use mysqltest1; +insert into t2 values(2,null),(3,null),(4,null); +delete from t2 where a>=2; +select f("try_42_"); +f("try_42_") +1 +insert into t2 values(3,null),(4,null); +delete from t2 where a>=3; +prepare stmt1 from 'select f(?)'; +set @string="try_43_"; +insert into t1 values(null,"try_44_"); +execute stmt1 using @string; +f(?) +1 +deallocate prepare stmt1; +create table t12 select * from t1; +drop table t1; +create table t1 (a int, b varchar(100), key(a)); +select f("try_45_"); +f("try_45_") +1 +create table t13 select * from t1; +drop table t1; +create table t1 (a int primary key auto_increment, b varchar(100)); +drop function f; +create table t14 (unique (a)) select * from t2; +truncate table t2; +create function f1 (x varchar(100)) returns int deterministic +begin +insert into t1 values(null,x); +return 1; +end| +create function f2 (x varchar(100)) returns int deterministic +begin +insert into t2 values(null,x); +return 1; +end| +select f1("try_46_"),f2("try_47_"); +f1("try_46_") f2("try_47_") +1 1 +insert into t2 values(2,null),(3,null),(4,null); +delete from t2 where a>=2; +select f1("try_48_"),f2("try_49_"); +f1("try_48_") f2("try_49_") +1 1 +insert into t3 values(concat("try_50_",f1("try_51_"),f2("try_52_"))); +drop function f2; +create function f2 (x varchar(100)) returns int deterministic +begin +declare y int; +insert into t1 values(null,x); +set y = (select count(*) from t2); +return y; +end| +select f1("try_53_"),f2("try_54_"); +f1("try_53_") f2("try_54_") +1 3 +drop function f2; +create trigger t1_bi before insert on t1 for each row +begin +insert into t2 values(null,"try_55_"); +end| +insert into t1 values(null,"try_56_"); +alter table t1 modify a int, drop primary key; +insert into t1 values(null,"try_57_"); +CREATE TEMPORARY TABLE t15 SELECT UUID(); +create table t16 like t15; +INSERT INTO t16 SELECT * FROM t15; +insert into t16 values("try_65_"); +drop table t15; +insert into t16 values("try_66_"); +select count(*) from t1; +count(*) +7 +select count(*) from t2; +count(*) +5 +select count(*) from t3; +count(*) +1 +select count(*) from t4; +count(*) +29 +select count(*) from t5; +count(*) +58 +select count(*) from t11; +count(*) +8 +select count(*) from t20; +count(*) +66 +select count(*) from t21; +count(*) +19 +select count(*) from t22; +count(*) +2 +select count(*) from t12; +count(*) +4 +select count(*) from t13; +count(*) +1 +select count(*) from t14; +count(*) +4 +select count(*) from t16; +count(*) +3 +DROP TABLE IF EXISTS t11; +SET SESSION BINLOG_FORMAT=STATEMENT; +CREATE TABLE t11 (song VARCHAR(255)); +LOCK TABLES t11 WRITE; +SET SESSION BINLOG_FORMAT=ROW; +INSERT INTO t11 VALUES('Several Species of Small Furry Animals Gathered Together in a Cave and Grooving With a Pict'); +SET SESSION BINLOG_FORMAT=STATEMENT; +INSERT INTO t11 VALUES('Careful With That Axe, Eugene'); +UNLOCK TABLES; +SELECT * FROM t11; +song Several Species of Small Furry Animals Gathered Together in a Cave and Grooving With a Pict +song Careful With That Axe, Eugene +USE mysqltest1; +SELECT * FROM t11; +song Several Species of Small Furry Animals Gathered Together in a Cave and Grooving With a Pict +song Careful With That Axe, Eugene +DROP TABLE IF EXISTS t12; +SET SESSION BINLOG_FORMAT=MIXED; +CREATE TABLE t12 (data LONG); +LOCK TABLES t12 WRITE; +INSERT INTO t12 VALUES(UUID()); +UNLOCK TABLES; +CREATE FUNCTION my_user() +RETURNS CHAR(64) +BEGIN +DECLARE user CHAR(64); +SELECT USER() INTO user; +RETURN user; +END $$ +CREATE FUNCTION my_current_user() +RETURNS CHAR(64) +BEGIN +DECLARE user CHAR(64); +SELECT CURRENT_USER() INTO user; +RETURN user; +END $$ +DROP TABLE IF EXISTS t13; +CREATE TABLE t13 (data CHAR(64)); +INSERT INTO t13 VALUES (USER()); +INSERT INTO t13 VALUES (my_user()); +INSERT INTO t13 VALUES (CURRENT_USER()); +INSERT INTO t13 VALUES (my_current_user()); +drop database mysqltest1; +set global binlog_format =@my_binlog_format; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.test b/mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.test new file mode 100644 index 0000000000000..cd826c6be1ee6 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_switch_stm_row_mixed.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_switch_stm_row_mixed.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_sync-master.opt b/mysql-test/suite/binlog_encryption/rpl_sync-master.opt new file mode 100644 index 0000000000000..04b06bfa0f225 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sync-master.opt @@ -0,0 +1,2 @@ +--default-storage-engine=MyISAM +--loose-innodb-file-per-table=0 diff --git a/mysql-test/suite/binlog_encryption/rpl_sync-slave.opt b/mysql-test/suite/binlog_encryption/rpl_sync-slave.opt new file mode 100644 index 0000000000000..2e8be18dbd7ae --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sync-slave.opt @@ -0,0 +1,2 @@ +--sync-relay-log-info=1 --relay-log-recovery=1 --loose-innodb_file_format_check=1 --default-storage-engine=MyISAM --loose-innodb-file-per-table=0 +--skip-core-file --skip-slave-start diff --git a/mysql-test/suite/binlog_encryption/rpl_sync.result b/mysql-test/suite/binlog_encryption/rpl_sync.result new file mode 100644 index 0000000000000..84c100970e497 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sync.result @@ -0,0 +1,41 @@ +=====Configuring the enviroment=======; +include/master-slave.inc +[connection master] +call mtr.add_suppression('Attempting backtrace'); +call mtr.add_suppression("Recovery from master pos .* and file master-bin.000001"); +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +flush tables; +CREATE TABLE t1(a INT, PRIMARY KEY(a)) engine=innodb; +insert into t1(a) values(1); +insert into t1(a) values(2); +insert into t1(a) values(3); +=====Inserting data on the master but without the SQL Thread being running=======; +include/stop_slave_sql.inc +insert into t1(a) values(4); +insert into t1(a) values(5); +insert into t1(a) values(6); +=====Removing relay log files and crashing/recoverying the slave=======; +include/stop_slave_io.inc +SET SESSION debug_dbug="d,crash_before_rotate_relaylog"; +FLUSH LOGS; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +=====Dumping and comparing tables=======; +include/start_slave.inc +include/diff_tables.inc [master:t1,slave:t1] +=====Corrupting the master.info=======; +include/stop_slave.inc +FLUSH LOGS; +insert into t1(a) values(7); +insert into t1(a) values(8); +insert into t1(a) values(9); +SET SESSION debug_dbug="d,crash_before_rotate_relaylog"; +FLUSH LOGS; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +=====Dumping and comparing tables=======; +include/start_slave.inc +include/diff_tables.inc [master:t1,slave:t1] +=====Clean up=======; +drop table t1; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_sync.test b/mysql-test/suite/binlog_encryption/rpl_sync.test new file mode 100644 index 0000000000000..189dd8220ef50 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_sync.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_sync.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.cnf b/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.cnf new file mode 100644 index 0000000000000..b8e22e97ae9ea --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.cnf @@ -0,0 +1,6 @@ +!include my.cnf + +[mysqld.2] +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename=@ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.result b/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.result new file mode 100644 index 0000000000000..2e9116ced2d20 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.result @@ -0,0 +1,83 @@ +include/master-slave.inc +[connection master] +SELECT @@global.mysql56_temporal_format AS on_master; +on_master +1 +SELECT @@global.mysql56_temporal_format AS on_slave; +on_slave +1 +CREATE TABLE t1 +( +c0 TIME(0), +c1 TIME(1), +c2 TIME(2), +c3 TIME(3), +c4 TIME(4), +c5 TIME(5), +c6 TIME(6) +); +CREATE TABLE t2 +( +c0 TIMESTAMP(0), +c1 TIMESTAMP(1), +c2 TIMESTAMP(2), +c3 TIMESTAMP(3), +c4 TIMESTAMP(4), +c5 TIMESTAMP(5), +c6 TIMESTAMP(6) +); +CREATE TABLE t3 +( +c0 DATETIME(0), +c1 DATETIME(1), +c2 DATETIME(2), +c3 DATETIME(3), +c4 DATETIME(4), +c5 DATETIME(5), +c6 DATETIME(6) +); +INSERT INTO t1 VALUES ('01:01:01','01:01:01.1','01:01:01.11','01:01:01.111','01:01:01.1111','01:01:01.11111','01:01:01.111111'); +INSERT INTO t2 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +INSERT INTO t3 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +TABLE_NAME TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH +t1 1 34 34 +t2 1 41 41 +t3 1 48 48 +SELECT * FROM t1;; +c0 01:01:01 +c1 01:01:01.1 +c2 01:01:01.11 +c3 01:01:01.111 +c4 01:01:01.1111 +c5 01:01:01.11111 +c6 01:01:01.111111 +SELECT * FROM t2;; +c0 2001-01-01 01:01:01 +c1 2001-01-01 01:01:01.1 +c2 2001-01-01 01:01:01.11 +c3 2001-01-01 01:01:01.111 +c4 2001-01-01 01:01:01.1111 +c5 2001-01-01 01:01:01.11111 +c6 2001-01-01 01:01:01.111111 +SELECT * FROM t3;; +c0 2001-01-01 01:01:01 +c1 2001-01-01 01:01:01.1 +c2 2001-01-01 01:01:01.11 +c3 2001-01-01 01:01:01.111 +c4 2001-01-01 01:01:01.1111 +c5 2001-01-01 01:01:01.11111 +c6 2001-01-01 01:01:01.111111 +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +TABLE_NAME TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH +t1 1 34 34 +t2 1 41 41 +t3 1 48 48 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +SET @@global.mysql56_temporal_format=DEFAULT; +SET @@global.mysql56_temporal_format=DEFAULT; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.test b/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.test new file mode 100644 index 0000000000000..99a70e011c47f --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_default_to_default.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_temporal_format_default_to_default.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.cnf b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.cnf new file mode 100644 index 0000000000000..b8e22e97ae9ea --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.cnf @@ -0,0 +1,6 @@ +!include my.cnf + +[mysqld.2] +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename=@ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.result b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.result new file mode 100644 index 0000000000000..74c287578d743 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.result @@ -0,0 +1,85 @@ +include/master-slave.inc +[connection master] +SET @@global.mysql56_temporal_format=false;; +SET @@global.mysql56_temporal_format=true;; +SELECT @@global.mysql56_temporal_format AS on_master; +on_master +0 +SELECT @@global.mysql56_temporal_format AS on_slave; +on_slave +1 +CREATE TABLE t1 +( +c0 TIME(0), +c1 TIME(1), +c2 TIME(2), +c3 TIME(3), +c4 TIME(4), +c5 TIME(5), +c6 TIME(6) +); +CREATE TABLE t2 +( +c0 TIMESTAMP(0), +c1 TIMESTAMP(1), +c2 TIMESTAMP(2), +c3 TIMESTAMP(3), +c4 TIMESTAMP(4), +c5 TIMESTAMP(5), +c6 TIMESTAMP(6) +); +CREATE TABLE t3 +( +c0 DATETIME(0), +c1 DATETIME(1), +c2 DATETIME(2), +c3 DATETIME(3), +c4 DATETIME(4), +c5 DATETIME(5), +c6 DATETIME(6) +); +INSERT INTO t1 VALUES ('01:01:01','01:01:01.1','01:01:01.11','01:01:01.111','01:01:01.1111','01:01:01.11111','01:01:01.111111'); +INSERT INTO t2 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +INSERT INTO t3 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +TABLE_NAME TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH +t1 1 33 33 +t2 1 41 41 +t3 1 50 50 +SELECT * FROM t1;; +c0 01:01:01 +c1 01:01:01.1 +c2 01:01:01.11 +c3 01:01:01.111 +c4 01:01:01.1111 +c5 01:01:01.11111 +c6 01:01:01.111111 +SELECT * FROM t2;; +c0 2001-01-01 01:01:01 +c1 2001-01-01 01:01:01.1 +c2 2001-01-01 01:01:01.11 +c3 2001-01-01 01:01:01.111 +c4 2001-01-01 01:01:01.1111 +c5 2001-01-01 01:01:01.11111 +c6 2001-01-01 01:01:01.111111 +SELECT * FROM t3;; +c0 2001-01-01 01:01:01 +c1 2001-01-01 01:01:01.1 +c2 2001-01-01 01:01:01.11 +c3 2001-01-01 01:01:01.111 +c4 2001-01-01 01:01:01.1111 +c5 2001-01-01 01:01:01.11111 +c6 2001-01-01 01:01:01.111111 +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +TABLE_NAME TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH +t1 1 34 34 +t2 1 41 41 +t3 1 48 48 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +SET @@global.mysql56_temporal_format=DEFAULT; +SET @@global.mysql56_temporal_format=DEFAULT; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.test b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.test new file mode 100644 index 0000000000000..1df4a48f0a90f --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mariadb53_to_mysql56.test @@ -0,0 +1,6 @@ +-- source include/have_binlog_format_mixed_or_statement.inc + +--let $force_master_mysql56_temporal_format=false; +--let $force_slave_mysql56_temporal_format=true; + +--source extra/rpl_tests/rpl_temporal_format_default_to_default.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.cnf b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.cnf new file mode 100644 index 0000000000000..b8e22e97ae9ea --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.cnf @@ -0,0 +1,6 @@ +!include my.cnf + +[mysqld.2] +plugin-load-add= @ENV.FILE_KEY_MANAGEMENT_SO +loose-file-key-management-filename=@ENV.MYSQLTEST_VARDIR/std_data/keys.txt +encrypt-binlog diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.result b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.result new file mode 100644 index 0000000000000..cc22e00aeb1b5 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.result @@ -0,0 +1,85 @@ +include/master-slave.inc +[connection master] +SET @@global.mysql56_temporal_format=true;; +SET @@global.mysql56_temporal_format=false;; +SELECT @@global.mysql56_temporal_format AS on_master; +on_master +1 +SELECT @@global.mysql56_temporal_format AS on_slave; +on_slave +0 +CREATE TABLE t1 +( +c0 TIME(0), +c1 TIME(1), +c2 TIME(2), +c3 TIME(3), +c4 TIME(4), +c5 TIME(5), +c6 TIME(6) +); +CREATE TABLE t2 +( +c0 TIMESTAMP(0), +c1 TIMESTAMP(1), +c2 TIMESTAMP(2), +c3 TIMESTAMP(3), +c4 TIMESTAMP(4), +c5 TIMESTAMP(5), +c6 TIMESTAMP(6) +); +CREATE TABLE t3 +( +c0 DATETIME(0), +c1 DATETIME(1), +c2 DATETIME(2), +c3 DATETIME(3), +c4 DATETIME(4), +c5 DATETIME(5), +c6 DATETIME(6) +); +INSERT INTO t1 VALUES ('01:01:01','01:01:01.1','01:01:01.11','01:01:01.111','01:01:01.1111','01:01:01.11111','01:01:01.111111'); +INSERT INTO t2 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +INSERT INTO t3 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +TABLE_NAME TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH +t1 1 34 34 +t2 1 41 41 +t3 1 48 48 +SELECT * FROM t1;; +c0 01:01:01 +c1 01:01:01.1 +c2 01:01:01.11 +c3 01:01:01.111 +c4 01:01:01.1111 +c5 01:01:01.11111 +c6 01:01:01.111111 +SELECT * FROM t2;; +c0 2001-01-01 01:01:01 +c1 2001-01-01 01:01:01.1 +c2 2001-01-01 01:01:01.11 +c3 2001-01-01 01:01:01.111 +c4 2001-01-01 01:01:01.1111 +c5 2001-01-01 01:01:01.11111 +c6 2001-01-01 01:01:01.111111 +SELECT * FROM t3;; +c0 2001-01-01 01:01:01 +c1 2001-01-01 01:01:01.1 +c2 2001-01-01 01:01:01.11 +c3 2001-01-01 01:01:01.111 +c4 2001-01-01 01:01:01.1111 +c5 2001-01-01 01:01:01.11111 +c6 2001-01-01 01:01:01.111111 +SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; +TABLE_NAME TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH +t1 1 33 33 +t2 1 41 41 +t3 1 50 50 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +SET @@global.mysql56_temporal_format=DEFAULT; +SET @@global.mysql56_temporal_format=DEFAULT; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.test b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.test new file mode 100644 index 0000000000000..f7436ed6fefc7 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_temporal_format_mysql56_to_mariadb53.test @@ -0,0 +1,4 @@ +--let $force_master_mysql56_temporal_format=true; +--let $force_slave_mysql56_temporal_format=false; + +--source extra/rpl_tests/rpl_temporal_format_default_to_default.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_typeconv.result b/mysql-test/suite/binlog_encryption/rpl_typeconv.result new file mode 100644 index 0000000000000..813c105c7dcb6 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_typeconv.result @@ -0,0 +1,540 @@ +include/master-slave.inc +[connection master] +set @saved_slave_type_conversions = @@global.slave_type_conversions; +CREATE TABLE type_conversions ( +TestNo INT AUTO_INCREMENT PRIMARY KEY, +Source TEXT, +Target TEXT, +Flags TEXT, +On_Master TEXT, +On_Slave TEXT, +Expected TEXT, +Compare INT, +Error TEXT); +SELECT @@global.slave_type_conversions; +@@global.slave_type_conversions + +SET GLOBAL SLAVE_TYPE_CONVERSIONS=''; +SELECT @@global.slave_type_conversions; +@@global.slave_type_conversions + +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_NON_LOSSY'; +SELECT @@global.slave_type_conversions; +@@global.slave_type_conversions +ALL_NON_LOSSY +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY'; +SELECT @@global.slave_type_conversions; +@@global.slave_type_conversions +ALL_LOSSY +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY'; +SELECT @@global.slave_type_conversions; +@@global.slave_type_conversions +ALL_LOSSY,ALL_NON_LOSSY +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY,NONEXISTING_BIT'; +ERROR 42000: Variable 'slave_type_conversions' can't be set to the value of 'NONEXISTING_BIT' +SELECT @@global.slave_type_conversions; +@@global.slave_type_conversions +ALL_LOSSY,ALL_NON_LOSSY +SET GLOBAL SLAVE_TYPE_CONVERSIONS=''; +**** Running tests with @@SLAVE_TYPE_CONVERSIONS = '' **** +include/rpl_reset.inc +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_NON_LOSSY'; +**** Running tests with @@SLAVE_TYPE_CONVERSIONS = 'ALL_NON_LOSSY' **** +include/rpl_reset.inc +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY'; +**** Running tests with @@SLAVE_TYPE_CONVERSIONS = 'ALL_LOSSY' **** +include/rpl_reset.inc +SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY'; +**** Running tests with @@SLAVE_TYPE_CONVERSIONS = 'ALL_LOSSY,ALL_NON_LOSSY' **** +include/rpl_reset.inc +**** Result of conversions **** +Source_Type Target_Type All_Type_Conversion_Flags Value_On_Slave +TINYBLOB TINYBLOB +TINYBLOB BLOB +TINYBLOB MEDIUMBLOB +TINYBLOB LONGBLOB +BLOB TINYBLOB +BLOB BLOB +BLOB MEDIUMBLOB +BLOB LONGBLOB +MEDIUMBLOB TINYBLOB +MEDIUMBLOB BLOB +MEDIUMBLOB MEDIUMBLOB +MEDIUMBLOB LONGBLOB +LONGBLOB TINYBLOB +LONGBLOB BLOB +LONGBLOB MEDIUMBLOB +LONGBLOB LONGBLOB +GEOMETRY BLOB +BLOB GEOMETRY +GEOMETRY GEOMETRY +BIT(1) BIT(1) +DATE DATE +ENUM('master',' ENUM('master',' +CHAR(10) ENUM('master',' +CHAR(10) SET('master','s +ENUM('master',' CHAR(10) +SET('master','s CHAR(10) +SET('master','s SET('master','s +SET('master','s SET('master','s +SET('0','1','2' SET('0','1','2' +SET('0','1','2' SET('0','1','2' +SET('0','1','2' SET('0','1','2' +SET('0','1','2' SET('0','1','2' +TINYINT TINYINT +TINYINT SMALLINT +TINYINT MEDIUMINT +TINYINT INT +TINYINT BIGINT +SMALLINT TINYINT +SMALLINT TINYINT +SMALLINT TINYINT UNSIGNE +SMALLINT SMALLINT +SMALLINT MEDIUMINT +SMALLINT INT +SMALLINT BIGINT +MEDIUMINT TINYINT +MEDIUMINT TINYINT +MEDIUMINT TINYINT UNSIGNE +MEDIUMINT SMALLINT +MEDIUMINT MEDIUMINT +MEDIUMINT INT +MEDIUMINT BIGINT +INT TINYINT +INT TINYINT +INT TINYINT UNSIGNE +INT SMALLINT +INT MEDIUMINT +INT INT +INT BIGINT +BIGINT TINYINT +BIGINT SMALLINT +BIGINT MEDIUMINT +BIGINT INT +BIGINT BIGINT +CHAR(20) CHAR(20) +CHAR(20) CHAR(30) +CHAR(20) CHAR(10) +CHAR(20) VARCHAR(20) +CHAR(20) VARCHAR(30) +CHAR(20) VARCHAR(10) +CHAR(20) TINYTEXT +CHAR(20) TEXT +CHAR(20) MEDIUMTEXT +CHAR(20) LONGTEXT +VARCHAR(20) VARCHAR(20) +VARCHAR(20) VARCHAR(30) +VARCHAR(20) VARCHAR(10) +VARCHAR(20) CHAR(30) +VARCHAR(20) CHAR(10) +VARCHAR(20) TINYTEXT +VARCHAR(20) TEXT +VARCHAR(20) MEDIUMTEXT +VARCHAR(20) LONGTEXT +VARCHAR(500) VARCHAR(500) +VARCHAR(500) VARCHAR(510) +VARCHAR(500) VARCHAR(255) +VARCHAR(500) TINYTEXT +VARCHAR(500) TEXT +VARCHAR(500) MEDIUMTEXT +VARCHAR(500) LONGTEXT +TINYTEXT VARCHAR(500) +TEXT VARCHAR(500) +MEDIUMTEXT VARCHAR(500) +LONGTEXT VARCHAR(500) +TINYTEXT CHAR(255) +TINYTEXT CHAR(250) +TEXT CHAR(255) +MEDIUMTEXT CHAR(255) +LONGTEXT CHAR(255) +TINYTEXT TINYTEXT +TINYTEXT TEXT +TEXT TINYTEXT +DECIMAL(10,5) DECIMAL(10,5) +DECIMAL(10,5) DECIMAL(10,6) +DECIMAL(10,5) DECIMAL(11,5) +DECIMAL(10,5) DECIMAL(11,6) +DECIMAL(10,5) DECIMAL(10,4) +DECIMAL(10,5) DECIMAL(9,5) +DECIMAL(10,5) DECIMAL(9,4) +FLOAT DECIMAL(10,5) +DOUBLE DECIMAL(10,5) +DECIMAL(10,5) FLOAT +DECIMAL(10,5) DOUBLE +FLOAT FLOAT +DOUBLE DOUBLE +FLOAT DOUBLE +DOUBLE FLOAT +BIT(5) BIT(5) +BIT(5) BIT(6) +BIT(6) BIT(5) +BIT(5) BIT(12) +BIT(12) BIT(5) +TINYBLOB TINYBLOB ALL_NON_LOSSY +TINYBLOB BLOB ALL_NON_LOSSY +TINYBLOB MEDIUMBLOB ALL_NON_LOSSY +TINYBLOB LONGBLOB ALL_NON_LOSSY +BLOB TINYBLOB ALL_NON_LOSSY +BLOB BLOB ALL_NON_LOSSY +BLOB MEDIUMBLOB ALL_NON_LOSSY +BLOB LONGBLOB ALL_NON_LOSSY +MEDIUMBLOB TINYBLOB ALL_NON_LOSSY +MEDIUMBLOB BLOB ALL_NON_LOSSY +MEDIUMBLOB MEDIUMBLOB ALL_NON_LOSSY +MEDIUMBLOB LONGBLOB ALL_NON_LOSSY +LONGBLOB TINYBLOB ALL_NON_LOSSY +LONGBLOB BLOB ALL_NON_LOSSY +LONGBLOB MEDIUMBLOB ALL_NON_LOSSY +LONGBLOB LONGBLOB ALL_NON_LOSSY +GEOMETRY BLOB ALL_NON_LOSSY +BLOB GEOMETRY ALL_NON_LOSSY +GEOMETRY GEOMETRY ALL_NON_LOSSY +BIT(1) BIT(1) ALL_NON_LOSSY +DATE DATE ALL_NON_LOSSY +ENUM('master',' ENUM('master',' ALL_NON_LOSSY +CHAR(10) ENUM('master',' ALL_NON_LOSSY +CHAR(10) SET('master','s ALL_NON_LOSSY +ENUM('master',' CHAR(10) ALL_NON_LOSSY +SET('master','s CHAR(10) ALL_NON_LOSSY +SET('master','s SET('master','s ALL_NON_LOSSY +SET('master','s SET('master','s ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_NON_LOSSY +TINYINT TINYINT ALL_NON_LOSSY +TINYINT SMALLINT ALL_NON_LOSSY +TINYINT MEDIUMINT ALL_NON_LOSSY +TINYINT INT ALL_NON_LOSSY +TINYINT BIGINT ALL_NON_LOSSY +SMALLINT TINYINT ALL_NON_LOSSY +SMALLINT TINYINT ALL_NON_LOSSY +SMALLINT TINYINT UNSIGNE ALL_NON_LOSSY +SMALLINT SMALLINT ALL_NON_LOSSY +SMALLINT MEDIUMINT ALL_NON_LOSSY +SMALLINT INT ALL_NON_LOSSY +SMALLINT BIGINT ALL_NON_LOSSY +MEDIUMINT TINYINT ALL_NON_LOSSY +MEDIUMINT TINYINT ALL_NON_LOSSY +MEDIUMINT TINYINT UNSIGNE ALL_NON_LOSSY +MEDIUMINT SMALLINT ALL_NON_LOSSY +MEDIUMINT MEDIUMINT ALL_NON_LOSSY +MEDIUMINT INT ALL_NON_LOSSY +MEDIUMINT BIGINT ALL_NON_LOSSY +INT TINYINT ALL_NON_LOSSY +INT TINYINT ALL_NON_LOSSY +INT TINYINT UNSIGNE ALL_NON_LOSSY +INT SMALLINT ALL_NON_LOSSY +INT MEDIUMINT ALL_NON_LOSSY +INT INT ALL_NON_LOSSY +INT BIGINT ALL_NON_LOSSY +BIGINT TINYINT ALL_NON_LOSSY +BIGINT SMALLINT ALL_NON_LOSSY +BIGINT MEDIUMINT ALL_NON_LOSSY +BIGINT INT ALL_NON_LOSSY +BIGINT BIGINT ALL_NON_LOSSY +CHAR(20) CHAR(20) ALL_NON_LOSSY +CHAR(20) CHAR(30) ALL_NON_LOSSY +CHAR(20) CHAR(10) ALL_NON_LOSSY +CHAR(20) VARCHAR(20) ALL_NON_LOSSY +CHAR(20) VARCHAR(30) ALL_NON_LOSSY +CHAR(20) VARCHAR(10) ALL_NON_LOSSY +CHAR(20) TINYTEXT ALL_NON_LOSSY +CHAR(20) TEXT ALL_NON_LOSSY +CHAR(20) MEDIUMTEXT ALL_NON_LOSSY +CHAR(20) LONGTEXT ALL_NON_LOSSY +VARCHAR(20) VARCHAR(20) ALL_NON_LOSSY +VARCHAR(20) VARCHAR(30) ALL_NON_LOSSY +VARCHAR(20) VARCHAR(10) ALL_NON_LOSSY +VARCHAR(20) CHAR(30) ALL_NON_LOSSY +VARCHAR(20) CHAR(10) ALL_NON_LOSSY +VARCHAR(20) TINYTEXT ALL_NON_LOSSY +VARCHAR(20) TEXT ALL_NON_LOSSY +VARCHAR(20) MEDIUMTEXT ALL_NON_LOSSY +VARCHAR(20) LONGTEXT ALL_NON_LOSSY +VARCHAR(500) VARCHAR(500) ALL_NON_LOSSY +VARCHAR(500) VARCHAR(510) ALL_NON_LOSSY +VARCHAR(500) VARCHAR(255) ALL_NON_LOSSY +VARCHAR(500) TINYTEXT ALL_NON_LOSSY +VARCHAR(500) TEXT ALL_NON_LOSSY +VARCHAR(500) MEDIUMTEXT ALL_NON_LOSSY +VARCHAR(500) LONGTEXT ALL_NON_LOSSY +TINYTEXT VARCHAR(500) ALL_NON_LOSSY +TEXT VARCHAR(500) ALL_NON_LOSSY +MEDIUMTEXT VARCHAR(500) ALL_NON_LOSSY +LONGTEXT VARCHAR(500) ALL_NON_LOSSY +TINYTEXT CHAR(255) ALL_NON_LOSSY +TINYTEXT CHAR(250) ALL_NON_LOSSY +TEXT CHAR(255) ALL_NON_LOSSY +MEDIUMTEXT CHAR(255) ALL_NON_LOSSY +LONGTEXT CHAR(255) ALL_NON_LOSSY +TINYTEXT TINYTEXT ALL_NON_LOSSY +TINYTEXT TEXT ALL_NON_LOSSY +TEXT TINYTEXT ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(10,5) ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(10,6) ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(11,5) ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(11,6) ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(10,4) ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(9,5) ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(9,4) ALL_NON_LOSSY +FLOAT DECIMAL(10,5) ALL_NON_LOSSY +DOUBLE DECIMAL(10,5) ALL_NON_LOSSY +DECIMAL(10,5) FLOAT ALL_NON_LOSSY +DECIMAL(10,5) DOUBLE ALL_NON_LOSSY +FLOAT FLOAT ALL_NON_LOSSY +DOUBLE DOUBLE ALL_NON_LOSSY +FLOAT DOUBLE ALL_NON_LOSSY +DOUBLE FLOAT ALL_NON_LOSSY +BIT(5) BIT(5) ALL_NON_LOSSY +BIT(5) BIT(6) ALL_NON_LOSSY +BIT(6) BIT(5) ALL_NON_LOSSY +BIT(5) BIT(12) ALL_NON_LOSSY +BIT(12) BIT(5) ALL_NON_LOSSY +TINYBLOB TINYBLOB ALL_LOSSY +TINYBLOB BLOB ALL_LOSSY +TINYBLOB MEDIUMBLOB ALL_LOSSY +TINYBLOB LONGBLOB ALL_LOSSY +BLOB TINYBLOB ALL_LOSSY +BLOB BLOB ALL_LOSSY +BLOB MEDIUMBLOB ALL_LOSSY +BLOB LONGBLOB ALL_LOSSY +MEDIUMBLOB TINYBLOB ALL_LOSSY +MEDIUMBLOB BLOB ALL_LOSSY +MEDIUMBLOB MEDIUMBLOB ALL_LOSSY +MEDIUMBLOB LONGBLOB ALL_LOSSY +LONGBLOB TINYBLOB ALL_LOSSY +LONGBLOB BLOB ALL_LOSSY +LONGBLOB MEDIUMBLOB ALL_LOSSY +LONGBLOB LONGBLOB ALL_LOSSY +GEOMETRY BLOB ALL_LOSSY +BLOB GEOMETRY ALL_LOSSY +GEOMETRY GEOMETRY ALL_LOSSY +BIT(1) BIT(1) ALL_LOSSY +DATE DATE ALL_LOSSY +ENUM('master',' ENUM('master',' ALL_LOSSY +CHAR(10) ENUM('master',' ALL_LOSSY +CHAR(10) SET('master','s ALL_LOSSY +ENUM('master',' CHAR(10) ALL_LOSSY +SET('master','s CHAR(10) ALL_LOSSY +SET('master','s SET('master','s ALL_LOSSY +SET('master','s SET('master','s ALL_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY +TINYINT TINYINT ALL_LOSSY +TINYINT SMALLINT ALL_LOSSY +TINYINT MEDIUMINT ALL_LOSSY +TINYINT INT ALL_LOSSY +TINYINT BIGINT ALL_LOSSY +SMALLINT TINYINT ALL_LOSSY +SMALLINT TINYINT ALL_LOSSY +SMALLINT TINYINT UNSIGNE ALL_LOSSY +SMALLINT SMALLINT ALL_LOSSY +SMALLINT MEDIUMINT ALL_LOSSY +SMALLINT INT ALL_LOSSY +SMALLINT BIGINT ALL_LOSSY +MEDIUMINT TINYINT ALL_LOSSY +MEDIUMINT TINYINT ALL_LOSSY +MEDIUMINT TINYINT UNSIGNE ALL_LOSSY +MEDIUMINT SMALLINT ALL_LOSSY +MEDIUMINT MEDIUMINT ALL_LOSSY +MEDIUMINT INT ALL_LOSSY +MEDIUMINT BIGINT ALL_LOSSY +INT TINYINT ALL_LOSSY +INT TINYINT ALL_LOSSY +INT TINYINT UNSIGNE ALL_LOSSY +INT SMALLINT ALL_LOSSY +INT MEDIUMINT ALL_LOSSY +INT INT ALL_LOSSY +INT BIGINT ALL_LOSSY +BIGINT TINYINT ALL_LOSSY +BIGINT SMALLINT ALL_LOSSY +BIGINT MEDIUMINT ALL_LOSSY +BIGINT INT ALL_LOSSY +BIGINT BIGINT ALL_LOSSY +CHAR(20) CHAR(20) ALL_LOSSY +CHAR(20) CHAR(30) ALL_LOSSY +CHAR(20) CHAR(10) ALL_LOSSY +CHAR(20) VARCHAR(20) ALL_LOSSY +CHAR(20) VARCHAR(30) ALL_LOSSY +CHAR(20) VARCHAR(10) ALL_LOSSY +CHAR(20) TINYTEXT ALL_LOSSY +CHAR(20) TEXT ALL_LOSSY +CHAR(20) MEDIUMTEXT ALL_LOSSY +CHAR(20) LONGTEXT ALL_LOSSY +VARCHAR(20) VARCHAR(20) ALL_LOSSY +VARCHAR(20) VARCHAR(30) ALL_LOSSY +VARCHAR(20) VARCHAR(10) ALL_LOSSY +VARCHAR(20) CHAR(30) ALL_LOSSY +VARCHAR(20) CHAR(10) ALL_LOSSY +VARCHAR(20) TINYTEXT ALL_LOSSY +VARCHAR(20) TEXT ALL_LOSSY +VARCHAR(20) MEDIUMTEXT ALL_LOSSY +VARCHAR(20) LONGTEXT ALL_LOSSY +VARCHAR(500) VARCHAR(500) ALL_LOSSY +VARCHAR(500) VARCHAR(510) ALL_LOSSY +VARCHAR(500) VARCHAR(255) ALL_LOSSY +VARCHAR(500) TINYTEXT ALL_LOSSY +VARCHAR(500) TEXT ALL_LOSSY +VARCHAR(500) MEDIUMTEXT ALL_LOSSY +VARCHAR(500) LONGTEXT ALL_LOSSY +TINYTEXT VARCHAR(500) ALL_LOSSY +TEXT VARCHAR(500) ALL_LOSSY +MEDIUMTEXT VARCHAR(500) ALL_LOSSY +LONGTEXT VARCHAR(500) ALL_LOSSY +TINYTEXT CHAR(255) ALL_LOSSY +TINYTEXT CHAR(250) ALL_LOSSY +TEXT CHAR(255) ALL_LOSSY +MEDIUMTEXT CHAR(255) ALL_LOSSY +LONGTEXT CHAR(255) ALL_LOSSY +TINYTEXT TINYTEXT ALL_LOSSY +TINYTEXT TEXT ALL_LOSSY +TEXT TINYTEXT ALL_LOSSY +DECIMAL(10,5) DECIMAL(10,5) ALL_LOSSY +DECIMAL(10,5) DECIMAL(10,6) ALL_LOSSY +DECIMAL(10,5) DECIMAL(11,5) ALL_LOSSY +DECIMAL(10,5) DECIMAL(11,6) ALL_LOSSY +DECIMAL(10,5) DECIMAL(10,4) ALL_LOSSY +DECIMAL(10,5) DECIMAL(9,5) ALL_LOSSY +DECIMAL(10,5) DECIMAL(9,4) ALL_LOSSY +FLOAT DECIMAL(10,5) ALL_LOSSY +DOUBLE DECIMAL(10,5) ALL_LOSSY +DECIMAL(10,5) FLOAT ALL_LOSSY +DECIMAL(10,5) DOUBLE ALL_LOSSY +FLOAT FLOAT ALL_LOSSY +DOUBLE DOUBLE ALL_LOSSY +FLOAT DOUBLE ALL_LOSSY +DOUBLE FLOAT ALL_LOSSY +BIT(5) BIT(5) ALL_LOSSY +BIT(5) BIT(6) ALL_LOSSY +BIT(6) BIT(5) ALL_LOSSY +BIT(5) BIT(12) ALL_LOSSY +BIT(12) BIT(5) ALL_LOSSY +TINYBLOB TINYBLOB ALL_LOSSY,ALL_NON_LOSSY +TINYBLOB BLOB ALL_LOSSY,ALL_NON_LOSSY +TINYBLOB MEDIUMBLOB ALL_LOSSY,ALL_NON_LOSSY +TINYBLOB LONGBLOB ALL_LOSSY,ALL_NON_LOSSY +BLOB TINYBLOB ALL_LOSSY,ALL_NON_LOSSY +BLOB BLOB ALL_LOSSY,ALL_NON_LOSSY +BLOB MEDIUMBLOB ALL_LOSSY,ALL_NON_LOSSY +BLOB LONGBLOB ALL_LOSSY,ALL_NON_LOSSY +MEDIUMBLOB TINYBLOB ALL_LOSSY,ALL_NON_LOSSY +MEDIUMBLOB BLOB ALL_LOSSY,ALL_NON_LOSSY +MEDIUMBLOB MEDIUMBLOB ALL_LOSSY,ALL_NON_LOSSY +MEDIUMBLOB LONGBLOB ALL_LOSSY,ALL_NON_LOSSY +LONGBLOB TINYBLOB ALL_LOSSY,ALL_NON_LOSSY +LONGBLOB BLOB ALL_LOSSY,ALL_NON_LOSSY +LONGBLOB MEDIUMBLOB ALL_LOSSY,ALL_NON_LOSSY +LONGBLOB LONGBLOB ALL_LOSSY,ALL_NON_LOSSY +GEOMETRY BLOB ALL_LOSSY,ALL_NON_LOSSY +BLOB GEOMETRY ALL_LOSSY,ALL_NON_LOSSY +GEOMETRY GEOMETRY ALL_LOSSY,ALL_NON_LOSSY +BIT(1) BIT(1) ALL_LOSSY,ALL_NON_LOSSY +DATE DATE ALL_LOSSY,ALL_NON_LOSSY +ENUM('master',' ENUM('master',' ALL_LOSSY,ALL_NON_LOSSY +CHAR(10) ENUM('master',' ALL_LOSSY,ALL_NON_LOSSY +CHAR(10) SET('master','s ALL_LOSSY,ALL_NON_LOSSY +ENUM('master',' CHAR(10) ALL_LOSSY,ALL_NON_LOSSY +SET('master','s CHAR(10) ALL_LOSSY,ALL_NON_LOSSY +SET('master','s SET('master','s ALL_LOSSY,ALL_NON_LOSSY +SET('master','s SET('master','s ALL_LOSSY,ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY,ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY,ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY,ALL_NON_LOSSY +SET('0','1','2' SET('0','1','2' ALL_LOSSY,ALL_NON_LOSSY +TINYINT TINYINT ALL_LOSSY,ALL_NON_LOSSY +TINYINT SMALLINT ALL_LOSSY,ALL_NON_LOSSY +TINYINT MEDIUMINT ALL_LOSSY,ALL_NON_LOSSY +TINYINT INT ALL_LOSSY,ALL_NON_LOSSY +TINYINT BIGINT ALL_LOSSY,ALL_NON_LOSSY +SMALLINT TINYINT ALL_LOSSY,ALL_NON_LOSSY +SMALLINT TINYINT ALL_LOSSY,ALL_NON_LOSSY +SMALLINT TINYINT UNSIGNE ALL_LOSSY,ALL_NON_LOSSY +SMALLINT SMALLINT ALL_LOSSY,ALL_NON_LOSSY +SMALLINT MEDIUMINT ALL_LOSSY,ALL_NON_LOSSY +SMALLINT INT ALL_LOSSY,ALL_NON_LOSSY +SMALLINT BIGINT ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT TINYINT ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT TINYINT ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT TINYINT UNSIGNE ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT SMALLINT ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT MEDIUMINT ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT INT ALL_LOSSY,ALL_NON_LOSSY +MEDIUMINT BIGINT ALL_LOSSY,ALL_NON_LOSSY +INT TINYINT ALL_LOSSY,ALL_NON_LOSSY +INT TINYINT ALL_LOSSY,ALL_NON_LOSSY +INT TINYINT UNSIGNE ALL_LOSSY,ALL_NON_LOSSY +INT SMALLINT ALL_LOSSY,ALL_NON_LOSSY +INT MEDIUMINT ALL_LOSSY,ALL_NON_LOSSY +INT INT ALL_LOSSY,ALL_NON_LOSSY +INT BIGINT ALL_LOSSY,ALL_NON_LOSSY +BIGINT TINYINT ALL_LOSSY,ALL_NON_LOSSY +BIGINT SMALLINT ALL_LOSSY,ALL_NON_LOSSY +BIGINT MEDIUMINT ALL_LOSSY,ALL_NON_LOSSY +BIGINT INT ALL_LOSSY,ALL_NON_LOSSY +BIGINT BIGINT ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) CHAR(20) ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) CHAR(30) ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) CHAR(10) ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) VARCHAR(20) ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) VARCHAR(30) ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) VARCHAR(10) ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) TINYTEXT ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) TEXT ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) MEDIUMTEXT ALL_LOSSY,ALL_NON_LOSSY +CHAR(20) LONGTEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) VARCHAR(20) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) VARCHAR(30) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) VARCHAR(10) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) CHAR(30) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) CHAR(10) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) TINYTEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) TEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) MEDIUMTEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(20) LONGTEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) VARCHAR(500) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) VARCHAR(510) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) VARCHAR(255) ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) TINYTEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) TEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) MEDIUMTEXT ALL_LOSSY,ALL_NON_LOSSY +VARCHAR(500) LONGTEXT ALL_LOSSY,ALL_NON_LOSSY +TINYTEXT VARCHAR(500) ALL_LOSSY,ALL_NON_LOSSY +TEXT VARCHAR(500) ALL_LOSSY,ALL_NON_LOSSY +MEDIUMTEXT VARCHAR(500) ALL_LOSSY,ALL_NON_LOSSY +LONGTEXT VARCHAR(500) ALL_LOSSY,ALL_NON_LOSSY +TINYTEXT CHAR(255) ALL_LOSSY,ALL_NON_LOSSY +TINYTEXT CHAR(250) ALL_LOSSY,ALL_NON_LOSSY +TEXT CHAR(255) ALL_LOSSY,ALL_NON_LOSSY +MEDIUMTEXT CHAR(255) ALL_LOSSY,ALL_NON_LOSSY +LONGTEXT CHAR(255) ALL_LOSSY,ALL_NON_LOSSY +TINYTEXT TINYTEXT ALL_LOSSY,ALL_NON_LOSSY +TINYTEXT TEXT ALL_LOSSY,ALL_NON_LOSSY +TEXT TINYTEXT ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(10,5) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(10,6) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(11,5) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(11,6) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(10,4) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(9,5) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DECIMAL(9,4) ALL_LOSSY,ALL_NON_LOSSY +FLOAT DECIMAL(10,5) ALL_LOSSY,ALL_NON_LOSSY +DOUBLE DECIMAL(10,5) ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) FLOAT ALL_LOSSY,ALL_NON_LOSSY +DECIMAL(10,5) DOUBLE ALL_LOSSY,ALL_NON_LOSSY +FLOAT FLOAT ALL_LOSSY,ALL_NON_LOSSY +DOUBLE DOUBLE ALL_LOSSY,ALL_NON_LOSSY +FLOAT DOUBLE ALL_LOSSY,ALL_NON_LOSSY +DOUBLE FLOAT ALL_LOSSY,ALL_NON_LOSSY +BIT(5) BIT(5) ALL_LOSSY,ALL_NON_LOSSY +BIT(5) BIT(6) ALL_LOSSY,ALL_NON_LOSSY +BIT(6) BIT(5) ALL_LOSSY,ALL_NON_LOSSY +BIT(5) BIT(12) ALL_LOSSY,ALL_NON_LOSSY +BIT(12) BIT(5) ALL_LOSSY,ALL_NON_LOSSY +DROP TABLE type_conversions; +call mtr.add_suppression("Slave SQL.*Column 1 of table .test.t1. cannot be converted from type.* error.* 1677"); +DROP TABLE t1; +set global slave_type_conversions = @saved_slave_type_conversions; +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/rpl_typeconv.test b/mysql-test/suite/binlog_encryption/rpl_typeconv.test new file mode 100644 index 0000000000000..4dbfc27d088e5 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/rpl_typeconv.test @@ -0,0 +1 @@ +--source extra/rpl_tests/rpl_typeconv.inc diff --git a/mysql-test/suite/binlog_encryption/suite.pm b/mysql-test/suite/binlog_encryption/suite.pm new file mode 100644 index 0000000000000..f1d5e3aaea73d --- /dev/null +++ b/mysql-test/suite/binlog_encryption/suite.pm @@ -0,0 +1,18 @@ +package My::Suite::BinlogEncryption; + +@ISA = qw(My::Suite); + +return "No file key management plugin" unless defined $ENV{FILE_KEY_MANAGEMENT_SO}; + +sub skip_combinations { + my @combinations; + + $skip{'encryption_algorithms.combinations'} = [ 'ctr' ] + unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/ + and $1 ge "1.0.1"; + + %skip; +} + +bless { }; + diff --git a/mysql-test/suite/binlog_encryption/testdata.inc b/mysql-test/suite/binlog_encryption/testdata.inc new file mode 100644 index 0000000000000..f9499112e91f0 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/testdata.inc @@ -0,0 +1,207 @@ +# +# This include file creates some basic events which should go to the binary log. +# What happens to the binary log depends on the test which calls the file, +# and should be checked from the test. +# +# Names are intentionally long and ugly, to make grepping more reliable. +# +# Some of events are considered unsafe for SBR (not necessarily correctly, +# but here isn't the place to check the logic), so we just suppress the warning. +# +# For those few queries which produce result sets (e.g. ANALYZE, CHECKSUM etc.), +# we don't care about the result, so it will not be printed to the output. + +call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); + +# +# Some DDL +# + +CREATE DATABASE database_name_to_encrypt; +USE database_name_to_encrypt; + +CREATE USER user_name_to_encrypt; +GRANT ALL ON database_name_to_encrypt.* TO user_name_to_encrypt; +SET PASSWORD FOR user_name_to_encrypt = PASSWORD('password_to_encrypt'); + +CREATE TABLE innodb_table_name_to_encrypt ( + int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, + timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, + blob_column_name_to_encrypt BLOB, + virt_column_name_to_encrypt INT AS (int_column_name_to_encrypt % 10) VIRTUAL, + pers_column_name_to_encrypt INT AS (int_column_name_to_encrypt) PERSISTENT, + INDEX `index_name_to_encrypt`(`timestamp_column_name_to_encrypt`) +) ENGINE=InnoDB + PARTITION BY RANGE (int_column_name_to_encrypt) + SUBPARTITION BY KEY (int_column_name_to_encrypt) + SUBPARTITIONS 2 ( + PARTITION partition0_name_to_encrypt VALUES LESS THAN (100), + PARTITION partition1_name_to_encrypt VALUES LESS THAN (MAXVALUE) + ) +; + +CREATE TABLE myisam_table_name_to_encrypt ( + int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, + char_column_name_to_encrypt VARCHAR(255), + datetime_column_name_to_encrypt DATETIME, + text_column_name_to_encrypt TEXT +) ENGINE=MyISAM; + +CREATE TABLE aria_table_name_to_encrypt ( + int_column_name_to_encrypt INT AUTO_INCREMENT PRIMARY KEY, + varchar_column_name_to_encrypt VARCHAR(1024), + enum_column_name_to_encrypt ENUM( + 'enum_value1_to_encrypt', + 'enum_value2_to_encrypt' + ), + timestamp_column_name_to_encrypt TIMESTAMP(6) NULL, + blob_column_name_to_encrypt BLOB +) ENGINE=Aria; + +CREATE TRIGGER trigger_name_to_encrypt + AFTER INSERT ON myisam_table_name_to_encrypt FOR EACH ROW + INSERT INTO aria_table_name_to_encrypt (varchar_column_name_to_encrypt) + VALUES (NEW.char_column_name_to_encrypt); + +CREATE DEFINER=user_name_to_encrypt VIEW view_name_to_encrypt + AS SELECT * FROM innodb_table_name_to_encrypt; + +CREATE FUNCTION func_name_to_encrypt (func_parameter_to_encrypt INT) + RETURNS VARCHAR(64) + RETURN 'func_result_to_encrypt'; + +--delimiter $$ +CREATE PROCEDURE proc_name_to_encrypt ( + IN proc_in_parameter_to_encrypt CHAR(32), + OUT proc_out_parameter_to_encrypt INT +) +BEGIN + DECLARE procvar_name_to_encrypt CHAR(64) DEFAULT 'procvar_val_to_encrypt'; + DECLARE cursor_name_to_encrypt CURSOR FOR + SELECT virt_column_name_to_encrypt FROM innodb_table_name_to_encrypt; + DECLARE EXIT HANDLER FOR NOT FOUND + BEGIN + SET @stmt_var_to_encrypt = CONCAT( + "SELECT + IF (RAND()>0.5,'enum_value2_to_encrypt','enum_value1_to_encrypt') + FROM innodb_table_name_to_encrypt + INTO OUTFILE '", proc_in_parameter_to_encrypt, "'"); + PREPARE stmt_to_encrypt FROM @stmt_var_to_encrypt; + EXECUTE stmt_to_encrypt; + DEALLOCATE PREPARE stmt_to_encrypt; + END; + OPEN cursor_name_to_encrypt; + proc_label_to_encrypt: LOOP + FETCH cursor_name_to_encrypt INTO procvar_name_to_encrypt; + END LOOP; + CLOSE cursor_name_to_encrypt; +END $$ +--delimiter ; + +CREATE SERVER server_name_to_encrypt + FOREIGN DATA WRAPPER mysql + OPTIONS (HOST 'host_name_to_encrypt'); + +--let $_cur_con= $CURRENT_CONNECTION +--connect (con1,localhost,user_name_to_encrypt,password_to_encrypt,database_name_to_encrypt) +CREATE TEMPORARY TABLE tmp_table_name_to_encrypt ( + float_column_name_to_encrypt FLOAT, + binary_column_name_to_encrypt BINARY(64) +); +--disconnect con1 +--connection $_cur_con + +CREATE INDEX index_name_to_encrypt + ON myisam_table_name_to_encrypt (datetime_column_name_to_encrypt); + +ALTER DATABASE database_name_to_encrypt CHARACTER SET utf8; + +ALTER TABLE innodb_table_name_to_encrypt + MODIFY timestamp_column_name_to_encrypt TIMESTAMP NOT NULL + DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +; + +ALTER ALGORITHM=MERGE VIEW view_name_to_encrypt + AS SELECT * FROM innodb_table_name_to_encrypt; + +RENAME TABLE innodb_table_name_to_encrypt TO new_table_name_to_encrypt; +ALTER TABLE new_table_name_to_encrypt RENAME TO innodb_table_name_to_encrypt; + +# +# Some DML +# + +--disable_warnings + +set @user_var1_to_encrypt= 'dyncol1_val_to_encrypt'; +set @user_var2_to_encrypt= 'dyncol2_name_to_encrypt'; + +INSERT INTO view_name_to_encrypt VALUES + (1, NOW(6), COLUMN_CREATE('dyncol1_name_to_encrypt',@user_var1_to_encrypt), NULL, NULL), + (2, NOW(6), COLUMN_CREATE(@user_var2_to_encrypt,'dyncol2_val_to_encrypt'), NULL, NULL) +; +--delimiter $$ +BEGIN NOT ATOMIC + DECLARE counter_name_to_encrypt INT DEFAULT 0; + START TRANSACTION; + WHILE counter_name_to_encrypt<12 DO + INSERT INTO innodb_table_name_to_encrypt + SELECT NULL, NOW(6), blob_column_name_to_encrypt, NULL, NULL + FROM innodb_table_name_to_encrypt + ORDER BY int_column_name_to_encrypt; + SET counter_name_to_encrypt = counter_name_to_encrypt+1; + END WHILE; + COMMIT; + END +$$ +--delimiter ; + +INSERT INTO myisam_table_name_to_encrypt + SELECT NULL, 'char_literal_to_encrypt', NULL, 'text_to_encrypt'; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) + SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) + SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; +INSERT INTO myisam_table_name_to_encrypt (char_column_name_to_encrypt) + SELECT char_column_name_to_encrypt FROM myisam_table_name_to_encrypt; + +CALL proc_name_to_encrypt('file_name_to_encrypt',@useless_var_to_encrypt); + +TRUNCATE TABLE aria_table_name_to_encrypt; + +LOAD DATA INFILE 'file_name_to_encrypt' INTO TABLE aria_table_name_to_encrypt + (enum_column_name_to_encrypt); + +--let datadir= `SELECT @@datadir` +--replace_result $datadir +eval LOAD DATA LOCAL INFILE '$datadir/database_name_to_encrypt/file_name_to_encrypt' + INTO TABLE aria_table_name_to_encrypt (enum_column_name_to_encrypt); +--remove_file $datadir/database_name_to_encrypt/file_name_to_encrypt + +UPDATE view_name_to_encrypt SET blob_column_name_to_encrypt = + COLUMN_CREATE('dyncol1_name_to_encrypt',func_name_to_encrypt(0)) +; + +DELETE FROM aria_table_name_to_encrypt ORDER BY int_column_name_to_encrypt LIMIT 10; + +--enable_warnings + +# +# Other statements +# + +--disable_result_log +ANALYZE TABLE myisam_table_name_to_encrypt; +CHECK TABLE aria_table_name_to_encrypt; +CHECKSUM TABLE innodb_table_name_to_encrypt, myisam_table_name_to_encrypt; +--enable_result_log +RENAME USER user_name_to_encrypt to new_user_name_to_encrypt; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM new_user_name_to_encrypt; + +# +# Cleanup +# + +DROP DATABASE database_name_to_encrypt; +DROP USER new_user_name_to_encrypt; +DROP SERVER server_name_to_encrypt; diff --git a/mysql-test/suite/binlog_encryption/testdata.opt b/mysql-test/suite/binlog_encryption/testdata.opt new file mode 100644 index 0000000000000..b0c5b9c818870 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/testdata.opt @@ -0,0 +1 @@ +--partition diff --git a/mysql-test/suite/multi_source/gtid.test b/mysql-test/suite/multi_source/gtid.test index bebee66068f70..c81ca20a25457 100644 --- a/mysql-test/suite/multi_source/gtid.test +++ b/mysql-test/suite/multi_source/gtid.test @@ -150,22 +150,22 @@ SET GLOBAL gtid_domain_id=0; --source include/wait_condition.inc --sorted_result STOP ALL SLAVES; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave1 --connection slave2 SET GLOBAL gtid_domain_id=0; --sorted_result STOP ALL SLAVES; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave2 --connection master1 SET GLOBAL gtid_domain_id=0; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 --connection master2 SET GLOBAL gtid_domain_id=0; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master2 diff --git a/mysql-test/suite/multi_source/gtid_ignore_duplicates.test b/mysql-test/suite/multi_source/gtid_ignore_duplicates.test index 4d98b5c2ee7a6..218d91aa7fbcc 100644 --- a/mysql-test/suite/multi_source/gtid_ignore_duplicates.test +++ b/mysql-test/suite/multi_source/gtid_ignore_duplicates.test @@ -431,20 +431,20 @@ SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates; --connection server_1 DROP TABLE t1; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect server_1 --connection server_2 DROP TABLE t1; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect server_2 --connection server_3 DROP TABLE t1; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect server_3 --connection server_4 DROP TABLE t1; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect server_4 diff --git a/mysql-test/suite/multi_source/info_logs.test b/mysql-test/suite/multi_source/info_logs.test index 2f3b193481737..372cd66e5cc21 100644 --- a/mysql-test/suite/multi_source/info_logs.test +++ b/mysql-test/suite/multi_source/info_logs.test @@ -187,14 +187,14 @@ show all slaves status; # Cleanup ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 --connection master2 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master2 diff --git a/mysql-test/suite/multi_source/load_data.test b/mysql-test/suite/multi_source/load_data.test index ca2391a9c8dbd..94b328d56aec8 100644 --- a/mysql-test/suite/multi_source/load_data.test +++ b/mysql-test/suite/multi_source/load_data.test @@ -61,11 +61,11 @@ drop table t2; --sorted_result stop all slaves; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 --connection master2 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master2 diff --git a/mysql-test/suite/multi_source/multisource.test b/mysql-test/suite/multi_source/multisource.test index de6874d642333..fc58fe8180300 100644 --- a/mysql-test/suite/multi_source/multisource.test +++ b/mysql-test/suite/multi_source/multisource.test @@ -1,291 +1 @@ -# -# Test basic replication functionality -# in multi-source setup -# - ---source include/not_embedded.inc ---source include/have_innodb.inc ---source include/binlog_start_pos.inc ---let $rpl_server_count= 0 - ---connect (slave,127.0.0.1,root,,,$SERVER_MYPORT_3) - -# MDEV-3984: crash/read of freed memory when changing master with named connection -# This fails after adding the new master 'abc', check we do not free twice. ---error ER_RELAY_LOG_INIT -change master 'abc' to relay_log_file=''; -# This fails before adding the new master, check that we do free it. ---error ER_WRONG_ARGUMENTS -change master 'abc2' to master_host=''; - - -# Start replication from the first master - ---replace_result $SERVER_MYPORT_1 MYPORT_1 -eval change master 'master1' to -master_port=$SERVER_MYPORT_1, -master_host='127.0.0.1', -master_user='root'; - -start slave 'master1'; -set default_master_connection = 'master1'; ---source include/wait_for_slave_to_start.inc - ---connect (master1,127.0.0.1,root,,,$SERVER_MYPORT_1) ---save_master_pos - ---connection slave ---sync_with_master 0,'master1' - -# Here and further: add an extra check on SQL thread status -# as the normal sync is not always enough ---source wait_for_sql_thread_read_all.inc - -# each of the 3 commands should produce -# 'master1' status - -let $wait_for_all= 1; -let $show_statement= SHOW ALL SLAVES STATUS; -let $field= Slave_IO_State; -let $condition= = 'Waiting for master to send event'; ---source include/wait_show_condition.inc - ---echo # ---echo # Checking SHOW SLAVE 'master1' STATUS ---echo # ---let $status_items= Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno ---let $slave_field_result_replace= /$SERVER_MYPORT_1/MYPORT_1/ ---let $slave_name= 'master1' ---source include/show_slave_status.inc ---let $slave_name= - ---echo # ---echo # Checking SHOW SLAVE STATUS ---echo # ---source include/show_slave_status.inc - ---echo # ---echo # Checking SHOW ALL SLAVES STATUS ---echo # ---let $all_slaves_status= 1 ---let $status_items= Connection_name, Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno, Slave_heartbeat_period ---source include/show_slave_status.inc ---let $all_slaves_status= ---echo # - - -# Check that replication actually works - ---connection master1 - ---disable_warnings -drop database if exists db1; ---enable_warnings -create database db1; -use db1; -create table t1 (i int auto_increment, f1 varchar(16), primary key pk (i,f1)) engine=MyISAM; -insert into t1 (f1) values ('one'),('two'); ---save_master_pos - ---connection slave ---sync_with_master 0,'master1' - ---sorted_result -select * from db1.t1; - ---let $datadir = `SELECT @@datadir` - ---echo # List of relay log files in the datadir ---list_files $datadir mysqld-relay-bin-master1.* - -# Check that relay logs are recognizable - -let binlog_start=4; -let binlog_file=; -source include/show_relaylog_events.inc; -let binlog_file= mysqld-relay-bin-master1.000002; -source include/show_relaylog_events.inc; - -# Try to configure connection with the same name again, -# should get an error because the slave is running - ---replace_result $SERVER_MYPORT_2 MYPORT_2 ---error ER_SLAVE_MUST_STOP -eval change master 'master1' to -master_port=$SERVER_MYPORT_2, -master_host='127.0.0.1', -master_user='root'; - -# Try to configure using the default connection name -# (which is 'master1' at the moment), -# again, should get an error - ---replace_result $SERVER_MYPORT_2 MYPORT_2 ---error ER_SLAVE_MUST_STOP -eval change master to -master_port=$SERVER_MYPORT_2, -master_host='127.0.0.1', -master_user='root'; - -# Try to configure a connection with the same master -# using a different name, should get a conflict - ---replace_result $SERVER_MYPORT_1 MYPORT_1 ---error ER_CONNECTION_ALREADY_EXISTS -eval change master 'master2' to -master_port=$SERVER_MYPORT_1, -master_host='127.0.0.1', -master_user='root'; - - -# Set up a proper 'default' connection to master2 - -set default_master_connection = ''; - ---replace_result $SERVER_MYPORT_2 MYPORT_2 -eval change master to -master_port=$SERVER_MYPORT_2, -master_host='127.0.0.1', -master_user='root'; - -start slave; ---source include/wait_for_slave_to_start.inc - ---source wait_for_sql_thread_read_all.inc - -# See both connections in the same status output - -let $wait_for_all= 1; -let $show_statement= SHOW ALL SLAVES STATUS; -let $field= Slave_IO_State; -let $condition= = 'Waiting for master to send event'; ---source include/wait_show_condition.inc - ---echo # ---echo # Checking SHOW ALL SLAVES STATUS ---echo # ---let $all_slaves_status= 1 ---let $status_items= Connection_name, Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno, Slave_heartbeat_period ---let $slave_field_result_replace= /$SERVER_MYPORT_1/MYPORT_1/ /$SERVER_MYPORT_2/MYPORT_2/ ---source include/show_slave_status.inc ---let $all_slaves_status= ---echo # - -# Check that replication from two servers actually works - ---connection master1 - -insert into t1 (f1) values ('three'); ---save_master_pos - ---connect (master2,127.0.0.1,root,,,$SERVER_MYPORT_2) - ---disable_warnings -drop database if exists db2; ---enable_warnings -create database db2; -use db2; -create table t1 (pk int auto_increment primary key, f1 int) engine=InnoDB; -begin; -insert into t1 (f1) values (1),(2); - ---connection slave ---sync_with_master 0,'master1' - ---connection master2 ---save_master_pos - ---connection slave ---sync_with_master 0 ---sorted_result -select * from db1.t1; -select * from db2.t1; - ---connection master2 -commit; ---save_master_pos - ---connection slave ---sync_with_master 0 ---sorted_result -select * from db2.t1; - -# Flush and purge logs on one master, -# make sure slaves don't get confused - ---connection master1 -flush logs; ---source include/wait_for_binlog_checkpoint.inc ---save_master_pos ---connection slave ---sync_with_master 0, 'master1' - ---connection master1 -purge binary logs to 'master-bin.000002'; -let filesize=`select $binlog_start_pos+119`; ---replace_result $filesize filesize -show binary logs; -insert into t1 (f1) values ('four'); -create table db1.t3 (f1 int) engine=InnoDB; ---save_master_pos - ---connection slave ---sync_with_master 0,'master1' - ---source wait_for_sql_thread_read_all.inc - -let $wait_for_all= 1; -let $show_statement= SHOW ALL SLAVES STATUS; -let $field= Slave_IO_State; -let $condition= = 'Waiting for master to send event'; ---source include/wait_show_condition.inc - ---echo # ---echo # Checking SHOW ALL SLAVES STATUS ---echo # ---let $all_slaves_status= 1 ---let $status_items= Connection_name, Master_Port, Relay_Log_File, Slave_IO_Running, Slave_SQL_Running, Last_Errno, Last_SQL_Errno, Slave_heartbeat_period ---let $slave_field_result_replace= /$SERVER_MYPORT_1/MYPORT_1/ /$SERVER_MYPORT_2/MYPORT_2/ ---source include/show_slave_status.inc ---let $all_slaves_status= ---echo # - ---sorted_result -select * from db1.t1; - -# This should show relay log events for the default master -# (the one with the empty name) -let binlog_file=; -source include/show_relaylog_events.inc; -let binlog_file= mysqld-relay-bin.000002; -source include/show_relaylog_events.inc; - -# Make sure we don't lose control over replication connections -# after reconnecting to the slave - ---disconnect slave ---connect (slave,127.0.0.1,root,,,$SERVER_MYPORT_3) - -stop slave io_thread; -show status like 'Slave_running'; -set default_master_connection = 'master1'; -show status like 'Slave_running'; - -# Cleanup - -drop database db1; -drop database db2; - ---source reset_master_slave.inc ---disconnect slave - ---connection master1 -drop database db1; ---source reset_master_slave.inc ---disconnect master1 - ---connection master2 -drop database db2; ---source reset_master_slave.inc ---disconnect master2 - +--source extra/rpl_tests/multisource.inc diff --git a/mysql-test/suite/multi_source/relaylog_events.test b/mysql-test/suite/multi_source/relaylog_events.test index 18e92c32e8e27..7e5257af83724 100644 --- a/mysql-test/suite/multi_source/relaylog_events.test +++ b/mysql-test/suite/multi_source/relaylog_events.test @@ -44,10 +44,10 @@ drop table t1; --connection slave --sync_with_master 0,'master1' ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 diff --git a/mysql-test/suite/multi_source/reset_slave.test b/mysql-test/suite/multi_source/reset_slave.test index ebbc33e4c2232..4f4b9617345c7 100644 --- a/mysql-test/suite/multi_source/reset_slave.test +++ b/mysql-test/suite/multi_source/reset_slave.test @@ -61,12 +61,12 @@ show slave 'master1' status; # Cleanup drop table t1; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 drop table t1; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 diff --git a/mysql-test/suite/multi_source/simple.test b/mysql-test/suite/multi_source/simple.test index c990a04acb070..179fb9a49333c 100644 --- a/mysql-test/suite/multi_source/simple.test +++ b/mysql-test/suite/multi_source/simple.test @@ -75,12 +75,12 @@ stop all slaves; # clean up # ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 --connection master2 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master2 diff --git a/mysql-test/suite/multi_source/skip_counter.test b/mysql-test/suite/multi_source/skip_counter.test index 937261350a830..e53d0276a91ff 100644 --- a/mysql-test/suite/multi_source/skip_counter.test +++ b/mysql-test/suite/multi_source/skip_counter.test @@ -134,16 +134,16 @@ drop database db; --eval set global max_relay_log_size = $max_relay_log_size_saved --eval set global max_binlog_size = $max_binlog_size_saved ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 drop database db; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 --connection master2 drop database db; ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master2 diff --git a/mysql-test/suite/multi_source/status_vars.test b/mysql-test/suite/multi_source/status_vars.test index d1cfda75d0135..e76a403db333c 100644 --- a/mysql-test/suite/multi_source/status_vars.test +++ b/mysql-test/suite/multi_source/status_vars.test @@ -127,13 +127,13 @@ show status like 'Slave_open_temp_tables'; # Cleanup ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect slave --connection master1 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master1 --connection master2 ---source reset_master_slave.inc +--source include/reset_master_slave.inc --disconnect master2 diff --git a/mysql-test/suite/rpl/t/rpl_binlog_errors.test b/mysql-test/suite/rpl/t/rpl_binlog_errors.test index 72de3a1ef37f3..6a2cf20d75621 100644 --- a/mysql-test/suite/rpl/t/rpl_binlog_errors.test +++ b/mysql-test/suite/rpl/t/rpl_binlog_errors.test @@ -1,403 +1 @@ -# BUG#46166: MYSQL_BIN_LOG::new_file_impl is not propagating error -# when generating new name. -# -# WHY -# === -# -# We want to check whether error is reported or not when -# new_file_impl fails (this may happen when rotation is not -# possible because there is some problem finding an -# unique filename). -# -# HOW -# === -# -# Test cases are documented inline. - --- source include/have_innodb.inc --- source include/have_debug.inc --- source include/master-slave.inc - --- echo ####################################################################### --- echo ####################### PART 1: MASTER TESTS ########################## --- echo ####################################################################### - - -### ACTION: stopping slave as it is not needed for the first part of -### the test - --- connection slave --- source include/stop_slave.inc --- connection master - -call mtr.add_suppression("Can't generate a unique log-filename"); -call mtr.add_suppression("Writing one row to the row-based binary log failed.*"); -call mtr.add_suppression("Error writing file .*"); - -SET @old_debug= @@global.debug; - -### ACTION: create a large file (> 4096 bytes) that will be later used -### in LOAD DATA INFILE to check binlog errors in its vacinity --- let $load_file= $MYSQLTEST_VARDIR/tmp/bug_46166.data --- let $MYSQLD_DATADIR= `select @@datadir` --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SELECT repeat('x',8192) INTO OUTFILE '$load_file' - -### ACTION: create a small file (< 4096 bytes) that will be later used -### in LOAD DATA INFILE to check for absence of binlog errors -### when file loading this file does not force flushing and -### rotating the binary log --- let $load_file2= $MYSQLTEST_VARDIR/tmp/bug_46166-2.data --- let $MYSQLD_DATADIR= `select @@datadir` --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval SELECT repeat('x',10) INTO OUTFILE '$load_file2' - -RESET MASTER; - --- echo ###################### TEST #1 - -### ASSERTION: no problem flushing logs (should show two binlogs) -FLUSH LOGS; --- echo # assert: must show two binlogs --- source include/show_binary_logs.inc - --- echo ###################### TEST #2 - -### ASSERTION: check that FLUSH LOGS actually fails and reports -### failure back to the user if find_uniq_filename fails -### (should show just one binlog) - -RESET MASTER; -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; --- error ER_NO_UNIQUE_LOGFILE -FLUSH LOGS; --- echo # assert: must show one binlog --- source include/show_binary_logs.inc - -### ACTION: clean up and move to next test -SET GLOBAL debug_dbug=@old_debug; -RESET MASTER; - --- echo ###################### TEST #3 - -### ACTION: create some tables (t1, t2, t4) and insert some values in -### table t1 -CREATE TABLE t1 (a INT); -CREATE TABLE t2 (a VARCHAR(16384)) Engine=InnoDB; -CREATE TABLE t4 (a VARCHAR(16384)); -INSERT INTO t1 VALUES (1); -RESET MASTER; - -### ASSERTION: we force rotation of the binary log because it exceeds -### the max_binlog_size option (should show two binary -### logs) - --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 - -# shows two binary logs --- echo # assert: must show two binlog --- source include/show_binary_logs.inc - -# clean up the table and the binlog to be used in next part of test -SET GLOBAL debug_dbug=@old_debug; -DELETE FROM t2; -RESET MASTER; - --- echo ###################### TEST #4 - -### ASSERTION: load the big file into a transactional table and check -### that it reports error. The table will contain the -### changes performed despite the fact that it reported an -### error. - -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- error ER_NO_UNIQUE_LOGFILE --- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 - -# show table --- echo # assert: must show one entry -SELECT count(*) FROM t2; - -# clean up the table and the binlog to be used in next part of test -SET GLOBAL debug_dbug=@old_debug; -DELETE FROM t2; -RESET MASTER; - --- echo ###################### TEST #5 - -### ASSERTION: load the small file into a transactional table and -### check that it succeeds - -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval LOAD DATA INFILE '$load_file2' INTO TABLE t2 - -# show table --- echo # assert: must show one entry -SELECT count(*) FROM t2; - -# clean up the table and the binlog to be used in next part of test -SET GLOBAL debug_dbug=@old_debug; -DELETE FROM t2; -RESET MASTER; - --- echo ###################### TEST #6 - -### ASSERTION: check that even if one is using a transactional table -### and explicit transactions (no autocommit) if rotation -### fails we get the error. Transaction is not rolledback -### because rotation happens after the commit. - -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; -SET AUTOCOMMIT=0; -INSERT INTO t2 VALUES ('muse'); --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 -INSERT INTO t2 VALUES ('muse'); --- error ER_NO_UNIQUE_LOGFILE -COMMIT; - -### ACTION: Show the contents of the table after the test --- echo # assert: must show three entries -SELECT count(*) FROM t2; - -### ACTION: clean up and move to the next test -SET AUTOCOMMIT= 1; -SET GLOBAL debug_dbug=@old_debug; -DELETE FROM t2; -RESET MASTER; - --- echo ###################### TEST #7 - -### ASSERTION: check that on a non-transactional table, if rotation -### fails then an error is reported and an incident event -### is written to the current binary log. - -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; -SELECT count(*) FROM t4; --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- error ER_NO_UNIQUE_LOGFILE --- eval LOAD DATA INFILE '$load_file' INTO TABLE t4 - --- echo # assert: must show 1 entry -SELECT count(*) FROM t4; - --- echo ### check that the incident event is written to the current log -SET GLOBAL debug_dbug=@old_debug; --- let $binlog_limit= 4,1 --- source include/show_binlog_events.inc - -# clean up and move to next test -DELETE FROM t4; -RESET MASTER; - --- echo ###################### TEST #8 - -### ASSERTION: check that statements end up in error but they succeed -### on changing the data. - -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; --- echo # must show 0 entries -SELECT count(*) FROM t4; -SELECT count(*) FROM t2; - --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- error ER_NO_UNIQUE_LOGFILE --- eval LOAD DATA INFILE '$load_file' INTO TABLE t4 --- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --- error ER_NO_UNIQUE_LOGFILE --- eval LOAD DATA INFILE '$load_file' INTO TABLE t2 --- error ER_NO_UNIQUE_LOGFILE -INSERT INTO t2 VALUES ('aaa'), ('bbb'), ('ccc'); - --- echo # INFO: Count(*) Before Offending DELETEs --- echo # assert: must show 1 entry -SELECT count(*) FROM t4; --- echo # assert: must show 4 entries -SELECT count(*) FROM t2; - --- error ER_NO_UNIQUE_LOGFILE -DELETE FROM t4; --- error ER_NO_UNIQUE_LOGFILE -DELETE FROM t2; - --- echo # INFO: Count(*) After Offending DELETEs --- echo # assert: must show zero entries -SELECT count(*) FROM t4; -SELECT count(*) FROM t2; - -# remove fault injection -SET GLOBAL debug_dbug=@old_debug; - --- echo ###################### TEST #9 - -### ASSERTION: check that if we disable binlogging, then statements -### succeed. -SET GLOBAL debug_dbug="+d,error_unique_log_filename"; -SET SQL_LOG_BIN=0; -INSERT INTO t2 VALUES ('aaa'), ('bbb'), ('ccc'), ('ddd'); -INSERT INTO t4 VALUES ('eee'), ('fff'), ('ggg'), ('hhh'); --- echo # assert: must show four entries -SELECT count(*) FROM t2; -SELECT count(*) FROM t4; -DELETE FROM t2; -DELETE FROM t4; --- echo # assert: must show zero entries -SELECT count(*) FROM t2; -SELECT count(*) FROM t4; -SET SQL_LOG_BIN=1; -SET GLOBAL debug_dbug=@old_debug; - --- echo ###################### TEST #10 - -### ASSERTION: check that error is reported if there is a failure -### while registering the index file and the binary log -### file or failure to write the rotate event. - -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); -call mtr.add_suppression("Could not open .*"); - -RESET MASTER; -SHOW WARNINGS; - -# +d,fault_injection_registering_index => injects fault on MYSQL_BIN_LOG::open -SET GLOBAL debug_dbug="+d,fault_injection_registering_index"; --- replace_regex /\.[\\\/]master/master/ --- error ER_CANT_OPEN_FILE -FLUSH LOGS; -SET GLOBAL debug_dbug="-d,fault_injection_registering_index"; - --- error ER_NO_BINARY_LOGGING -SHOW BINARY LOGS; - -# issue some statements and check that they don't fail -CREATE TABLE t5 (a INT); -INSERT INTO t4 VALUES ('bbbbb'); -INSERT INTO t2 VALUES ('aaaaa'); -DELETE FROM t4; -DELETE FROM t2; -DROP TABLE t5; - --- echo ###################### TEST #11 - -### ASSERTION: check that error is reported if there is a failure -### while opening the index file and the binary log file or -### failure to write the rotate event. - -# restart the server so that we have binlog again ---let $rpl_server_number= 1 ---source include/rpl_restart_server.inc - -# +d,fault_injection_openning_index => injects fault on MYSQL_BIN_LOG::open_index_file -SET GLOBAL debug_dbug="+d,fault_injection_openning_index"; --- replace_regex /\.[\\\/]master/master/ --- error ER_CANT_OPEN_FILE -FLUSH LOGS; -SET GLOBAL debug_dbug="-d,fault_injection_openning_index"; - --- error ER_FLUSH_MASTER_BINLOG_CLOSED -RESET MASTER; - -# issue some statements and check that they don't fail -CREATE TABLE t5 (a INT); -INSERT INTO t4 VALUES ('bbbbb'); -INSERT INTO t2 VALUES ('aaaaa'); -DELETE FROM t4; -DELETE FROM t2; -DROP TABLE t5; - -# restart the server so that we have binlog again ---let $rpl_server_number= 1 ---source include/rpl_restart_server.inc - --- echo ###################### TEST #12 - -### ASSERTION: check that error is reported if there is a failure -### while writing the rotate event when creating a new log -### file. - -# +d,fault_injection_new_file_rotate_event => injects fault on MYSQL_BIN_LOG::MYSQL_BIN_LOG::new_file_impl -SET GLOBAL debug_dbug="+d,fault_injection_new_file_rotate_event"; --- error ER_ERROR_ON_WRITE -FLUSH LOGS; -SET GLOBAL debug_dbug="-d,fault_injection_new_file_rotate_event"; - --- error ER_FLUSH_MASTER_BINLOG_CLOSED -RESET MASTER; - -# issue some statements and check that they don't fail -CREATE TABLE t5 (a INT); -INSERT INTO t4 VALUES ('bbbbb'); -INSERT INTO t2 VALUES ('aaaaa'); -DELETE FROM t4; -DELETE FROM t2; -DROP TABLE t5; - -# restart the server so that we have binlog again ---let $rpl_server_number= 1 ---source include/rpl_restart_server.inc - -## clean up -DROP TABLE t1, t2, t4; -RESET MASTER; - -# restart slave again --- connection slave --- source include/start_slave.inc --- connection master - --- echo ####################################################################### --- echo ####################### PART 2: SLAVE TESTS ########################### --- echo ####################################################################### - -### setup ---source include/rpl_reset.inc --- connection slave - -# slave suppressions - -call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); -call mtr.add_suppression("Error writing file .*"); -call mtr.add_suppression("Could not open .*"); -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); -call mtr.add_suppression("Can't generate a unique log-filename .*"); --- echo ###################### TEST #13 - -#### ASSERTION: check against unique log filename error --- let $io_thd_injection_fault_flag= error_unique_log_filename --- let $slave_io_errno= 1595 --- let $show_slave_io_error= 1 --- source include/io_thd_fault_injection.inc - --- echo ###################### TEST #14 - -#### ASSERTION: check against rotate failing --- let $io_thd_injection_fault_flag= fault_injection_new_file_rotate_event --- let $slave_io_errno= 1595 --- let $show_slave_io_error= 1 --- source include/io_thd_fault_injection.inc - --- echo ###################### TEST #15 - -#### ASSERTION: check against relay log open failure --- let $io_thd_injection_fault_flag= fault_injection_registering_index --- let $slave_io_errno= 1595 --- let $show_slave_io_error= 1 --- source include/io_thd_fault_injection.inc - --- echo ###################### TEST #16 - -#### ASSERTION: check against relay log index open failure --- let $io_thd_injection_fault_flag= fault_injection_openning_index --- let $slave_io_errno= 1595 --- let $show_slave_io_error= 1 --- source include/io_thd_fault_injection.inc - -### clean up --- source include/stop_slave_sql.inc -RESET SLAVE; -RESET MASTER; ---let $rpl_only_running_threads= 1 ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_binlog_errors.inc diff --git a/mysql-test/suite/rpl/t/rpl_cant_read_event_incident.test b/mysql-test/suite/rpl/t/rpl_cant_read_event_incident.test index a97801f9ab0d0..6d222cba1151c 100644 --- a/mysql-test/suite/rpl/t/rpl_cant_read_event_incident.test +++ b/mysql-test/suite/rpl/t/rpl_cant_read_event_incident.test @@ -1,77 +1 @@ -# -# Bug#11747416 : 32228 A disk full makes binary log corrupt. -# -# -# The test demonstrates reading from binlog error propagation to slave -# and reporting there. -# Conditions for the bug include a crash at time of the last event to -# the binlog was written partly. With the fixes the event is not sent out -# any longer, but rather the dump thread sends out a sound error message. -# -# Crash is not simulated. A binlog with partly written event in its end is installed -# and replication is started from it. -# - ---source include/master-slave.inc ---source include/have_binlog_format_mixed.inc - ---connection slave -# Make sure the slave is stopped while we are messing with master. -# Otherwise we get occasional failures as the slave manages to re-connect -# to the newly started master and we get extra events applied, causing -# conflicts. ---source include/stop_slave.inc - ---connection master -call mtr.add_suppression("Error in Log_event::read_log_event()"); ---let $datadir= `SELECT @@datadir` - ---let $rpl_server_number= 1 ---source include/rpl_stop_server.inc - ---remove_file $datadir/master-bin.000001 ---copy_file $MYSQL_TEST_DIR/std_data/bug11747416_32228_binlog.000001 $datadir/master-bin.000001 - ---let $rpl_server_number= 1 ---source include/rpl_start_server.inc - ---source include/wait_until_connected_again.inc - -# evidence of the partial binlog ---error ER_ERROR_WHEN_EXECUTING_COMMAND -show binlog events; - ---connection slave -call mtr.add_suppression("Slave I/O: Got fatal error 1236 from master when reading data from binary log"); -reset slave; -start slave; - -# ER_MASTER_FATAL_ERROR_READING_BINLOG 1236 ---let $slave_param=Last_IO_Errno ---let $slave_param_value=1236 ---source include/wait_for_slave_param.inc - ---let $slave_field_result_replace= / at [0-9]*/ at XXX/ ---let $status_items= Last_IO_Errno, Last_IO_Error ---source include/show_slave_status.inc - -# -# Cleanup -# - ---connection master -reset master; - ---connection slave -stop slave; -reset slave; -# Table was created from binlog, it may not be created if SQL thread is running -# slowly and IO thread reaches incident before SQL thread applies it. ---disable_warnings -drop table if exists t; ---enable_warnings -reset master; - ---echo End of the tests ---let $rpl_only_running_threads= 1 ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_cant_read_event_incident.inc diff --git a/mysql-test/suite/rpl/t/rpl_checksum.test b/mysql-test/suite/rpl/t/rpl_checksum.test index 50b6e712b90cc..8e006b1b6a0c5 100644 --- a/mysql-test/suite/rpl/t/rpl_checksum.test +++ b/mysql-test/suite/rpl/t/rpl_checksum.test @@ -1,327 +1 @@ -# WL2540 replication events checksum -# Testing configuration parameters - ---source include/master-slave.inc ---source include/have_debug.inc ---source include/have_binlog_format_mixed.inc - -call mtr.add_suppression('Slave can not handle replication events with the checksum that master is configured to log'); -call mtr.add_suppression('Replication event checksum verification failed'); -# due to C failure simulation -call mtr.add_suppression('Relay log write failure: could not queue event from master'); -call mtr.add_suppression('Master is configured to log replication events with checksum, but will not send such events to slaves that cannot process them'); - -# A. read/write access to the global vars: -# binlog_checksum master_verify_checksum slave_sql_verify_checksum - -connection master; - -set @master_save_binlog_checksum= @@global.binlog_checksum; -set @save_master_verify_checksum = @@global.master_verify_checksum; - -select @@global.binlog_checksum as 'must be CRC32 because of the command line option'; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -select @@session.binlog_checksum as 'no session var'; - -select @@global.master_verify_checksum as 'must be zero because of default'; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -select @@session.master_verify_checksum as 'no session var'; - -connection slave; - -set @slave_save_binlog_checksum= @@global.binlog_checksum; -set @save_slave_sql_verify_checksum = @@global.slave_sql_verify_checksum; - -select @@global.slave_sql_verify_checksum as 'must be one because of default'; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -select @@session.slave_sql_verify_checksum as 'no session var'; - -connection master; - -source include/show_binary_logs.inc; -set @@global.binlog_checksum = NONE; -select @@global.binlog_checksum; ---echo *** must be rotations seen *** -source include/show_binary_logs.inc; - -set @@global.binlog_checksum = default; -select @@global.binlog_checksum; - -# testing lack of side-effects in non-effective update of binlog_checksum: -set @@global.binlog_checksum = CRC32; -select @@global.binlog_checksum; -set @@global.binlog_checksum = CRC32; - -set @@global.master_verify_checksum = 0; -set @@global.master_verify_checksum = default; - ---error ER_WRONG_VALUE_FOR_VAR -set @@global.binlog_checksum = ADLER32; ---error ER_WRONG_VALUE_FOR_VAR -set @@global.master_verify_checksum = 2; # the var is of bool type - -connection slave; - -set @@global.slave_sql_verify_checksum = 0; -set @@global.slave_sql_verify_checksum = default; ---error ER_WRONG_VALUE_FOR_VAR -set @@global.slave_sql_verify_checksum = 2; # the var is of bool type - -# -# B. Old Slave to New master conditions -# -# while master does not send a checksum-ed binlog the Old Slave can -# work with the New Master - -connection master; - -set @@global.binlog_checksum = NONE; -create table t1 (a int); - -# testing that binlog rotation preserves opt_binlog_checksum value -flush logs; -flush logs; -flush logs; - -sync_slave_with_master; -#connection slave; -# checking that rotation on the slave side leaves slave stable -flush logs; -flush logs; -flush logs; -select count(*) as zero from t1; - -source include/stop_slave.inc; - -connection master; -set @@global.binlog_checksum = CRC32; --- source include/wait_for_binlog_checkpoint.inc -insert into t1 values (1) /* will not be applied on slave due to simulation */; - -# instruction to the dump thread - -connection slave; -set @@global.debug_dbug='d,simulate_slave_unaware_checksum'; -start slave; ---let $slave_io_errno= 1236 ---let $show_slave_io_error= 1 -source include/wait_for_slave_io_error.inc; - -select count(*) as zero from t1; - -###connection master; -set @@global.debug_dbug=''; - -connection slave; -source include/start_slave.inc; - -# -# C. checksum failure simulations -# - -# C1. Failure by a client thread -connection master; -set @@global.master_verify_checksum = 1; -set @@session.debug_dbug='d,simulate_checksum_test_failure'; ---error ER_ERROR_WHEN_EXECUTING_COMMAND -show binlog events; -set @@session.debug_dbug=''; -set @@global.master_verify_checksum = default; - -#connection master; -sync_slave_with_master; - -connection slave; -source include/stop_slave.inc; - -connection master; -create table t2 (a int); -let $pos_master= query_get_value(SHOW MASTER STATUS, Position, 1); - -connection slave; - -# C2. Failure by IO thread -# instruction to io thread -set @@global.debug_dbug='d,simulate_checksum_test_failure'; -start slave io_thread; -# When the checksum error is detected, the slave sets error code 1913 -# (ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE) in queue_event(), then immediately -# sets error 1595 (ER_SLAVE_RELAY_LOG_WRITE_FAILURE) in handle_slave_io(). -# So we usually get 1595, but it is occasionally possible to get 1913. ---let $slave_io_errno= 1595,1913 ---let $show_slave_io_error= 0 -source include/wait_for_slave_io_error.inc; -set @@global.debug_dbug=''; - -# to make IO thread re-read it again w/o the failure -start slave io_thread; -let $slave_param= Read_Master_Log_Pos; -let $slave_param_value= $pos_master; -source include/wait_for_slave_param.inc; - -# C3. Failure by SQL thread -# instruction to sql thread; -set @@global.slave_sql_verify_checksum = 1; - -set @@global.debug_dbug='d,simulate_checksum_test_failure'; - -start slave sql_thread; ---let $slave_sql_errno= 1593 ---let $show_slave_sql_error= 1 -source include/wait_for_slave_sql_error.inc; - -# resuming SQL thread to parse out the event w/o the failure - -set @@global.debug_dbug=''; -source include/start_slave.inc; - -connection master; -sync_slave_with_master; - -#connection slave; -select count(*) as 'must be zero' from t2; - -# -# D. Reset slave, Change-Master, Binlog & Relay-log rotations with -# random value on binlog_checksum on both master and slave -# -connection slave; -stop slave; -reset slave; - -# randomize slave server's own checksum policy -set @@global.binlog_checksum= IF(floor((rand()*1000)%2), "CRC32", "NONE"); -flush logs; - -connection master; -set @@global.binlog_checksum= CRC32; -reset master; -flush logs; -create table t3 (a int, b char(5)); - -connection slave; -source include/start_slave.inc; - -connection master; -sync_slave_with_master; - -#connection slave; -select count(*) as 'must be zero' from t3; -source include/stop_slave.inc; ---replace_result $MASTER_MYPORT MASTER_PORT -eval change master to master_host='127.0.0.1',master_port=$MASTER_MYPORT, master_user='root'; - -connection master; -flush logs; -reset master; -insert into t3 value (1, @@global.binlog_checksum); - -connection slave; -source include/start_slave.inc; -flush logs; - -connection master; -sync_slave_with_master; - -#connection slave; -select count(*) as 'must be one' from t3; - -connection master; -set @@global.binlog_checksum= IF(floor((rand()*1000)%2), "CRC32", "NONE"); -insert into t3 value (1, @@global.binlog_checksum); -sync_slave_with_master; - -#connection slave; - -#clean-up - -connection master; -drop table t1, t2, t3; -set @@global.binlog_checksum = @master_save_binlog_checksum; -set @@global.master_verify_checksum = @save_master_verify_checksum; - -# -# BUG#58564: flush_read_lock fails in mysql-trunk-bugfixing after merging with WL#2540 -# -# Sanity check that verifies that no assertions are triggered because -# of old FD events (generated by versions prior to server released with -# checksums feature) -# -# There is no need for query log, if something wrong this should trigger -# an assertion - ---disable_query_log - -BINLOG ' -MfmqTA8BAAAAZwAAAGsAAAABAAQANS41LjctbTMtZGVidWctbG9nAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAx+apMEzgNAAgAEgAEBAQEEgAAVAAEGggAAAAICAgCAA== -'; - ---enable_query_log - -#connection slave; -sync_slave_with_master; - - ---echo *** Bug#59123 / MDEV-5799: INCIDENT_EVENT checksum written to error log as garbage characters *** - ---connection master - ---source include/wait_for_binlog_checkpoint.inc -CREATE TABLE t4 (a INT PRIMARY KEY); -INSERT INTO t4 VALUES (1); - -SET sql_log_bin=0; -CALL mtr.add_suppression("\\[ERROR\\] Can't generate a unique log-filename"); -SET sql_log_bin=1; -SET @old_dbug= @@GLOBAL.debug_dbug; -SET debug_dbug= '+d,binlog_inject_new_name_error'; ---error ER_NO_UNIQUE_LOGFILE -FLUSH LOGS; -SET debug_dbug= @old_dbug; - -INSERT INTO t4 VALUES (2); - ---connection slave ---let $slave_sql_errno= 1590 ---source include/wait_for_slave_sql_error.inc - -# Search the error log for the error message. -# The bug was that 4 garbage bytes were output in the middle of the error -# message; by searching for a pattern that spans that location, we can -# catch the error. -let $log_error_= `SELECT @@GLOBAL.log_error`; -if(!$log_error_) -{ - # MySQL Server on windows is started with --console and thus - # does not know the location of its .err log, use default location - let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.2.err; -} ---let SEARCH_FILE= $log_error_ ---let SEARCH_RANGE=-50000 ---let SEARCH_PATTERN= Slave SQL: The incident LOST_EVENTS occurred on the master\. Message: error writing to the binary log, Internal MariaDB error code: 1590 ---source include/search_pattern_in_file.inc - -SELECT * FROM t4 ORDER BY a; -STOP SLAVE IO_THREAD; -SET sql_slave_skip_counter= 1; ---source include/start_slave.inc - ---connection master ---save_master_pos - ---connection slave ---sync_with_master -SELECT * FROM t4 ORDER BY a; - - ---connection slave -set @@global.binlog_checksum = @slave_save_binlog_checksum; -set @@global.slave_sql_verify_checksum = @save_slave_sql_verify_checksum; - ---echo End of tests - ---connection master -DROP TABLE t4; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_checksum.inc diff --git a/mysql-test/suite/rpl/t/rpl_checksum_cache.test b/mysql-test/suite/rpl/t/rpl_checksum_cache.test index 5667d599afff5..56c3e1e1cb5f1 100644 --- a/mysql-test/suite/rpl/t/rpl_checksum_cache.test +++ b/mysql-test/suite/rpl/t/rpl_checksum_cache.test @@ -1,255 +1 @@ --- source include/have_innodb.inc --- source include/master-slave.inc - ---disable_warnings -call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. Statement: insert into t2 set data=repeat.*'a', @act_size.*"); -call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. Statement: insert into t1 values.* NAME_CONST.*'n',.*, @data .*"); ---enable_warnings - -connection master; -set @save_binlog_cache_size = @@global.binlog_cache_size; -set @save_binlog_checksum = @@global.binlog_checksum; -set @save_master_verify_checksum = @@global.master_verify_checksum; -set @@global.binlog_cache_size = 4096; -set @@global.binlog_checksum = CRC32; -set @@global.master_verify_checksum = 1; - -# restart slave to force the dump thread to verify events (on master side) -connection slave; -source include/stop_slave.inc; -source include/start_slave.inc; - -connection master; - -# -# Testing a critical part of checksum handling dealing with transaction cache. -# The cache's buffer size is set to be less than the transaction's footprint -# in binlog. -# -# To verify combined buffer-by-buffer read out of the file and fixing crc per event -# there are the following parts: -# -# 1. the event size is much less than the cache's buffer -# 2. the event size is bigger than the cache's buffer -# 3. the event size if approximately the same as the cache's buffer -# 4. all in above - -# -# 1. the event size is much less than the cache's buffer -# - -flush status; -show status like "binlog_cache_use"; -show status like "binlog_cache_disk_use"; ---disable_warnings -drop table if exists t1; ---enable_warnings - -# -# parameter to ensure the test slightly varies binlog content -# between different invocations -# -let $deviation_size=32; -eval create table t1 (a int PRIMARY KEY, b CHAR($deviation_size)) engine=innodb; - -# Now we are going to create transaction which is long enough so its -# transaction binlog will be flushed to disk... - -delimiter |; -create procedure test.p_init (n int, size int) -begin - while n > 0 do - select round(RAND() * size) into @act_size; - set @data = repeat('a', @act_size); - insert into t1 values(n, @data ); - set n= n-1; - end while; -end| - -delimiter ;| - -let $1 = 4000; # PB2 can run it slow to time out on following sync_slave_with_master:s - -begin; ---disable_warnings -# todo: check if it is really so. -#+Note 1592 Unsafe statement binlogged in statement format since BINLOG_FORMAT = STATEMENT. Reason for unsafeness: Statement uses a system function whose value may differ on slave. -eval call test.p_init($1, $deviation_size); ---enable_warnings -commit; - -show status like "binlog_cache_use"; ---echo *** binlog_cache_disk_use must be non-zero *** -show status like "binlog_cache_disk_use"; - -sync_slave_with_master; - -let $diff_tables=master:test.t1, slave:test.t1; -source include/diff_tables.inc; - -# undoing changes with verifying the above once again -connection master; - -begin; -delete from t1; -commit; - -sync_slave_with_master; - - -# -# 2. the event size is bigger than the cache's buffer -# -connection master; - -flush status; -let $t2_data_size= `select 3 * @@global.binlog_cache_size`; -let $t2_aver_size= `select 2 * @@global.binlog_cache_size`; -let $t2_max_rand= `select 1 * @@global.binlog_cache_size`; - -eval create table t2(a int auto_increment primary key, data VARCHAR($t2_data_size)) ENGINE=Innodb; -let $1=100; ---disable_query_log -begin; -while ($1) -{ - eval select round($t2_aver_size + RAND() * $t2_max_rand) into @act_size; - set @data = repeat('a', @act_size); - insert into t2 set data = @data; - dec $1; -} -commit; ---enable_query_log -show status like "binlog_cache_use"; ---echo *** binlog_cache_disk_use must be non-zero *** -show status like "binlog_cache_disk_use"; - -sync_slave_with_master; - -let $diff_tables=master:test.t2, slave:test.t2; -source include/diff_tables.inc; - -# undoing changes with verifying the above once again -connection master; - -begin; -delete from t2; -commit; - -sync_slave_with_master; - -# -# 3. the event size if approximately the same as the cache's buffer -# - -connection master; - -flush status; -let $t3_data_size= `select 2 * @@global.binlog_cache_size`; -let $t3_aver_size= `select (9 * @@global.binlog_cache_size) / 10`; -let $t3_max_rand= `select (2 * @@global.binlog_cache_size) / 10`; - -eval create table t3(a int auto_increment primary key, data VARCHAR($t3_data_size)) engine=innodb; - -let $1= 300; ---disable_query_log -begin; -while ($1) -{ - eval select round($t3_aver_size + RAND() * $t3_max_rand) into @act_size; - insert into t3 set data= repeat('a', @act_size); - dec $1; -} -commit; ---enable_query_log -show status like "binlog_cache_use"; ---echo *** binlog_cache_disk_use must be non-zero *** -show status like "binlog_cache_disk_use"; - -sync_slave_with_master; - -let $diff_tables=master:test.t3, slave:test.t3; -source include/diff_tables.inc; - -# undoing changes with verifying the above once again -connection master; - -begin; -delete from t3; -commit; - -sync_slave_with_master; - - -# -# 4. all in above -# - -connection master; -flush status; - -delimiter |; -eval create procedure test.p1 (n int) -begin - while n > 0 do - case (select (round(rand()*100) % 3) + 1) - when 1 then - select round(RAND() * $deviation_size) into @act_size; - set @data = repeat('a', @act_size); - insert into t1 values(n, @data); - when 2 then - begin - select round($t2_aver_size + RAND() * $t2_max_rand) into @act_size; - insert into t2 set data=repeat('a', @act_size); - end; - when 3 then - begin - select round($t3_aver_size + RAND() * $t3_max_rand) into @act_size; - insert into t3 set data= repeat('a', @act_size); - end; - end case; - set n= n-1; - end while; -end| -delimiter ;| - -let $1= 1000; -set autocommit= 0; -begin; ---disable_warnings -eval call test.p1($1); ---enable_warnings -commit; - -show status like "binlog_cache_use"; ---echo *** binlog_cache_disk_use must be non-zero *** -show status like "binlog_cache_disk_use"; - -sync_slave_with_master; - -let $diff_tables=master:test.t1, slave:test.t1; -source include/diff_tables.inc; - -let $diff_tables=master:test.t2, slave:test.t2; -source include/diff_tables.inc; - -let $diff_tables=master:test.t3, slave:test.t3; -source include/diff_tables.inc; - - -connection master; - -begin; -delete from t1; -delete from t2; -delete from t3; -commit; - -drop table t1, t2, t3; -set @@global.binlog_cache_size = @save_binlog_cache_size; -set @@global.binlog_checksum = @save_binlog_checksum; -set @@global.master_verify_checksum = @save_master_verify_checksum; -drop procedure test.p_init; -drop procedure test.p1; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_checksum_cache.inc diff --git a/mysql-test/suite/rpl/t/rpl_corruption.test b/mysql-test/suite/rpl/t/rpl_corruption.test index da87b133cb5bb..310b0cef8e8f3 100644 --- a/mysql-test/suite/rpl/t/rpl_corruption.test +++ b/mysql-test/suite/rpl/t/rpl_corruption.test @@ -1,170 +1 @@ -############################################################ -# Purpose: WL#5064 Testing with corrupted events. -# The test emulates the corruption at the vary stages -# of replication: -# - in binlog file -# - in network -# - in relay log -############################################################ - -# -# The tests intensively utilize @@global.debug. Note, -# Bug#11765758 - 58754, -# @@global.debug is read by the slave threads through dbug-interface. -# Hence, before a client thread set @@global.debug we have to ensure that: -# (a) the slave threads are stopped, or (b) the slave threads are in -# sync and waiting. - ---source include/have_debug.inc ---source include/master-slave.inc - -# Block legal errors for MTR -call mtr.add_suppression('Found invalid event in binary log'); -call mtr.add_suppression('Slave I/O: Relay log write failure: could not queue event from master'); -call mtr.add_suppression('event read from binlog did not pass crc check'); -call mtr.add_suppression('Replication event checksum verification failed'); -call mtr.add_suppression('Event crc check failed! Most likely there is event corruption'); -call mtr.add_suppression('Slave SQL: Error initializing relay log position: I/O error reading event at position .*, error.* 1593'); - -SET @old_master_verify_checksum = @@master_verify_checksum; - -# Creating test table/data and set corruption position for testing ---echo # 1. Creating test table/data and set corruption position for testing ---connection master ---echo * insert/update/delete rows in table t1 * -# Corruption algorithm modifies only the first event and -# then will be reset. To avoid checking always the first event -# from binlog (usually it is FD) we randomly execute different -# statements and set position for corruption inside events. - -CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY, b VARCHAR(10), c VARCHAR(100)); ---disable_query_log -let $i=`SELECT 3+CEILING(10*RAND())`; -let $j=1; -let $pos=0; -while ($i) { - eval INSERT INTO t1 VALUES ($j, 'a', NULL); - if (`SELECT RAND() > 0.7`) - { - eval UPDATE t1 SET c = REPEAT('a', 20) WHERE a = $j; - } - if (`SELECT RAND() > 0.8`) - { - eval DELETE FROM t1 WHERE a = $j; - } - if (!$pos) { - let $pos= query_get_value(SHOW MASTER STATUS, Position, 1); - --sync_slave_with_master - --source include/stop_slave.inc - --disable_query_log - --connection master - } - dec $i; - inc $j; -} ---enable_query_log - - -# Emulate corruption in binlog file when SHOW BINLOG EVENTS is executing ---echo # 2. Corruption in master binlog and SHOW BINLOG EVENTS -SET GLOBAL debug_dbug="+d,corrupt_read_log_event_char"; ---echo SHOW BINLOG EVENTS; ---disable_query_log -send_eval SHOW BINLOG EVENTS FROM $pos; ---enable_query_log ---error ER_ERROR_WHEN_EXECUTING_COMMAND -reap; - -SET GLOBAL debug_dbug="-d,corrupt_read_log_event_char"; - -# Emulate corruption on master with crc checking on master ---echo # 3. Master read a corrupted event from binlog and send the error to slave - -# We have a rare but nasty potential race here: if the dump thread on -# the master for the _old_ slave connection has not yet discovered -# that the slave has disconnected, we will inject the corrupt event on -# the wrong connection, and the test will fail -# (+d,corrupt_read_log_event2 corrupts only one event). -# So kill any lingering dump thread (we need to kill; otherwise dump thread -# could manage to send all events down the socket before seeing it close, and -# hang forever waiting for new binlog events to be created). -let $id= `select id from information_schema.processlist where command = "Binlog Dump"`; -if ($id) -{ - --disable_query_log - --error 0,1094 - eval kill $id; - --enable_query_log -} -let $wait_condition= - SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE command = 'Binlog Dump'; ---source include/wait_condition.inc - -SET GLOBAL debug_dbug="+d,corrupt_read_log_event2_set"; ---connection slave -START SLAVE IO_THREAD; -let $slave_io_errno= 1236; ---let $slave_timeout= 10 ---source include/wait_for_slave_io_error.inc ---connection master -SET GLOBAL debug_dbug="-d,corrupt_read_log_event2_set"; - -# Emulate corruption on master without crc checking on master ---echo # 4. Master read a corrupted event from binlog and send it to slave ---connection master -SET GLOBAL master_verify_checksum=0; -SET GLOBAL debug_dbug="+d,corrupt_read_log_event2_set"; ---connection slave -START SLAVE IO_THREAD; -# When the checksum error is detected, the slave sets error code 1913 -# (ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE) in queue_event(), then immediately -# sets error 1595 (ER_SLAVE_RELAY_LOG_WRITE_FAILURE) in handle_slave_io(). -# So we usually get 1595, but it is occasionally possible to get 1913. -let $slave_io_errno= 1595,1913; ---source include/wait_for_slave_io_error.inc ---connection master -SET GLOBAL debug_dbug="-d,corrupt_read_log_event2_set"; -SET GLOBAL debug_dbug= ""; -SET GLOBAL master_verify_checksum=1; - -# Emulate corruption in network ---echo # 5. Slave. Corruption in network ---connection slave -SET GLOBAL debug_dbug="+d,corrupt_queue_event"; -START SLAVE IO_THREAD; -let $slave_io_errno= 1595,1913; ---source include/wait_for_slave_io_error.inc -SET GLOBAL debug_dbug="-d,corrupt_queue_event"; - -# Emulate corruption in relay log ---echo # 6. Slave. Corruption in relay log - -SET GLOBAL debug_dbug="+d,corrupt_read_log_event_char"; - -START SLAVE SQL_THREAD; -let $slave_sql_errno= 1593; ---source include/wait_for_slave_sql_error.inc - -SET GLOBAL debug_dbug="-d,corrupt_read_log_event_char"; -SET GLOBAL debug_dbug= ""; - -# Start normal replication and compare same table on master -# and slave ---echo # 7. Seek diff for tables on master and slave ---connection slave ---source include/start_slave.inc ---connection master ---sync_slave_with_master -let $diff_tables= master:test.t1, slave:test.t1; ---source include/diff_tables.inc - -# Clean up ---echo # 8. Clean up ---connection master -SET GLOBAL debug_dbug= ""; -SET GLOBAL master_verify_checksum = @old_master_verify_checksum; -DROP TABLE t1; ---sync_slave_with_master -SET GLOBAL debug_dbug= ""; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_corruption.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_basic.test b/mysql-test/suite/rpl/t/rpl_gtid_basic.test index 772194e55f4fe..70be6fbff1fd2 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_basic.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_basic.test @@ -1,562 +1 @@ ---source include/have_innodb.inc ---let $rpl_topology=1->2->3->4 ---source include/rpl_init.inc - -# Set up a 4-deep replication topology, then test various fail-overs -# using GTID. -# -# A -> B -> C -> D - -connection server_1; ---source include/wait_for_binlog_checkpoint.inc ---let $binlog_file = query_get_value(SHOW MASTER STATUS,File,1) ---let $binlog_pos = query_get_value(SHOW MASTER STATUS,Position,1) ---echo *** GTID position should be empty here *** ---replace_result $binlog_file $binlog_pos -eval SELECT BINLOG_GTID_POS('$binlog_file',$binlog_pos); - -CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=MyISAM; -CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; -INSERT INTO t1 VALUES (1, "m1"); -INSERT INTO t1 VALUES (2, "m2"), (3, "m3"), (4, "m4"); -INSERT INTO t2 VALUES (1, "i1"); -BEGIN; -INSERT INTO t2 VALUES (2, "i2"), (3, "i3"); -INSERT INTO t2 VALUES (4, "i4"); -COMMIT; -save_master_pos; -source include/wait_for_binlog_checkpoint.inc; ---let $binlog_file = query_get_value(SHOW MASTER STATUS,File,1) ---let $binlog_pos = query_get_value(SHOW MASTER STATUS,Position,1) ---let $gtid_pos_server_1 = `SELECT @@gtid_binlog_pos` ---echo *** GTID position should be non-empty here *** ---replace_result $binlog_file $binlog_pos $gtid_pos_server_1 -eval SELECT BINLOG_GTID_POS('$binlog_file',$binlog_pos); - -connection server_2; -sync_with_master; -source include/wait_for_binlog_checkpoint.inc; ---let $binlog_file = query_get_value(SHOW MASTER STATUS,File,1) ---let $binlog_pos = query_get_value(SHOW MASTER STATUS,Position,1) ---echo *** GTID position should be the same as on server_1 *** ---replace_result $binlog_file $binlog_pos $gtid_pos_server_1 -eval SELECT BINLOG_GTID_POS('$binlog_file',$binlog_pos); -SELECT * FROM t1 ORDER BY a; -SELECT * FROM t2 ORDER BY a; -save_master_pos; - -connection server_3; -sync_with_master; -SELECT * FROM t1 ORDER BY a; -SELECT * FROM t2 ORDER BY a; -save_master_pos; - -connection server_4; -sync_with_master; -SELECT * FROM t1 ORDER BY a; -SELECT * FROM t2 ORDER BY a; - - ---echo *** Now take out D, let it fall behind a bit, and then test re-attaching it to A *** -connection server_4; ---source include/stop_slave.inc - -connection server_1; -INSERT INTO t1 VALUES (5, "m1a"); -INSERT INTO t2 VALUES (5, "i1a"); -save_master_pos; - -connection server_4; ---replace_result $MASTER_MYPORT MASTER_PORT -eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT, - MASTER_USE_GTID=CURRENT_POS; ---source include/start_slave.inc -sync_with_master; -SELECT * FROM t1 ORDER BY a; -SELECT * FROM t2 ORDER BY a; - ---echo *** Now move B to D (C is still replicating from B) *** -connection server_2; ---source include/stop_slave.inc ---replace_result $SERVER_MYPORT_4 SERVER_MYPORT_4 -eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SERVER_MYPORT_4, - MASTER_USE_GTID=CURRENT_POS; ---source include/start_slave.inc - -connection server_4; -UPDATE t2 SET b="j1a" WHERE a=5; -save_master_pos; - -connection server_2; -sync_with_master; -SELECT * FROM t1 ORDER BY a; -SELECT * FROM t2 ORDER BY a; - ---echo *** Now move C to D, after letting it fall a little behind *** -connection server_3; ---source include/stop_slave.inc - -connection server_1; -INSERT INTO t2 VALUES (6, "i6b"); -INSERT INTO t2 VALUES (7, "i7b"); ---source include/save_master_gtid.inc - -connection server_3; ---replace_result $SERVER_MYPORT_4 SERVER_MYPORT_4 -eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SERVER_MYPORT_4, - MASTER_USE_GTID=CURRENT_POS; ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc -SELECT * FROM t2 ORDER BY a; - ---echo *** Now change everything back to what it was, to make rpl_end.inc happy -# Also check that MASTER_USE_GTID=CURRENT_POS is still enabled. -connection server_2; -# We need to sync up server_2 before switching. If it happened to have reached -# the point 'UPDATE t2 SET b="j1a" WHERE a=5' it will fail to connect to -# server_1, which is (deliberately) missing that transaction. ---source include/sync_with_master_gtid.inc ---source include/stop_slave.inc ---replace_result $MASTER_MYPORT MASTER_MYPORT -eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT; ---source include/start_slave.inc ---source include/wait_for_slave_to_start.inc - -connection server_3; ---source include/stop_slave.inc ---replace_result $SLAVE_MYPORT SLAVE_MYPORT -eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SLAVE_MYPORT; ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc - -connection server_4; ---source include/stop_slave.inc ---replace_result $SERVER_MYPORT_3 SERVER_MYPORT_3 -eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $SERVER_MYPORT_3; ---source include/start_slave.inc - -connection server_1; -DROP TABLE t1,t2; ---source include/save_master_gtid.inc - ---echo *** A few more checks for BINLOG_GTID_POS function *** ---let $valid_binlog_name = query_get_value(SHOW BINARY LOGS,Log_name,1) ---error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT -SELECT BINLOG_GTID_POS(); ---error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT -SELECT BINLOG_GTID_POS('a'); ---error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT -SELECT BINLOG_GTID_POS('a',1,NULL); -SELECT BINLOG_GTID_POS(1,'a'); -SELECT BINLOG_GTID_POS(NULL,NULL); -SELECT BINLOG_GTID_POS('',1); -SELECT BINLOG_GTID_POS('a',1); -eval SELECT BINLOG_GTID_POS('$valid_binlog_name',-1); -eval SELECT BINLOG_GTID_POS('$valid_binlog_name',0); -eval SELECT BINLOG_GTID_POS('$valid_binlog_name',18446744073709551615); -eval SELECT BINLOG_GTID_POS('$valid_binlog_name',18446744073709551616); - - ---echo *** Some tests of @@GLOBAL.gtid_binlog_state *** ---connection server_2 ---source include/sync_with_master_gtid.inc ---source include/stop_slave.inc - ---connection server_1 -SET @old_state= @@GLOBAL.gtid_binlog_state; - ---error ER_BINLOG_MUST_BE_EMPTY -SET GLOBAL gtid_binlog_state = ''; -RESET MASTER; -SET GLOBAL gtid_binlog_state = ''; -FLUSH LOGS; ---source include/show_binary_logs.inc -SET GLOBAL gtid_binlog_state = '0-1-10,1-2-20,0-3-30'; ---source include/show_binary_logs.inc ---let $binlog_file= master-bin.000001 ---let $binlog_start= 4 ---source include/show_binlog_events.inc -#SELECT @@GLOBAL.gtid_binlog_pos; -#SELECT @@GLOBAL.gtid_binlog_state; ---error ER_BINLOG_MUST_BE_EMPTY -SET GLOBAL gtid_binlog_state = @old_state; -RESET MASTER; -SET GLOBAL gtid_binlog_state = @old_state; - -# Check that slave can reconnect again, despite the RESET MASTER, as we -# restored the state. - -CREATE TABLE t1 (a INT PRIMARY KEY); -SET gtid_seq_no=100; -INSERT INTO t1 VALUES (1); ---source include/save_master_gtid.inc - ---connection server_2 ---source include/start_slave.inc -# We cannot just use sync_with_master as we've done RESET MASTER, so -# slave old-style position is wrong. -# So sync on gtid position instead. ---source include/sync_with_master_gtid.inc - -SELECT * FROM t1; -# Check that the IO gtid position in SHOW SLAVE STATUS is also correct. ---let $status_items= Gtid_IO_Pos ---source include/show_slave_status.inc - ---echo *** Test @@LAST_GTID and MASTER_GTID_WAIT() *** - ---connection server_1 -DROP TABLE t1; -CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; ---save_master_pos - ---connection server_2 ---sync_with_master ---source include/stop_slave.inc - ---connect (m1,127.0.0.1,root,,test,$SERVER_MYPORT_1,) -SELECT @@last_gtid; -SET gtid_seq_no=110; -SELECT @@last_gtid; -BEGIN; -SELECT @@last_gtid; -INSERT INTO t1 VALUES (2); -SELECT @@last_gtid; -COMMIT; -SELECT @@last_gtid; ---let $pos= `SELECT @@gtid_binlog_pos` - ---connect (s1,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -eval SET @pos= '$pos'; -# Check NULL argument. -SELECT master_gtid_wait(NULL); -# Check empty argument returns immediately. -SELECT master_gtid_wait('', NULL); -# Check this gets counted -SHOW STATUS LIKE 'Master_gtid_wait_count'; -SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; -SHOW STATUS LIKE 'Master_gtid_wait_time'; -# Let's check that we get a timeout -SELECT master_gtid_wait(@pos, 0.5); -SELECT * FROM t1 ORDER BY a; -# Now actually wait until the slave reaches the position -send SELECT master_gtid_wait(@pos); - ---connection server_2 ---source include/start_slave.inc - ---connection s1 -reap; -SELECT * FROM t1 ORDER BY a; - -# Test waiting on a domain that does not exist yet. ---source include/stop_slave.inc - ---connection server_1 -SET gtid_domain_id= 1; -INSERT INTO t1 VALUES (3); ---let $pos= `SELECT @@gtid_binlog_pos` - ---connection s1 ---replace_result $pos POS -eval SET @pos= '$pos'; -SELECT master_gtid_wait(@pos, 0); -SELECT * FROM t1 WHERE a >= 3; -send SELECT master_gtid_wait(@pos, -1); - ---connection server_2 ---source include/start_slave.inc - ---connection s1 -reap; -SELECT * FROM t1 WHERE a >= 3; -# Waiting for only part of the position. -SELECT master_gtid_wait('1-1-1', 0); - -# Now test a lot of parallel master_gtid_wait() calls, completing in different -# order, and some of which time out or get killed on the way. - ---connection s1 -send SELECT master_gtid_wait('2-1-1,1-1-4,0-1-110'); - ---connect (s2,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -# This will time out. No event 0-1-1000 exists -send SELECT master_gtid_wait('0-1-1000', 0.5); - ---connect (s3,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -# This one we will kill ---let $kill1_id= `SELECT connection_id()` -send SELECT master_gtid_wait('0-1-2000'); - ---connect (s4,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -send SELECT master_gtid_wait('2-1-10'); - ---connect (s5,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -send SELECT master_gtid_wait('2-1-6', 1); - -# This one we will kill also. ---connect (s6,127.0.0.1,root,,test,$SERVER_MYPORT_2,) ---let $kill2_id= `SELECT connection_id()` -send SELECT master_gtid_wait('2-1-5'); - ---connect (s7,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -send SELECT master_gtid_wait('2-1-10'); - ---connect (s8,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -send SELECT master_gtid_wait('2-1-5,1-1-4,0-1-110'); - ---connect (s9,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -send SELECT master_gtid_wait('2-1-2'); - ---connection server_2 -# This one completes immediately. -SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; -SHOW STATUS LIKE 'Master_gtid_wait_count'; -SELECT master_gtid_wait('1-1-1'); -SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; -SHOW STATUS LIKE 'Master_gtid_wait_count'; -let $wait_time = query_get_value(SHOW STATUS LIKE 'Master_gtid_wait_time', Value, 1); ---replace_result $wait_time MASTER_GTID_WAIT_TIME -eval SET @a= $wait_time; -SELECT IF(@a <= 100*1000*1000, "OK", CONCAT("Error: wait time ", @a, " is larger than expected")) - AS Master_gtid_wait_time_as_expected; - - ---connect (s10,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -send SELECT master_gtid_wait('0-1-109'); - ---connection server_2 -# This one should time out. -SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; -SHOW STATUS LIKE 'Master_gtid_wait_count'; -SELECT master_gtid_wait('2-1-2', 0.5); -SHOW STATUS LIKE 'Master_gtid_wait_timeouts'; -SHOW STATUS LIKE 'Master_gtid_wait_count'; -let $wait_time = query_get_value(SHOW STATUS LIKE 'Master_gtid_wait_time', Value, 1); ---replace_result $wait_time MASTER_GTID_WAIT_TIME -eval SET @a= $wait_time; -# We expect a wait time of just a bit over 0.5 seconds. But thread scheduling -# and timer inaccuracies could introduce significant jitter. So allow a -# generous interval. -SELECT IF(@a BETWEEN 0.4*1000*1000 AND 100*1000*1000, "OK", CONCAT("Error: wait time ", @a, " not as expected")) AS Master_gtid_wait_time_as_expected; - ---replace_result $kill1_id KILL_ID -eval KILL QUERY $kill1_id; ---connection s3 ---error ER_QUERY_INTERRUPTED -reap; - ---connection server_1 -SET gtid_domain_id=2; -SET gtid_seq_no=2; -INSERT INTO t1 VALUES (4); - ---connection s9 -reap; - ---connection server_2 ---replace_result $kill2_id KILL_ID -eval KILL CONNECTION $kill2_id; - ---connection s6 ---error 2013,ER_CONNECTION_KILLED -reap; - ---connection server_1 -SET gtid_domain_id=1; -SET gtid_seq_no=4; -INSERT INTO t1 VALUES (5); -SET gtid_domain_id=2; -SET gtid_seq_no=5; -INSERT INTO t1 VALUES (6); - ---connection s8 -reap; ---connection s1 -reap; ---connection s2 -reap; ---connection s5 -reap; ---connection s10 -reap; - ---connection server_1 -SET gtid_domain_id=2; -SET gtid_seq_no=10; -INSERT INTO t1 VALUES (7); - ---connection s4 -reap; ---connection s7 -reap; - - ---echo *** Test gtid_slave_pos when used with GTID *** - ---connection server_2 ---source include/stop_slave.inc - ---connection server_1 -SET gtid_domain_id=2; -SET gtid_seq_no=1000; -INSERT INTO t1 VALUES (10); -INSERT INTO t1 VALUES (11); ---save_master_pos - ---connection server_2 -SET sql_slave_skip_counter= 1; ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t1 WHERE a >= 10 ORDER BY a; -SELECT IF(LOCATE("2-1-1001", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1001 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - ---source include/stop_slave.inc - ---connection server_1 -SET gtid_domain_id=2; -SET gtid_seq_no=1010; -INSERT INTO t1 VALUES (12); -INSERT INTO t1 VALUES (13); ---save_master_pos - ---connection server_2 -SET sql_slave_skip_counter= 2; ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t1 WHERE a >= 10 ORDER BY a; -SELECT IF(LOCATE("2-1-1011", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1011 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - ---source include/stop_slave.inc - ---connection server_1 -SET gtid_domain_id=2; -SET gtid_seq_no=1020; -INSERT INTO t1 VALUES (14); -INSERT INTO t1 VALUES (15); -INSERT INTO t1 VALUES (16); ---save_master_pos - ---connection server_2 -SET sql_slave_skip_counter= 3; ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t1 WHERE a >= 10 ORDER BY a; -SELECT IF(LOCATE("2-1-1022", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1022 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - ---source include/stop_slave.inc - ---connection server_1 -SET gtid_domain_id=2; -SET gtid_seq_no=1030; -INSERT INTO t1 VALUES (17); -INSERT INTO t1 VALUES (18); -INSERT INTO t1 VALUES (19); ---save_master_pos - ---connection server_2 -SET sql_slave_skip_counter= 5; ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t1 WHERE a >= 10 ORDER BY a; -SELECT IF(LOCATE("2-1-1032", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1032 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - - ---source include/stop_slave.inc - ---connection server_1 -SET gtid_domain_id=3; -SET gtid_seq_no=100; -CREATE TABLE t2 (a INT PRIMARY KEY); -DROP TABLE t2; -SET gtid_domain_id=2; -SET gtid_seq_no=1040; -INSERT INTO t1 VALUES (20); ---save_master_pos - ---connection server_2 -SET @saved_mode= @@GLOBAL.slave_ddl_exec_mode; -SET GLOBAL slave_ddl_exec_mode=STRICT; -SET sql_slave_skip_counter=1; -START SLAVE UNTIL master_gtid_pos="3-1-100"; ---let $master_pos=3-1-100 ---source include/sync_with_master_gtid.inc ---source include/wait_for_slave_to_stop.inc ---error ER_NO_SUCH_TABLE -SELECT * FROM t2; -SELECT IF(LOCATE("3-1-100", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-100 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - -# Start the slave again, it should fail on the DROP TABLE as the table is not there. -SET sql_log_bin=0; -CALL mtr.add_suppression("Slave: Unknown table 'test\\.t2' Error_code: 1051"); -SET sql_log_bin=1; -START SLAVE; ---let $slave_sql_errno=1051 ---source include/wait_for_slave_sql_error.inc -SELECT IF(LOCATE("3-1-100", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-100 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - -STOP SLAVE IO_THREAD; -SET sql_slave_skip_counter=2; ---source include/start_slave.inc ---sync_with_master - -SELECT * FROM t1 WHERE a >= 20 ORDER BY a; -SELECT IF(LOCATE("3-1-101", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 3-1-101 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; -SELECT IF(LOCATE("2-1-1040", @@GLOBAL.gtid_slave_pos)>0, "Ok", CONCAT("ERROR! expected GTID 2-1-1040 not found in gtid_slave_pos: ", @@GLOBAL.gtid_slave_pos)) AS status; - -SET GLOBAL slave_ddl_exec_mode= @saved_mode; - - ---echo *** Test GTID-connecting to a master with out-of-order sequence numbers in the binlog. *** - -# Create an out-of-order binlog on server 2. -# Let server 3 replicate to an out-of-order point, stop it, restart it, -# and check that it replicates correctly despite the out-of-order. - ---connection server_1 -SET gtid_domain_id= @@GLOBAL.gtid_domain_id; -INSERT INTO t1 VALUES (31); ---save_master_pos - ---connection server_2 ---sync_with_master -SET gtid_domain_id= @@GLOBAL.gtid_domain_id; -INSERT INTO t1 VALUES (32); - ---connection server_1 -INSERT INTO t1 VALUES (33); ---save_master_pos - ---connection server_2 ---sync_with_master ---save_master_pos - ---connection server_3 ---sync_with_master ---source include/stop_slave.inc - ---connection server_1 -INSERT INTO t1 VALUES (34); ---save_master_pos - ---connection server_2 ---sync_with_master ---save_master_pos - ---connection server_3 ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t1 WHERE a >= 30 ORDER BY a; ---save_master_pos - ---connection server_4 ---sync_with_master -SELECT * FROM t1 WHERE a >= 30 ORDER BY a; - - -# Clean up. ---connection server_1 -DROP TABLE t1; - - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_gtid_basic.inc diff --git a/mysql-test/suite/rpl/t/rpl_incident.test b/mysql-test/suite/rpl/t/rpl_incident.test index adf20953b0f10..9be855e1a8b17 100644 --- a/mysql-test/suite/rpl/t/rpl_incident.test +++ b/mysql-test/suite/rpl/t/rpl_incident.test @@ -1,48 +1 @@ ---source include/master-slave.inc ---source include/have_debug.inc - ---echo **** On Master **** -CREATE TABLE t1 (a INT); - -INSERT INTO t1 VALUES (1),(2),(3); -SELECT * FROM t1; - -let $debug_save= `SELECT @@GLOBAL.debug`; -SET GLOBAL debug_dbug= '+d,incident_database_resync_on_replace,*'; - -# This will generate an incident log event and store it in the binary -# log before the replace statement. -REPLACE INTO t1 VALUES (4); ---save_master_pos -SELECT * FROM t1; - ---disable_query_log -eval SET GLOBAL debug_dbug= '$debug_save'; ---enable_query_log - -connection slave; -# Wait until SQL thread stops with error LOST_EVENT on master -call mtr.add_suppression("Slave SQL.*The incident LOST_EVENTS occurred on the master.* 1590"); -let $slave_sql_errno= 1590; -let $show_slave_sql_error= 1; -source include/wait_for_slave_sql_error.inc; - -# The 4 should not be inserted into the table, since the incident log -# event should have stop the slave. ---echo **** On Slave **** -SELECT * FROM t1; - -SET GLOBAL SQL_SLAVE_SKIP_COUNTER=1; -START SLAVE; ---sync_with_master - -# Now, we should have inserted the row into the table and the slave -# should be running. We should also have rotated to a new binary log. - -SELECT * FROM t1; -source include/check_slave_is_running.inc; - -connection master; -DROP TABLE t1; ---sync_slave_with_master ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_incident.inc diff --git a/mysql-test/suite/rpl/t/rpl_init_slave_errors.test b/mysql-test/suite/rpl/t/rpl_init_slave_errors.test index 067d21a4e461d..6f515b9390a4a 100644 --- a/mysql-test/suite/rpl/t/rpl_init_slave_errors.test +++ b/mysql-test/suite/rpl/t/rpl_init_slave_errors.test @@ -1,89 +1 @@ -###################################################################### -# Some errors that cause the slave SQL thread to stop are not shown in -# the Slave_SQL_Error column of "SHOW SLAVE STATUS". Instead, the error -# is only in the server's error log. -# -# Two failures and their respective reporting are verified: -# -# 1 - Failures during slave thread initialization -# 2 - Failures while processing queries passed through the init_slave -# option. -# -# In order to check the first type of failure, we inject a fault in the -# SQL/IO Threads through SET GLOBAL debug. -# -# To check the second type, we set @@global.init_slave to an invalid -# command thus preventing the initialization of the SQL Thread. -# -# Obs: -# 1 - Note that testing failures while initializing the relay log position -# is hard as the same function is called before the code reaches the point -# that we want to test. -# -# 2 - This test does not target failures that are reported while applying -# events such as duplicate keys, errors while reading the relay-log.bin*, -# etc. Such errors are already checked on other tests. -###################################################################### - -###################################################################### -# Configuring the Environment -###################################################################### -source include/have_debug.inc; -source include/master-slave.inc; -source include/have_log_bin.inc; - -connection slave; - ---disable_warnings -stop slave; ---enable_warnings -reset slave; - -###################################################################### -# Injecting faults in the threads' initialization -###################################################################### -connection slave; - -# Set debug flags on slave to force errors to occur -SET GLOBAL debug_dbug= "d,simulate_io_slave_error_on_init,simulate_sql_slave_error_on_init"; - -start slave; - -# -# slave is going to stop because of emulated failures -# but there won't be any crashes nor asserts hit. -# -# 1593 = ER_SLAVE_FATAL_ERROR ---let $slave_sql_errno= 1593 ---let $show_slave_sql_error= 1 ---source include/wait_for_slave_sql_error.inc - -call mtr.add_suppression("Failed during slave.* thread initialization"); - -SET GLOBAL debug_dbug= ""; - -###################################################################### -# Injecting faults in the init_slave option -###################################################################### -connection slave; - -reset slave; - -SET GLOBAL init_slave= "garbage"; - -start slave; -# 1064 = ER_PARSE_ERROR ---let $slave_sql_errno= 1064 ---let $show_slave_sql_error= 1 ---source include/wait_for_slave_sql_error.inc - -###################################################################### -# Clean up -###################################################################### -SET GLOBAL init_slave= ""; - -# Clean up Last_SQL_Error ---source include/stop_slave_io.inc -RESET SLAVE; ---let $rpl_only_running_threads= 1 ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_init_slave_errors.inc diff --git a/mysql-test/suite/rpl/t/rpl_loaddatalocal.test b/mysql-test/suite/rpl/t/rpl_loaddatalocal.test index 30c9ff1e9e25e..8d90afaed27fa 100644 --- a/mysql-test/suite/rpl/t/rpl_loaddatalocal.test +++ b/mysql-test/suite/rpl/t/rpl_loaddatalocal.test @@ -1,240 +1 @@ -# See if "LOAD DATA LOCAL INFILE" is well replicated -# (LOAD DATA LOCAL INFILE is not written to the binlog -# the same way as LOAD DATA INFILE : Append_blocks are smaller). -# In MySQL 4.0 <4.0.12 there were 2 bugs with LOAD DATA LOCAL INFILE : -# - the loaded file was not written entirely to the master's binlog, -# only the first 4KB, 8KB or 16KB usually. -# - the loaded file's first line was not written entirely to the -# master's binlog (1st char was absent) -source include/master-slave.inc; - -create table t1(a int); -let $1=10000; -disable_query_log; -set SQL_LOG_BIN=0; -while ($1) -{ - insert into t1 values(1); - dec $1; -} -set SQL_LOG_BIN=1; -enable_query_log; -let $MYSQLD_DATADIR= `select @@datadir`; ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval select * into outfile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; -#This will generate a 20KB file, now test LOAD DATA LOCAL -truncate table t1; ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval load data local infile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; ---remove_file $MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile -sync_slave_with_master; -select a,count(*) from t1 group by a; -connection master; -drop table t1; -sync_slave_with_master; - -# End of 4.1 tests - -# -# Now let us test how well we replicate LOAD DATA LOCAL in situation when -# we met duplicates in tables to which we are adding rows. -# (It supposed that LOAD DATA LOCAL ignores such errors) -# -connection master; -create table t1(a int); -insert into t1 values (1), (2), (2), (3); ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval select * into outfile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; -drop table t1; -create table t1(a int primary key); ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval load data local infile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; ---remove_file $MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile -SELECT * FROM t1 ORDER BY a; -save_master_pos; -connection slave; -sync_with_master; -SELECT * FROM t1 ORDER BY a; -connection master; -drop table t1; -save_master_pos; -connection slave; -sync_with_master; - - -# -# Bug22504 load data infile sql statement in replication architecture get error -# ---echo ==== Bug22504 Initialize ==== - ---echo [on master] ---connection master - -SET sql_mode='ignore_space'; -CREATE TABLE t1(a int); -insert into t1 values (1), (2), (3), (4); ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval select * into outfile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' from t1; -truncate table t1; ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval load data local infile '$MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile' into table t1; ---remove_file $MYSQLD_DATADIR/rpl_loaddatalocal.select_outfile -SELECT * FROM t1 ORDER BY a; - ---echo [on slave] -sync_slave_with_master; -SELECT * FROM t1 ORDER BY a; - ---echo ==== Clean up ==== - ---echo [on master] -connection master; -DROP TABLE t1; - ---echo [on slave] -sync_slave_with_master; - ---echo ---echo Bug #43746: ---echo "return wrong query string when parse 'load data infile' sql statement" ---echo - ---echo [master] -connection master; -let $MYSQLD_DATADIR= `select @@datadir`; -SELECT @@SESSION.sql_mode INTO @old_mode; - -SET sql_mode='ignore_space'; - -CREATE TABLE t1(a int); -INSERT INTO t1 VALUES (1), (2), (3), (4); - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval SELECT * INTO OUTFILE '$MYSQLD_DATADIR/bug43746.sql' FROM t1; -TRUNCATE TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD/* look mum, with comments in weird places! */DATA/* oh hai */LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql'/* we are */INTO/* from the internets */TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' /*!10000 INTO */ TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' /*!10000 INTO TABLE */ t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA /*!10000 LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE */ t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql'/*!10000 INTO*/TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql'/* empty */INTO TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA/*!10000 LOCAL */INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO/* empty */TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD/*!999999 special comments that do not expand */DATA/*!999999 code from the future */LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql'/*!999999 have flux capacitor */INTO/*!999999 will travel */TABLE t1; - -SET sql_mode='PIPES_AS_CONCAT,ANSI_QUOTES,NO_KEY_OPTIONS,NO_TABLE_OPTIONS,NO_FIELD_OPTIONS,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER'; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug43746.sql' INTO TABLE t1; - ---echo [slave] -sync_slave_with_master; - ---echo ---echo Bug #59267: ---echo "LOAD DATA LOCAL INFILE not executed on slave with SBR" ---echo - ---echo [master] -connection master; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval SELECT * INTO OUTFILE '$MYSQLD_DATADIR/bug59267.sql' FROM t1; -TRUNCATE TABLE t1; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug59267.sql' INTO TABLE t1; - -SELECT 'Master', COUNT(*) FROM t1; - ---echo [slave] ---sync_slave_with_master -SELECT 'Slave', COUNT(*) FROM t1; - -# cleanup ---echo [master] -connection master; - ---remove_file $MYSQLD_DATADIR/bug43746.sql ---remove_file $MYSQLD_DATADIR/bug59267.sql - -DROP TABLE t1; -SET SESSION sql_mode=@old_mode; - ---echo [slave] -sync_slave_with_master; - -connection master; - ---echo ---echo Bug #60580/#11902767: ---echo "statement improperly replicated crashes slave sql thread" ---echo - ---echo [master] -connection master; -let $MYSQLD_DATADIR= `select @@datadir`; - -CREATE TABLE t1(f1 INT, f2 INT); -CREATE TABLE t2(f1 INT, f2 TIMESTAMP); - -INSERT INTO t2 VALUES(1, '2011-03-22 21:01:28'); -INSERT INTO t2 VALUES(2, '2011-03-21 21:01:28'); -INSERT INTO t2 VALUES(3, '2011-03-20 21:01:28'); - -CREATE TABLE t3 AS SELECT * FROM t2; - -CREATE VIEW v1 AS SELECT * FROM t2 - WHERE f1 IN (SELECT f1 FROM t3 WHERE (t3.f2 IS NULL)); - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval SELECT 1 INTO OUTFILE '$MYSQLD_DATADIR/bug60580.csv' FROM DUAL; - ---replace_result $MYSQLD_DATADIR MYSQLD_DATADIR -eval LOAD DATA LOCAL INFILE '$MYSQLD_DATADIR/bug60580.csv' INTO TABLE t1 (@f1) SET f2 = (SELECT f1 FROM v1 WHERE f1=@f1); - -SELECT * FROM t1; - -sleep 1; - ---echo [slave] -sync_slave_with_master; - -SELECT * FROM t1; - ---remove_file $MYSQLD_DATADIR/bug60580.csv - ---echo [master] -connection master; - -DROP VIEW v1; -DROP TABLE t1, t2, t3; - ---echo [slave] -sync_slave_with_master; - -connection master; ---source include/rpl_end.inc - ---echo # End of 5.1 tests +--source extra/rpl_tests/rpl_loaddata_local.inc diff --git a/mysql-test/suite/rpl/t/rpl_loadfile.test b/mysql-test/suite/rpl/t/rpl_loadfile.test index 30a7ae2f6138f..babf4208b3d62 100644 --- a/mysql-test/suite/rpl/t/rpl_loadfile.test +++ b/mysql-test/suite/rpl/t/rpl_loadfile.test @@ -1,114 +1 @@ -############################################################################# -# Original Author: JBM # -# Original Date: Aug/18/2005 # -############################################################################# -# TEST: To test the LOAD_FILE() in rbr # -############################################################################# -# Change Author: JBM -# Change Date: 2006-01-16 -########## - -# Includes --- source include/have_binlog_format_mixed_or_row.inc --- source include/master-slave.inc - --- source extra/rpl_tests/rpl_loadfile.test - -# BUG#39701: Mixed binlog format does not switch to row mode on LOAD_FILE -# -# DESCRIPTION -# -# Problem: when using load_file string function and mixed binlogging format -# there was no switch to row based binlogging format. This leads -# to scenarios on which the slave replicates the statement and it -# will try to load the file from local file system, which in most -# likely it will not exist. -# -# Solution: -# Marking this function as unsafe for statement format, makes the -# statement using it to be logged in row based format. As such, data -# replicated from the master, becomes the content of the loaded file. -# Consequently, the slave receives the necessary data to complete -# the load_file instruction correctly. -# -# IMPLEMENTATION -# -# The test is implemented as follows: -# -# On Master, -# i) write to file the desired content. -# ii) create table and stored procedure with load_file -# iii) stop slave -# iii) execute load_file -# iv) remove file -# -# On Slave, -# v) start slave -# vi) sync it with master so that it gets the updates from binlog (which -# should have bin logged in row format). -# -# If the the binlog format does not change to row, then the assertion -# done in the following step fails. This happens because tables differ -# since the file does not exist anymore, meaning that when slave -# attempts to execute LOAD_FILE statement it inserts NULL on table -# instead of the same contents that the master loaded when it executed -# the procedure (which was executed when file existed). -# -# vii) assert that the contents of master and slave -# table are the same - ---source include/rpl_reset.inc - -connection master; -let $file= $MYSQLTEST_VARDIR/tmp/bug_39701.data; - ---replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---eval SELECT repeat('x',20) INTO OUTFILE '$file' - -disable_warnings; -DROP TABLE IF EXISTS t1; -enable_warnings; - -CREATE TABLE t1 (t text); -DELIMITER |; -CREATE PROCEDURE p(file varchar(4096)) - BEGIN - INSERT INTO t1 VALUES (LOAD_FILE(file)); - END| -DELIMITER ;| - -# stop slave before issuing the load_file on master -connection slave; -source include/stop_slave.inc; - -connection master; - -# test: check that logging falls back to rbr. ---replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---eval CALL p('$file') - -# test: remove the file from the filesystem and assert that slave still -# gets the loaded file -remove_file $file; - -# now that the file is removed it is safe (regarding what we want to test) -# to start slave -connection slave; -source include/start_slave.inc; - -connection master; -sync_slave_with_master; - -# assertion: assert that the slave got the updates even -# if the file was removed before the slave started, -# meaning that contents were indeed transfered -# through binlog (in row format) -let $diff_tables= master:t1, slave:t1; -source include/diff_tables.inc; - -# CLEAN UP ---connection master -DROP TABLE t1; -DROP PROCEDURE p; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_loadfile.inc diff --git a/mysql-test/suite/rpl/t/rpl_packet.test b/mysql-test/suite/rpl/t/rpl_packet.test index 0e74481bb6143..31357cb148e5e 100644 --- a/mysql-test/suite/rpl/t/rpl_packet.test +++ b/mysql-test/suite/rpl/t/rpl_packet.test @@ -1,177 +1 @@ -# ==== Purpose ==== -# -# Check replication protocol packet size handling -# -# ==== Related bugs ==== -# Bug#19402 SQL close to the size of the max_allowed_packet fails on slave -# BUG#23755: Replicated event larger that max_allowed_packet infinitely re-transmits -# BUG#42914: No LAST_IO_ERROR for max_allowed_packet errors -# BUG#55322: SHOW BINLOG EVENTS increases @@SESSION.MAX_ALLOWED_PACKET - -# max-out size db name -source include/master-slave.inc; -source include/have_binlog_format_row.inc; -call mtr.add_suppression("Slave I/O: Got a packet bigger than 'slave_max_allowed_packet' bytes, .*error.* 1153"); -call mtr.add_suppression("Log entry on master is longer than slave_max_allowed_packet"); -let $db= DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________; -disable_warnings; -eval drop database if exists $db; -enable_warnings; -eval create database $db; - -connection master; -let $old_max_allowed_packet= `SELECT @@global.max_allowed_packet`; -let $old_net_buffer_length= `SELECT @@global.net_buffer_length`; -let $old_slave_max_allowed_packet= `SELECT @@global.slave_max_allowed_packet`; -SET @@global.max_allowed_packet=1024; -SET @@global.net_buffer_length=1024; - -sync_slave_with_master; -# Restart slave for setting to take effect -source include/stop_slave.inc; -source include/start_slave.inc; - -# Reconnect to master for new setting to take effect -disconnect master; - -# alas, can't use eval here; if db name changed apply the change here -connect (master,localhost,root,,DB_NAME_OF_MAX_LENGTH_AKA_NAME_LEN_64_BYTES_____________________); - -connection master; -select @@net_buffer_length, @@max_allowed_packet; - -create table `t1` (`f1` LONGTEXT) ENGINE=MyISAM; - -INSERT INTO `t1`(`f1`) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1023'); -sync_slave_with_master; - -eval select count(*) from `$db`.`t1` /* must be 1 */; - -SHOW STATUS LIKE 'Slave_running'; -select * from information_schema.session_status where variable_name= 'SLAVE_RUNNING'; -connection master; -eval drop database $db; -sync_slave_with_master; - -# -# Bug #23755: Replicated event larger that max_allowed_packet infinitely re-transmits -# -# Check that a situation when the size of event on the master is greater than -# max_allowed_packet on the slave does not lead to infinite re-transmits. - -connection master; - -# Change the max packet size on master - -SET @@global.max_allowed_packet=4096; -SET @@global.net_buffer_length=4096; - -# Restart slave for new setting to take effect -connection slave; -source include/stop_slave.inc; -source include/start_slave.inc; - -# Reconnect to master for new setting to take effect -disconnect master; -connect (master, localhost, root); -connection master; - -CREATE TABLE `t1` (`f1` LONGTEXT) ENGINE=MyISAM; - -sync_slave_with_master; - -connection master; - -INSERT INTO `t1`(`f1`) VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2048'); - - -# -# Bug#42914: The slave I/O thread must stop after trying to read the above -# event, However there is no Last_IO_Error report. -# - -# The slave I/O thread must stop after trying to read the above event -connection slave; -# 1153 = ER_NET_PACKET_TOO_LARGE ---let $slave_io_errno= 1153 ---let $show_slave_io_error= 1 ---source include/wait_for_slave_io_error.inc - -# TODO: this is needed because of BUG#55790. Remove once that is fixed. ---source include/stop_slave_sql.inc - -# -# Bug#42914: On the master, if a binary log event is larger than -# max_allowed_packet, the error message ER_MASTER_FATAL_ERROR_READING_BINLOG -# is sent to a slave when it requests a dump from the master, thus leading the -# I/O thread to stop. However, there is no Last_IO_Error reported. -# - ---let $rpl_only_running_threads= 1 ---source include/rpl_reset.inc ---connection master -DROP TABLE t1; ---sync_slave_with_master - - -connection master; -CREATE TABLE t1 (f1 int PRIMARY KEY, f2 LONGTEXT, f3 LONGTEXT) ENGINE=MyISAM; -sync_slave_with_master; - -connection master; -INSERT INTO t1(f1, f2, f3) VALUES(1, REPEAT('a', @@global.max_allowed_packet), REPEAT('b', @@global.max_allowed_packet)); - -connection slave; -# The slave I/O thread must stop after receiving -# 1153 = ER_NET_PACKET_TOO_LARGE ---let $slave_io_errno= 1153 ---let $show_slave_io_error= 1 ---source include/wait_for_slave_io_error.inc - -# Remove the bad binlog and clear error status on slave. -STOP SLAVE; -RESET SLAVE; ---connection master -RESET MASTER; - - -# -# BUG#55322: SHOW BINLOG EVENTS increases @@SESSION.MAX_ALLOWED_PACKET -# -# In BUG#55322, @@session.max_allowed_packet increased each time SHOW -# BINLOG EVENTS was issued. To verify that this bug is fixed, we -# execute SHOW BINLOG EVENTS twice and check that max_allowed_packet -# never changes. We turn off the result log because we don't care -# about the contents of the binlog. - ---disable_result_log -SET @max_allowed_packet_0= @@session.max_allowed_packet; -SHOW BINLOG EVENTS; -SET @max_allowed_packet_1= @@session.max_allowed_packet; -SHOW BINLOG EVENTS; -SET @max_allowed_packet_2= @@session.max_allowed_packet; ---enable_result_log -if (`SELECT NOT(@max_allowed_packet_0 = @max_allowed_packet_1 AND @max_allowed_packet_1 = @max_allowed_packet_2)`) -{ - --echo ERROR: max_allowed_packet changed after executing SHOW BINLOG EVENTS - --source include/show_rpl_debug_info.inc - SELECT @max_allowed_packet_0, @max_allowed_packet_1, @max_allowed_packet_2; - --die @max_allowed_packet changed after executing SHOW BINLOG EVENTS -} - - ---echo ==== clean up ==== -connection master; -DROP TABLE t1; -eval SET @@global.max_allowed_packet= $old_max_allowed_packet; -eval SET @@global.net_buffer_length= $old_net_buffer_length; -eval SET @@global.slave_max_allowed_packet= $old_slave_max_allowed_packet; -# slave is stopped -connection slave; -DROP TABLE t1; - -# Clear Last_IO_Error -RESET SLAVE; - ---source include/rpl_end.inc -# End of tests +--source extra/rpl_tests/rpl_packet.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel.test b/mysql-test/suite/rpl/t/rpl_parallel.test index a7e5353a9fc5c..b7c4bb429a493 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel.test +++ b/mysql-test/suite/rpl/t/rpl_parallel.test @@ -1,2473 +1 @@ ---source include/have_innodb.inc ---source include/have_debug.inc ---source include/have_debug_sync.inc ---source include/master-slave.inc - -# Test various aspects of parallel replication. - ---connection server_2 -SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; ---error ER_SLAVE_MUST_STOP -SET GLOBAL slave_parallel_threads=10; ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=10; - -# Check that we do not spawn any worker threads when no slave is running. -SELECT IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; - -CHANGE MASTER TO master_use_gtid=slave_pos; ---source include/start_slave.inc - -# Check that worker threads get spawned when slave starts. -SELECT IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; -# ... and that worker threads get removed when slave stops. ---source include/stop_slave.inc -SELECT IF(COUNT(*) < 10, "OK", CONCAT("Found too many system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; ---source include/start_slave.inc -SELECT IF(COUNT(*) >= 10, "OK", CONCAT("Found too few system user processes: ", COUNT(*))) FROM information_schema.processlist WHERE user = "system user"; - ---echo *** Test long-running query in domain 1 can run in parallel with short queries in domain 0 *** - ---connection server_1 -ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; -CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=MyISAM; -CREATE TABLE t2 (a int PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO t1 VALUES (1); -INSERT INTO t2 VALUES (1); ---save_master_pos - ---connection server_2 ---sync_with_master - -# Block the table t1 to simulate a replicated query taking a long time. ---connect (con_temp1,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -LOCK TABLE t1 WRITE; - ---connection server_1 -SET gtid_domain_id=1; -# This query will be blocked on the slave until UNLOCK TABLES. -INSERT INTO t1 VALUES (2); -SET gtid_domain_id=0; -# These t2 queries can be replicated in parallel with the prior t1 query, as -# they are in a separate replication domain. -INSERT INTO t2 VALUES (2); -INSERT INTO t2 VALUES (3); -BEGIN; -INSERT INTO t2 VALUES (4); -INSERT INTO t2 VALUES (5); -COMMIT; -INSERT INTO t2 VALUES (6); - ---connection server_2 ---let $wait_condition= SELECT COUNT(*) = 6 FROM t2 ---source include/wait_condition.inc - -SELECT * FROM t2 ORDER by a; - ---connection con_temp1 -SELECT * FROM t1; -UNLOCK TABLES; - ---connection server_2 ---let $wait_condition= SELECT COUNT(*) = 2 FROM t1 ---source include/wait_condition.inc - -SELECT * FROM t1 ORDER BY a; - - ---echo *** Test two transactions in different domains committed in opposite order on slave but in a single group commit. *** ---connection server_2 ---source include/stop_slave.inc - ---connection server_1 -# Use a stored function to inject a debug_sync into the appropriate THD. -# The function does nothing on the master, and on the slave it injects the -# desired debug_sync action(s). -SET sql_log_bin=0; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - -SET @old_format= @@SESSION.binlog_format; -SET binlog_format='statement'; -SET gtid_domain_id=1; -INSERT INTO t2 VALUES (foo(10, - 'commit_before_enqueue SIGNAL ready1 WAIT_FOR cont1', - 'commit_after_release_LOCK_prepare_ordered SIGNAL ready2')); - ---connection server_2 -FLUSH LOGS; ---source include/wait_for_binlog_checkpoint.inc -SET sql_log_bin=0; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - IF d1 != '' THEN - SET debug_sync = d1; - END IF; - IF d2 != '' THEN - SET debug_sync = d2; - END IF; - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; -SET @old_format=@@GLOBAL.binlog_format; -SET GLOBAL binlog_format=statement; -# We need to restart all parallel threads for the new global setting to -# be copied to the session-level values. -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - -# First make sure the first insert is ready to commit, but not queued yet. -SET debug_sync='now WAIT_FOR ready1'; - ---connection server_1 -SET gtid_domain_id=2; -INSERT INTO t2 VALUES (foo(11, - 'commit_before_enqueue SIGNAL ready3 WAIT_FOR cont3', - 'commit_after_release_LOCK_prepare_ordered SIGNAL ready4 WAIT_FOR cont4')); -SET gtid_domain_id=0; -SELECT * FROM t2 WHERE a >= 10 ORDER BY a; - ---connection server_2 -# Now wait for the second insert to queue itself as the leader, and then -# wait for more commits to queue up. -SET debug_sync='now WAIT_FOR ready3'; -SET debug_sync='now SIGNAL cont3'; -SET debug_sync='now WAIT_FOR ready4'; -# Now allow the first insert to queue up to participate in group commit. -SET debug_sync='now SIGNAL cont1'; -SET debug_sync='now WAIT_FOR ready2'; -# Finally allow the second insert to proceed and do the group commit. -SET debug_sync='now SIGNAL cont4'; - ---let $wait_condition= SELECT COUNT(*) = 2 FROM t2 WHERE a >= 10 ---source include/wait_condition.inc -SELECT * FROM t2 WHERE a >= 10 ORDER BY a; -# The two INSERT transactions should have been committed in opposite order, -# but in the same group commit (seen by precense of cid=# in the SHOW -# BINLOG output). ---let $binlog_file= slave-bin.000002 ---source include/show_binlog_events.inc -FLUSH LOGS; ---source include/wait_for_binlog_checkpoint.inc - -# Restart all the slave parallel worker threads, to clear all debug_sync actions. ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; -SET debug_sync='RESET'; ---source include/start_slave.inc - - ---echo *** Test that group-committed transactions on the master can replicate in parallel on the slave. *** ---connection server_1 -SET debug_sync='RESET'; -FLUSH LOGS; ---source include/wait_for_binlog_checkpoint.inc -CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; -# Create some sentinel rows so that the rows inserted in parallel fall into -# separate gaps and do not cause gap lock conflicts. -INSERT INTO t3 VALUES (1,1), (3,3), (5,5), (7,7); ---save_master_pos ---connection server_2 ---sync_with_master - -# We want to test that the transactions can execute out-of-order on -# the slave, but still end up committing in-order, and in a single -# group commit. -# -# The idea is to group-commit three transactions together on the master: -# A, B, and C. On the slave, C will execute the insert first, then A, -# and then B. But B manages to complete before A has time to commit, so -# all three end up committing together. -# -# So we start by setting up some row locks that will block transactions -# A and B from executing, allowing C to run first. - ---connection con_temp1 -BEGIN; -INSERT INTO t3 VALUES (2,102); ---connect (con_temp2,127.0.0.1,root,,test,$SERVER_MYPORT_2,) -BEGIN; -INSERT INTO t3 VALUES (4,104); - -# On the master, queue three INSERT transactions as a single group commit. ---connect (con_temp3,127.0.0.1,root,,test,$SERVER_MYPORT_1,) -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (2, foo(12, - 'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued1 WAIT_FOR slave_cont1', - '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connect (con_temp4,127.0.0.1,root,,test,$SERVER_MYPORT_1,) -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (4, foo(14, - 'commit_after_release_LOCK_prepare_ordered SIGNAL slave_queued2', - '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; - ---connect (con_temp5,127.0.0.1,root,,test,$SERVER_MYPORT_1,) -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (6, foo(16, - 'group_commit_waiting_for_prior SIGNAL slave_queued3', - '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued3'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con_temp3 -REAP; ---connection con_temp4 -REAP; ---connection con_temp5 -REAP; -SET debug_sync='RESET'; - ---connection server_1 -SELECT * FROM t3 ORDER BY a; ---let $binlog_file= master-bin.000002 ---source include/show_binlog_events.inc - -# First, wait until insert 3 is ready to queue up for group commit, but is -# waiting for insert 2 to commit before it can do so itself. ---connection server_2 -SET debug_sync='now WAIT_FOR slave_queued3'; - -# Next, let insert 1 proceed, and allow it to queue up as the group commit -# leader, but let it wait for insert 2 to also queue up before proceeding. ---connection con_temp1 -ROLLBACK; ---connection server_2 -SET debug_sync='now WAIT_FOR slave_queued1'; - -# Now let insert 2 proceed and queue up. ---connection con_temp2 -ROLLBACK; ---connection server_2 -SET debug_sync='now WAIT_FOR slave_queued2'; -# And finally, we can let insert 1 proceed and do the group commit with all -# three insert transactions together. -SET debug_sync='now SIGNAL slave_cont1'; - -# Wait for the commit to complete and check that all three transactions -# group-committed together (will be seen in the binlog as all three having -# cid=# on their GTID event). ---let $wait_condition= SELECT COUNT(*) = 3 FROM t3 WHERE a IN (2,4,6) ---source include/wait_condition.inc -SELECT * FROM t3 ORDER BY a; ---let $binlog_file= slave-bin.000003 ---source include/show_binlog_events.inc - - ---echo *** Test STOP SLAVE in parallel mode *** ---connection server_2 ---source include/stop_slave.inc -# Respawn all worker threads to clear any left-over debug_sync or other stuff. -SET debug_sync='RESET'; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; - ---connection server_1 -# Set up a couple of transactions. The first will be blocked halfway -# through on a lock, and while it is blocked we initiate STOP SLAVE. -# We then test that the halfway-initiated transaction is allowed to -# complete, but no subsequent ones. -# We have to use statement-based mode and set -# binlog_direct_non_transactional_updates=0; otherwise the binlog will -# be split into two event groups, one for the MyISAM part and one for the -# InnoDB part. -SET binlog_direct_non_transactional_updates=0; -SET sql_log_bin=0; -CALL mtr.add_suppression("Statement is unsafe because it accesses a non-transactional table after accessing a transactional table within the same transaction"); -SET sql_log_bin=1; -BEGIN; -INSERT INTO t2 VALUES (20); ---disable_warnings -INSERT INTO t1 VALUES (20); ---enable_warnings -INSERT INTO t2 VALUES (21); -INSERT INTO t3 VALUES (20, 20); -COMMIT; -INSERT INTO t3 VALUES(21, 21); -INSERT INTO t3 VALUES(22, 22); -SET binlog_format=@old_format; ---save_master_pos - -# Start a connection that will block the replicated transaction halfway. ---connection con_temp1 -BEGIN; -INSERT INTO t2 VALUES (21); - ---connection server_2 -START SLAVE; -# Wait for the MyISAM change to be visible, after which replication will wait -# for con_temp1 to roll back. ---let $wait_condition= SELECT COUNT(*) = 1 FROM t1 WHERE a=20 ---source include/wait_condition.inc - ---connection con_temp2 -# Initiate slave stop. It will have to wait for the current event group -# to complete. -# The dbug injection causes debug_sync to signal 'wait_for_done_waiting' -# when the SQL driver thread is ready. -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; -send STOP SLAVE; - ---connection con_temp1 -SET debug_sync='now WAIT_FOR wait_for_done_waiting'; -ROLLBACK; - ---connection con_temp2 -reap; -SET GLOBAL debug_dbug=@old_dbug; -SET debug_sync='RESET'; - ---connection server_2 ---source include/wait_for_slave_to_stop.inc -# We should see the first transaction applied, but not the two others. -SELECT * FROM t1 WHERE a >= 20 ORDER BY a; -SELECT * FROM t2 WHERE a >= 20 ORDER BY a; -SELECT * FROM t3 WHERE a >= 20 ORDER BY a; - ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t1 WHERE a >= 20 ORDER BY a; -SELECT * FROM t2 WHERE a >= 20 ORDER BY a; -SELECT * FROM t3 WHERE a >= 20 ORDER BY a; - - ---connection server_2 -# Respawn all worker threads to clear any left-over debug_sync or other stuff. ---source include/stop_slave.inc -SET GLOBAL binlog_format=@old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** Test killing slave threads at various wait points *** ---echo *** 1. Test killing transaction waiting in commit for previous transaction to commit *** - -# Set up three transactions on the master that will be group-committed -# together so they can be replicated in parallel on the slave. ---connection con_temp3 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (31, foo(31, - 'commit_before_prepare_ordered WAIT_FOR t2_waiting', - 'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con_temp4 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -SET binlog_format=statement; -BEGIN; -# This insert is just so we can get T2 to wait while a query is running that we -# can see in SHOW PROCESSLIST so we can get its thread_id to kill later. -INSERT INTO t3 VALUES (32, foo(32, - 'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', - '')); -# This insert sets up debug_sync points so that T2 will tell when it is at its -# wait point where we want to kill it - and when it has been killed. -INSERT INTO t3 VALUES (33, foo(33, - 'group_commit_waiting_for_prior SIGNAL t2_waiting', - 'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); -send COMMIT; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; - ---connection con_temp5 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (34, foo(34, - '', - '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued3'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con_temp3 -REAP; ---connection con_temp4 -REAP; ---connection con_temp5 -REAP; - ---connection server_1 -SELECT * FROM t3 WHERE a >= 30 ORDER BY a; -SET debug_sync='RESET'; - ---connection server_2 -SET sql_log_bin=0; -CALL mtr.add_suppression("Query execution was interrupted"); -CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); -CALL mtr.add_suppression("Slave: Connection was killed"); -SET sql_log_bin=1; -# Wait until T2 is inside executing its insert of 32, then find it in SHOW -# PROCESSLIST to know its thread id for KILL later. -SET debug_sync='now WAIT_FOR t2_query'; ---let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(32%' AND INFO NOT LIKE '%LIKE%'` -SET debug_sync='now SIGNAL t2_cont'; - -# Wait until T2 has entered its wait for T1 to commit, and T1 has -# progressed into its commit phase. -SET debug_sync='now WAIT_FOR t1_ready'; - -# Now kill the transaction T2. ---replace_result $thd_id THD_ID -eval KILL $thd_id; - -# Wait until T2 has reacted on the kill. -SET debug_sync='now WAIT_FOR t2_killed'; - -# Now we can allow T1 to proceed. -SET debug_sync='now SIGNAL t1_cont'; - ---let $slave_sql_errno= 1317,1927,1964 ---source include/wait_for_slave_sql_error.inc -STOP SLAVE IO_THREAD; -SELECT * FROM t3 WHERE a >= 30 ORDER BY a; - -# Now we have to disable the debug_sync statements, so they do not trigger -# when the events are retried. -SET debug_sync='RESET'; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - ---connection server_1 -INSERT INTO t3 VALUES (39,0); ---save_master_pos - ---connection server_2 ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t3 WHERE a >= 30 ORDER BY a; -# Restore the foo() function. -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - IF d1 != '' THEN - SET debug_sync = d1; - END IF; - IF d2 != '' THEN - SET debug_sync = d2; - END IF; - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - - ---connection server_2 -# Respawn all worker threads to clear any left-over debug_sync or other stuff. ---source include/stop_slave.inc -SET GLOBAL binlog_format=@old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** 2. Same as (1), but without restarting IO thread after kill of SQL threads *** - -# Set up three transactions on the master that will be group-committed -# together so they can be replicated in parallel on the slave. ---connection con_temp3 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (41, foo(41, - 'commit_before_prepare_ordered WAIT_FOR t2_waiting', - 'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con_temp4 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -SET binlog_format=statement; -BEGIN; -# This insert is just so we can get T2 to wait while a query is running that we -# can see in SHOW PROCESSLIST so we can get its thread_id to kill later. -INSERT INTO t3 VALUES (42, foo(42, - 'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', - '')); -# This insert sets up debug_sync points so that T2 will tell when it is at its -# wait point where we want to kill it - and when it has been killed. -INSERT INTO t3 VALUES (43, foo(43, - 'group_commit_waiting_for_prior SIGNAL t2_waiting', - 'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); -send COMMIT; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; - ---connection con_temp5 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (44, foo(44, - '', - '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued3'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con_temp3 -REAP; ---connection con_temp4 -REAP; ---connection con_temp5 -REAP; - ---connection server_1 -SELECT * FROM t3 WHERE a >= 40 ORDER BY a; -SET debug_sync='RESET'; - ---connection server_2 -# Wait until T2 is inside executing its insert of 42, then find it in SHOW -# PROCESSLIST to know its thread id for KILL later. -SET debug_sync='now WAIT_FOR t2_query'; ---let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(42%' AND INFO NOT LIKE '%LIKE%'` -SET debug_sync='now SIGNAL t2_cont'; - -# Wait until T2 has entered its wait for T1 to commit, and T1 has -# progressed into its commit phase. -SET debug_sync='now WAIT_FOR t1_ready'; - -# Now kill the transaction T2. ---replace_result $thd_id THD_ID -eval KILL $thd_id; - -# Wait until T2 has reacted on the kill. -SET debug_sync='now WAIT_FOR t2_killed'; - -# Now we can allow T1 to proceed. -SET debug_sync='now SIGNAL t1_cont'; - ---let $slave_sql_errno= 1317,1927,1964 ---source include/wait_for_slave_sql_error.inc - -# Now we have to disable the debug_sync statements, so they do not trigger -# when the events are retried. -SET debug_sync='RESET'; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - ---connection server_1 -INSERT INTO t3 VALUES (49,0); ---save_master_pos - ---connection server_2 -START SLAVE SQL_THREAD; ---sync_with_master -SELECT * FROM t3 WHERE a >= 40 ORDER BY a; -# Restore the foo() function. -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - IF d1 != '' THEN - SET debug_sync = d1; - END IF; - IF d2 != '' THEN - SET debug_sync = d2; - END IF; - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - - ---connection server_2 -# Respawn all worker threads to clear any left-over debug_sync or other stuff. ---source include/stop_slave.inc -SET GLOBAL binlog_format=@old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** 3. Same as (2), but not using gtid mode *** - ---connection server_2 ---source include/stop_slave.inc -CHANGE MASTER TO master_use_gtid=no; ---source include/start_slave.inc - ---connection server_1 -# Set up three transactions on the master that will be group-committed -# together so they can be replicated in parallel on the slave. ---connection con_temp3 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (51, foo(51, - 'commit_before_prepare_ordered WAIT_FOR t2_waiting', - 'commit_after_prepare_ordered SIGNAL t1_ready WAIT_FOR t1_cont')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con_temp4 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -SET binlog_format=statement; -BEGIN; -# This insert is just so we can get T2 to wait while a query is running that we -# can see in SHOW PROCESSLIST so we can get its thread_id to kill later. -INSERT INTO t3 VALUES (52, foo(52, - 'ha_write_row_end SIGNAL t2_query WAIT_FOR t2_cont', - '')); -# This insert sets up debug_sync points so that T2 will tell when it is at its -# wait point where we want to kill it - and when it has been killed. -INSERT INTO t3 VALUES (53, foo(53, - 'group_commit_waiting_for_prior SIGNAL t2_waiting', - 'group_commit_waiting_for_prior_killed SIGNAL t2_killed')); -send COMMIT; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; - ---connection con_temp5 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; -SET binlog_format=statement; -send INSERT INTO t3 VALUES (54, foo(54, - '', - '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued3'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con_temp3 -REAP; ---connection con_temp4 -REAP; ---connection con_temp5 -REAP; - ---connection server_1 -SELECT * FROM t3 WHERE a >= 50 ORDER BY a; -SET debug_sync='RESET'; - ---connection server_2 -# Wait until T2 is inside executing its insert of 52, then find it in SHOW -# PROCESSLIST to know its thread id for KILL later. -SET debug_sync='now WAIT_FOR t2_query'; ---let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(52%' AND INFO NOT LIKE '%LIKE%'` -SET debug_sync='now SIGNAL t2_cont'; - -# Wait until T2 has entered its wait for T1 to commit, and T1 has -# progressed into its commit phase. -SET debug_sync='now WAIT_FOR t1_ready'; - -# Now kill the transaction T2. ---replace_result $thd_id THD_ID -eval KILL $thd_id; - -# Wait until T2 has reacted on the kill. -SET debug_sync='now WAIT_FOR t2_killed'; - -# Now we can allow T1 to proceed. -SET debug_sync='now SIGNAL t1_cont'; - ---let $slave_sql_errno= 1317,1927,1964 ---source include/wait_for_slave_sql_error.inc -SELECT * FROM t3 WHERE a >= 50 ORDER BY a; - -# Now we have to disable the debug_sync statements, so they do not trigger -# when the events are retried. -SET debug_sync='RESET'; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - ---connection server_1 -INSERT INTO t3 VALUES (59,0); ---save_master_pos - ---connection server_2 -START SLAVE SQL_THREAD; ---sync_with_master -SELECT * FROM t3 WHERE a >= 50 ORDER BY a; -# Restore the foo() function. -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - IF d1 != '' THEN - SET debug_sync = d1; - END IF; - IF d2 != '' THEN - SET debug_sync = d2; - END IF; - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - - ---source include/stop_slave.inc -CHANGE MASTER TO master_use_gtid=slave_pos; ---source include/start_slave.inc - ---connection server_2 -# Respawn all worker threads to clear any left-over debug_sync or other stuff. ---source include/stop_slave.inc -SET GLOBAL binlog_format=@old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=4; ---source include/start_slave.inc - - ---echo *** 4. Test killing thread that is waiting to start transaction until previous transaction commits *** - -# We set up four transactions T1, T2, T3, and T4 on the master. T2, T3, and T4 -# can run in parallel with each other (same group commit and commit id), -# but not in parallel with T1. -# -# We use four worker threads, each Ti will be queued on each their own -# worker thread. We will delay T1 commit, T3 will wait for T1 to begin -# commit before it can start. We will kill T3 during this wait, and -# check that everything works correctly. -# -# It is rather tricky to get the correct thread id of the worker to kill. -# We start by injecting four dummy transactions in a debug_sync-controlled -# manner to be able to get known thread ids for the workers in a pool with -# just 4 worker threads. Then we let in each of the real test transactions -# T1-T4 one at a time in a way which allows us to know which transaction -# ends up with which thread id. - ---connection server_1 -SET binlog_format=statement; -SET gtid_domain_id=2; -BEGIN; -# This debug_sync will linger on and be used to control T4 later. -INSERT INTO t3 VALUES (70, foo(70, - 'rpl_parallel_start_waiting_for_prior SIGNAL t4_waiting', '')); -INSERT INTO t3 VALUES (60, foo(60, - 'ha_write_row_end SIGNAL d2_query WAIT_FOR d2_cont2', - 'rpl_parallel_end_of_group SIGNAL d2_done WAIT_FOR d2_cont')); -COMMIT; -SET gtid_domain_id=0; - ---connection server_2 -SET debug_sync='now WAIT_FOR d2_query'; ---let $d2_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(60%' AND INFO NOT LIKE '%LIKE%'` - ---connection server_1 -SET gtid_domain_id=1; -BEGIN; -# These debug_sync's will linger on and be used to control T3 later. -INSERT INTO t3 VALUES (61, foo(61, - 'rpl_parallel_start_waiting_for_prior SIGNAL t3_waiting', - 'rpl_parallel_start_waiting_for_prior_killed SIGNAL t3_killed')); -INSERT INTO t3 VALUES (62, foo(62, - 'ha_write_row_end SIGNAL d1_query WAIT_FOR d1_cont2', - 'rpl_parallel_end_of_group SIGNAL d1_done WAIT_FOR d1_cont')); -COMMIT; -SET gtid_domain_id=0; - ---connection server_2 -SET debug_sync='now WAIT_FOR d1_query'; ---let $d1_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(62%' AND INFO NOT LIKE '%LIKE%'` - ---connection server_1 -SET gtid_domain_id=0; -INSERT INTO t3 VALUES (63, foo(63, - 'ha_write_row_end SIGNAL d0_query WAIT_FOR d0_cont2', - 'rpl_parallel_end_of_group SIGNAL d0_done WAIT_FOR d0_cont')); - ---connection server_2 -SET debug_sync='now WAIT_FOR d0_query'; ---let $d0_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(63%' AND INFO NOT LIKE '%LIKE%'` - ---connection server_1 -SET gtid_domain_id=3; -BEGIN; -# These debug_sync's will linger on and be used to control T2 later. -INSERT INTO t3 VALUES (68, foo(68, - 'rpl_parallel_start_waiting_for_prior SIGNAL t2_waiting', '')); -INSERT INTO t3 VALUES (69, foo(69, - 'ha_write_row_end SIGNAL d3_query WAIT_FOR d3_cont2', - 'rpl_parallel_end_of_group SIGNAL d3_done WAIT_FOR d3_cont')); -COMMIT; -SET gtid_domain_id=0; - ---connection server_2 -SET debug_sync='now WAIT_FOR d3_query'; ---let $d3_thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%foo(69%' AND INFO NOT LIKE '%LIKE%'` - -SET debug_sync='now SIGNAL d2_cont2'; -SET debug_sync='now WAIT_FOR d2_done'; -SET debug_sync='now SIGNAL d1_cont2'; -SET debug_sync='now WAIT_FOR d1_done'; -SET debug_sync='now SIGNAL d0_cont2'; -SET debug_sync='now WAIT_FOR d0_done'; -SET debug_sync='now SIGNAL d3_cont2'; -SET debug_sync='now WAIT_FOR d3_done'; - -# Now prepare the real transactions T1, T2, T3, T4 on the master. - ---connection con_temp3 -# Create transaction T1. -SET binlog_format=statement; -INSERT INTO t3 VALUES (64, foo(64, - 'rpl_parallel_before_mark_start_commit SIGNAL t1_waiting WAIT_FOR t1_cont', '')); - -# Create transaction T2, as a group commit leader on the master. -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2 WAIT_FOR master_cont2'; -send INSERT INTO t3 VALUES (65, foo(65, '', '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; - ---connection con_temp4 -# Create transaction T3, participating in T2's group commit. -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3'; -send INSERT INTO t3 VALUES (66, foo(66, '', '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued3'; - ---connection con_temp5 -# Create transaction T4, participating in group commit with T2 and T3. -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued4'; -send INSERT INTO t3 VALUES (67, foo(67, '', '')); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued4'; -SET debug_sync='now SIGNAL master_cont2'; - ---connection con_temp3 -REAP; ---connection con_temp4 -REAP; ---connection con_temp5 -REAP; - ---connection server_1 -SELECT * FROM t3 WHERE a >= 60 ORDER BY a; -SET debug_sync='RESET'; - ---connection server_2 -# Now we have the four transactions pending for replication on the slave. -# Let them be queued for our three worker threads in a controlled fashion. -# We put them at a stage where T1 is delayed and T3 is waiting for T1 to -# commit before T3 can start. Then we kill T3. - -# Make the worker D0 free, and wait for T1 to be queued in it. -SET debug_sync='now SIGNAL d0_cont'; -SET debug_sync='now WAIT_FOR t1_waiting'; - -# Make the worker D3 free, and wait for T2 to be queued in it. -SET debug_sync='now SIGNAL d3_cont'; -SET debug_sync='now WAIT_FOR t2_waiting'; - -# Now release worker D1, and wait for T3 to be queued in it. -# T3 will wait for T1 to commit before it can start. -SET debug_sync='now SIGNAL d1_cont'; -SET debug_sync='now WAIT_FOR t3_waiting'; - -# Release worker D2. Wait for T4 to be queued, so we are sure it has -# received the debug_sync signal (else we might overwrite it with the -# next debug_sync). -SET debug_sync='now SIGNAL d2_cont'; -SET debug_sync='now WAIT_FOR t4_waiting'; - -# Now we kill the waiting transaction T3 in worker D1. ---replace_result $d1_thd_id THD_ID -eval KILL $d1_thd_id; - -# Wait until T3 has reacted on the kill. -SET debug_sync='now WAIT_FOR t3_killed'; - -# Now we can allow T1 to proceed. -SET debug_sync='now SIGNAL t1_cont'; - ---let $slave_sql_errno= 1317,1927,1964 ---source include/wait_for_slave_sql_error.inc -STOP SLAVE IO_THREAD; -# Since T2, T3, and T4 run in parallel, we can not be sure if T2 will have time -# to commit or not before the stop. However, T1 should commit, and T3/T4 may -# not have committed. (After slave restart we check that all become committed -# eventually). -SELECT * FROM t3 WHERE a >= 60 AND a != 65 ORDER BY a; - -# Now we have to disable the debug_sync statements, so they do not trigger -# when the events are retried. -SET debug_sync='RESET'; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - ---connection server_1 -UPDATE t3 SET b=b+1 WHERE a=60; ---save_master_pos - ---connection server_2 ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t3 WHERE a >= 60 ORDER BY a; -# Restore the foo() function. -SET sql_log_bin=0; -DROP FUNCTION foo; ---delimiter || -CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) - RETURNS INT DETERMINISTIC - BEGIN - IF d1 != '' THEN - SET debug_sync = d1; - END IF; - IF d2 != '' THEN - SET debug_sync = d2; - END IF; - RETURN x; - END -|| ---delimiter ; -SET sql_log_bin=1; - ---connection server_2 -# Respawn all worker threads to clear any left-over debug_sync or other stuff. ---source include/stop_slave.inc -SET GLOBAL binlog_format=@old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** 5. Test killing thread that is waiting for queue of max length to shorten *** - -# Find the thread id of the driver SQL thread that we want to kill. ---let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE '%Slave has read all relay log%' ---source include/wait_condition.inc ---let $thd_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE '%Slave has read all relay log%'` -SET @old_max_queued= @@GLOBAL.slave_parallel_max_queued; -SET GLOBAL slave_parallel_max_queued=9000; - ---connection server_1 ---let bigstring= `SELECT REPEAT('x', 10000)` -SET binlog_format=statement; -# Create an event that will wait to be signalled. -INSERT INTO t3 VALUES (80, foo(0, - 'ha_write_row_end SIGNAL query_waiting WAIT_FOR query_cont', '')); - ---connection server_2 -SET debug_sync='now WAIT_FOR query_waiting'; -# Inject that the SQL driver thread will signal `wait_queue_ready' to debug_sync -# as it goes to wait for the event queue to become smaller than the value of -# @@slave_parallel_max_queued. -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,rpl_parallel_wait_queue_max"; - ---connection server_1 ---disable_query_log -# Create an event that will fill up the queue. -# The Xid event at the end of the event group will have to wait for the Query -# event with the INSERT to drain so the queue becomes shorter. However that in -# turn waits for the prior event group to continue. -eval INSERT INTO t3 VALUES (81, LENGTH('$bigstring')); ---enable_query_log -SELECT * FROM t3 WHERE a >= 80 ORDER BY a; - ---connection server_2 -SET debug_sync='now WAIT_FOR wait_queue_ready'; - ---replace_result $thd_id THD_ID -eval KILL $thd_id; - -SET debug_sync='now WAIT_FOR wait_queue_killed'; -SET debug_sync='now SIGNAL query_cont'; - ---let $slave_sql_errno= 1317,1927,1964 ---source include/wait_for_slave_sql_error.inc -STOP SLAVE IO_THREAD; - -SET GLOBAL debug_dbug=@old_dbug; -SET GLOBAL slave_parallel_max_queued= @old_max_queued; - ---connection server_1 -INSERT INTO t3 VALUES (82,0); -SET binlog_format=@old_format; ---save_master_pos - ---connection server_2 -SET debug_sync='RESET'; ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t3 WHERE a >= 80 ORDER BY a; - - ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL binlog_format=@old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - ---echo *** MDEV-5788 Incorrect free of rgi->deferred_events in parallel replication *** - ---connection server_2 -# Use just two worker threads, so we are sure to get the rpl_group_info added -# to the free list, which is what triggered the bug. ---source include/stop_slave.inc -SET GLOBAL replicate_ignore_table="test.t3"; -SET GLOBAL slave_parallel_threads=2; ---source include/start_slave.inc - ---connection server_1 -INSERT INTO t3 VALUES (100, rand()); -INSERT INTO t3 VALUES (101, rand()); - ---save_master_pos - ---connection server_2 ---sync_with_master - ---connection server_1 -INSERT INTO t3 VALUES (102, rand()); -INSERT INTO t3 VALUES (103, rand()); -INSERT INTO t3 VALUES (104, rand()); -INSERT INTO t3 VALUES (105, rand()); - ---save_master_pos - ---connection server_2 ---sync_with_master ---source include/stop_slave.inc -SET GLOBAL replicate_ignore_table=""; ---source include/start_slave.inc - ---connection server_1 -INSERT INTO t3 VALUES (106, rand()); -INSERT INTO t3 VALUES (107, rand()); ---save_master_pos - ---connection server_2 ---sync_with_master ---replace_column 2 # -SELECT * FROM t3 WHERE a >= 100 ORDER BY a; - - ---echo *** MDEV-5921: In parallel replication, an error is not correctly signalled to the next transaction *** - ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - ---connection server_1 -INSERT INTO t3 VALUES (110, 1); ---save_master_pos - ---connection server_2 ---sync_with_master -SELECT * FROM t3 WHERE a >= 110 ORDER BY a; -# Inject a duplicate key error. -SET sql_log_bin=0; -INSERT INTO t3 VALUES (111, 666); -SET sql_log_bin=1; - ---connection server_1 - -# Create a group commit with two inserts, the first one conflicts with a row on the slave ---connect (con1,127.0.0.1,root,,test,$SERVER_MYPORT_1,) -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -send INSERT INTO t3 VALUES (111, 2); ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connect (con2,127.0.0.1,root,,test,$SERVER_MYPORT_1,) -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -send INSERT INTO t3 VALUES (112, 3); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con1 -REAP; ---connection con2 -REAP; -SET debug_sync='RESET'; ---save_master_pos - ---connection server_2 ---let $slave_sql_errno= 1062 ---source include/wait_for_slave_sql_error.inc ---source include/wait_for_slave_sql_to_stop.inc -# We should not see the row (112,3) here, it should be rolled back due to -# error signal from the prior transaction. -SELECT * FROM t3 WHERE a >= 110 ORDER BY a; -SET sql_log_bin=0; -DELETE FROM t3 WHERE a=111 AND b=666; -SET sql_log_bin=1; -START SLAVE SQL_THREAD; ---sync_with_master -SELECT * FROM t3 WHERE a >= 110 ORDER BY a; - - ---echo ***MDEV-5914: Parallel replication deadlock due to InnoDB lock conflicts *** ---connection server_2 ---source include/stop_slave.inc - ---connection server_1 -CREATE TABLE t4 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB; -INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); - -# Create a group commit with UPDATE and DELETE, in that order. -# The bug was that while the UPDATE's row lock does not block the DELETE, the -# DELETE's gap lock _does_ block the UPDATE. This could cause a deadlock -# on the slave. ---connection con1 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -send UPDATE t4 SET b=NULL WHERE a=6; ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con2 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -send DELETE FROM t4 WHERE b <= 3; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con1 -REAP; ---connection con2 -REAP; -SET debug_sync='RESET'; ---save_master_pos - ---connection server_2 ---source include/start_slave.inc ---sync_with_master ---source include/stop_slave.inc - -SELECT * FROM t4 ORDER BY a; - - -# Another example, this one with INSERT vs. DELETE ---connection server_1 -DELETE FROM t4; -INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); - -# Create a group commit with INSERT and DELETE, in that order. -# The bug was that while the INSERT's insert intention lock does not block -# the DELETE, the DELETE's gap lock _does_ block the INSERT. This could cause -# a deadlock on the slave. ---connection con1 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -send INSERT INTO t4 VALUES (7, NULL); ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con2 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -send DELETE FROM t4 WHERE b <= 3; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con1 -REAP; ---connection con2 -REAP; -SET debug_sync='RESET'; ---save_master_pos - ---connection server_2 ---source include/start_slave.inc ---sync_with_master ---source include/stop_slave.inc - -SELECT * FROM t4 ORDER BY a; - - -# MDEV-6549, failing to update gtid_slave_pos for a transaction that was retried. -# The problem was that when a transaction updates the mysql.gtid_slave_pos -# table, it clears the flag that marks that there is a GTID position that -# needs to be updated. Then, if the transaction got killed after that due -# to a deadlock, the subsequent retry would fail to notice that the GTID needs -# to be recorded in gtid_slave_pos. -# -# (In the original bug report, the symptom was an assertion; this was however -# just a side effect of the missing update of gtid_slave_pos, which also -# happened to cause a missing clear of OPTION_GTID_BEGIN). ---connection server_1 -DELETE FROM t4; -INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); - -# Create two transactions that can run in parallel on the slave but cause -# a deadlock if the second runs before the first. ---connection con1 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -send UPDATE t4 SET b=NULL WHERE a=6; ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con2 -# Must use statement-based binlogging. Otherwise the transaction will not be -# binlogged at all, as it modifies no rows. -SET @old_format= @@SESSION.binlog_format; -SET binlog_format='statement'; -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -send DELETE FROM t4 WHERE b <= 1; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con1 -REAP; ---connection con2 -REAP; -SET @old_format=@@GLOBAL.binlog_format; -SET debug_sync='RESET'; ---save_master_pos ---let $last_gtid= `SELECT @@last_gtid` - ---connection server_2 -# Disable the usual skip of gap locks for transactions that are run in -# parallel, using DBUG. This allows the deadlock to occur, and this in turn -# triggers a retry of the second transaction, and the code that was buggy and -# caused the gtid_slave_pos update to be skipped in the retry. -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,disable_thd_need_ordering_with"; ---source include/start_slave.inc ---sync_with_master -SET GLOBAL debug_dbug=@old_dbug; - -SELECT * FROM t4 ORDER BY a; -# Check that the GTID of the second transaction was correctly recorded in -# gtid_slave_pos, in the variable as well as in the table. ---replace_result $last_gtid GTID -eval SET @last_gtid= '$last_gtid'; -SELECT IF(@@gtid_slave_pos LIKE CONCAT('%',@last_gtid,'%'), "GTID found ok", - CONCAT("GTID ", @last_gtid, " not found in gtid_slave_pos=", @@gtid_slave_pos)) - AS result; -SELECT "ROW FOUND" AS `Is the row found?` - FROM mysql.gtid_slave_pos - WHERE CONCAT(domain_id, "-", server_id, "-", seq_no) = @last_gtid; - - ---echo *** MDEV-5938: Exec_master_log_pos not updated at log rotate in parallel replication *** ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=1; -SET DEBUG_SYNC= 'RESET'; ---source include/start_slave.inc - ---connection server_1 -CREATE TABLE t5 (a INT PRIMARY KEY, b INT); -INSERT INTO t5 VALUES (1,1); -INSERT INTO t5 VALUES (2,2), (3,8); -INSERT INTO t5 VALUES (4,16); ---save_master_pos - ---connection server_2 ---sync_with_master -let $io_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1); -let $io_pos= query_get_value(SHOW SLAVE STATUS, Read_Master_Log_Pos, 1); -let $sql_file= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); -let $sql_pos= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1); ---disable_query_log -eval SELECT IF('$io_file' = '$sql_file', "OK", "Not ok, $io_file <> $sql_file") AS test_check; -eval SELECT IF('$io_pos' = '$sql_pos', "OK", "Not ok, $io_pos <> $sql_pos") AS test_check; ---enable_query_log - ---connection server_1 -FLUSH LOGS; ---source include/wait_for_binlog_checkpoint.inc ---save_master_pos - ---connection server_2 ---sync_with_master -let $io_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1); -let $io_pos= query_get_value(SHOW SLAVE STATUS, Read_Master_Log_Pos, 1); -let $sql_file= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); -let $sql_pos= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1); ---disable_query_log -eval SELECT IF('$io_file' = '$sql_file', "OK", "Not ok, $io_file <> $sql_file") AS test_check; -eval SELECT IF('$io_pos' = '$sql_pos', "OK", "Not ok, $io_pos <> $sql_pos") AS test_check; ---enable_query_log - - ---echo *** MDEV_6435: Incorrect error handling when query binlogged partially on master with "killed" error *** - ---connection server_1 -CREATE TABLE t6 (a INT) ENGINE=MyISAM; -CREATE TRIGGER tr AFTER INSERT ON t6 FOR EACH ROW SET @a = 1; - ---connection con1 -SET @old_format= @@binlog_format; -SET binlog_format= statement; ---let $conid = `SELECT CONNECTION_ID()` -SET debug_sync='sp_head_execute_before_loop SIGNAL ready WAIT_FOR cont'; -send INSERT INTO t6 VALUES (1), (2), (3); - ---connection server_1 -SET debug_sync='now WAIT_FOR ready'; ---replace_result $conid CONID -eval KILL QUERY $conid; -SET debug_sync='now SIGNAL cont'; - ---connection con1 ---error ER_QUERY_INTERRUPTED ---reap -SET binlog_format= @old_format; -SET debug_sync='RESET'; ---let $after_error_gtid_pos= `SELECT @@gtid_binlog_pos` - ---connection server_1 -SET debug_sync='RESET'; - - ---connection server_2 ---let $slave_sql_errno= 1317 ---source include/wait_for_slave_sql_error.inc -STOP SLAVE IO_THREAD; ---replace_result $after_error_gtid_pos AFTER_ERROR_GTID_POS -eval SET GLOBAL gtid_slave_pos= '$after_error_gtid_pos'; ---source include/start_slave.inc - ---connection server_1 -INSERT INTO t6 VALUES (4); -SELECT * FROM t6 ORDER BY a; ---save_master_pos - ---connection server_2 ---sync_with_master -SELECT * FROM t6 ORDER BY a; - - ---echo *** MDEV-6551: Some replication errors are ignored if slave_parallel_threads > 0 *** - ---connection server_1 -INSERT INTO t2 VALUES (31); ---let $gtid1= `SELECT @@LAST_GTID` ---source include/save_master_gtid.inc - ---connection server_2 ---source include/sync_with_master_gtid.inc ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads= 0; ---source include/start_slave.inc - -# Force a duplicate key error on the slave. -SET sql_log_bin= 0; -INSERT INTO t2 VALUES (32); -SET sql_log_bin= 1; - ---connection server_1 -INSERT INTO t2 VALUES (32); ---let $gtid2= `SELECT @@LAST_GTID` -# Rotate the binlog; the bug is triggered when the master binlog file changes -# after the event group that causes the duplicate key error. -FLUSH LOGS; -INSERT INTO t2 VALUES (33); -INSERT INTO t2 VALUES (34); -SELECT * FROM t2 WHERE a >= 30 ORDER BY a; ---source include/save_master_gtid.inc - ---connection server_2 ---let $slave_sql_errno= 1062 ---source include/wait_for_slave_sql_error.inc - ---connection server_2 ---source include/stop_slave_io.inc -SET GLOBAL slave_parallel_threads=10; -START SLAVE; - ---let $slave_sql_errno= 1062 ---source include/wait_for_slave_sql_error.inc - -# Note: IO thread is still running at this point. -# The bug seems to have been that restarting the SQL thread after an error with -# the IO thread still running, somehow picks up a later relay log position and -# thus ends up skipping the failing event, rather than re-executing. - -START SLAVE SQL_THREAD; ---let $slave_sql_errno= 1062 ---source include/wait_for_slave_sql_error.inc - -SELECT * FROM t2 WHERE a >= 30 ORDER BY a; - -# Skip the duplicate error, so we can proceed. ---error ER_SLAVE_SKIP_NOT_IN_GTID -SET sql_slave_skip_counter= 1; ---source include/stop_slave_io.inc ---disable_query_log -eval SET GLOBAL gtid_slave_pos = REPLACE(@@gtid_slave_pos, "$gtid1", "$gtid2"); ---enable_query_log ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc - -SELECT * FROM t2 WHERE a >= 30 ORDER BY a; - - ---echo *** MDEV-6775: Wrong binlog order in parallel replication *** ---connection server_1 -# A bit tricky bug to reproduce. On the master, we binlog in statement-mode -# two transactions, an UPDATE followed by a DELETE. On the slave, we replicate -# with binlog-mode set to ROW, which means the DELETE, which modifies no rows, -# is not binlogged. Then we inject a wait in the group commit code on the -# slave, shortly before the actual commit of the UPDATE. The bug was that the -# DELETE could wake up from wait_for_prior_commit() before the commit of the -# UPDATE. So the test could see the slave position updated to after DELETE, -# while the UPDATE was still not visible. -DELETE FROM t4; -INSERT INTO t4 VALUES (1,NULL), (3,NULL), (4,4), (5, NULL), (6, 6); ---source include/save_master_gtid.inc - ---connection server_2 ---source include/sync_with_master_gtid.inc ---source include/stop_slave.inc -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,inject_binlog_commit_before_get_LOCK_log"; -SET @old_format=@@GLOBAL.binlog_format; -SET GLOBAL binlog_format=ROW; -# Re-spawn the worker threads to be sure they pick up the new binlog format -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; - ---connection con1 -SET @old_format= @@binlog_format; -SET binlog_format= statement; -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -send UPDATE t4 SET b=NULL WHERE a=6; ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con2 -SET @old_format= @@binlog_format; -SET binlog_format= statement; -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -send DELETE FROM t4 WHERE b <= 3; - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con1 -REAP; -SET binlog_format= @old_format; ---connection con2 -REAP; -SET binlog_format= @old_format; -SET debug_sync='RESET'; ---save_master_pos -SELECT * FROM t4 ORDER BY a; - ---connection server_2 ---source include/start_slave.inc -SET debug_sync= 'now WAIT_FOR waiting'; ---sync_with_master -SELECT * FROM t4 ORDER BY a; -SET debug_sync= 'now SIGNAL cont'; - -# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; -SET GLOBAL binlog_format= @old_format; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** MDEV-7237: Parallel replication: incorrect relaylog position after stop/start the slave *** ---connection server_1 -INSERT INTO t2 VALUES (40); ---save_master_pos - ---connection server_2 ---sync_with_master ---source include/stop_slave.inc -CHANGE MASTER TO master_use_gtid=no; -SET @old_dbug= @@GLOBAL.debug_dbug; -# This DBUG injection causes a DEBUG_SYNC signal "scheduled_gtid_0_x_100" when -# GTID 0-1-100 has been scheduled for and fetched by a worker thread. -SET GLOBAL debug_dbug="+d,rpl_parallel_scheduled_gtid_0_x_100"; -# This DBUG injection causes a DEBUG_SYNC signal "wait_for_done_waiting" when -# STOP SLAVE has signalled all worker threads to stop. -SET GLOBAL debug_dbug="+d,rpl_parallel_wait_for_done_trigger"; -# Reset worker threads to make DBUG setting catch on. -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; - - ---connection server_1 -# Setup some transaction for the slave to replicate. -INSERT INTO t2 VALUES (41); -INSERT INTO t2 VALUES (42); -# Need to log the DELETE in statement format, so we can see it in processlist. -SET @old_format= @@binlog_format; -SET binlog_format= statement; -DELETE FROM t2 WHERE a=40; -SET binlog_format= @old_format; -INSERT INTO t2 VALUES (43); -INSERT INTO t2 VALUES (44); -# Force the slave to switch to a new relay log file. -FLUSH LOGS; -INSERT INTO t2 VALUES (45); -# Inject a GTID 0-1-100, which will trigger a DEBUG_SYNC signal when this -# transaction has been fetched by a worker thread. -SET gtid_seq_no=100; -INSERT INTO t2 VALUES (46); ---save_master_pos - ---connection con_temp2 -# Temporarily block the DELETE on a=40 from completing. -BEGIN; -SELECT * FROM t2 WHERE a=40 FOR UPDATE; - - ---connection server_2 ---source include/start_slave.inc - -# Wait for a worker thread to start on the DELETE that will be blocked -# temporarily by the SELECT FOR UPDATE. ---let $wait_condition= SELECT count(*) > 0 FROM information_schema.processlist WHERE state='updating' and info LIKE '%DELETE FROM t2 WHERE a=40%' ---source include/wait_condition.inc - -# The DBUG injection set above will make the worker thread signal the following -# debug_sync when the GTID 0-1-100 has been reached by a worker thread. -# Thus, at this point, the SQL driver thread has reached the next -# relay log file name, while a worker thread is still processing a -# transaction in the previous relay log file, blocked on the SELECT FOR -# UPDATE. -SET debug_sync= 'now WAIT_FOR scheduled_gtid_0_x_100'; -# At this point, the SQL driver thread is in the new relay log file, while -# the DELETE from the old relay log file is not yet complete. We will stop -# the slave at this point. The bug was that the DELETE statement would -# update the slave position to the _new_ relay log file name instead of -# its own old file name. Thus, by stoping and restarting the slave at this -# point, we would get an error at restart due to incorrect position. (If -# we would let the slave catch up before stopping, the incorrect position -# would be corrected by a later transaction). - -send STOP SLAVE; - ---connection con_temp2 -# Wait for STOP SLAVE to have proceeded sufficiently that it has signalled -# all worker threads to stop; this ensures that we will stop after the DELETE -# transaction (and not after a later transaction that might have been able -# to set a fixed position). -SET debug_sync= 'now WAIT_FOR wait_for_done_waiting'; -# Now release the row lock that was blocking the replication of DELETE. -ROLLBACK; - ---connection server_2 -reap; ---source include/wait_for_slave_sql_to_stop.inc -SELECT * FROM t2 WHERE a >= 40 ORDER BY a; -# Now restart the slave. With the bug present, this would start at an -# incorrect relay log position, causing relay log read error (or if unlucky, -# silently skip a number of events). ---source include/start_slave.inc ---sync_with_master -SELECT * FROM t2 WHERE a >= 40 ORDER BY a; ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; -SET DEBUG_SYNC= 'RESET'; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; -CHANGE MASTER TO master_use_gtid=slave_pos; ---source include/start_slave.inc - - ---echo *** MDEV-7326 Server deadlock in connection with parallel replication *** -# We use three transactions, each in a separate group commit. -# T1 does mark_start_commit(), then gets a deadlock error. -# T2 wakes up and starts running -# T1 does unmark_start_commit() -# T3 goes to wait for T2 to start its commit -# T2 does mark_start_commit() -# The bug was that at this point, T3 got deadlocked. Because T1 has unmarked(), -# T3 did not yet see the count_committing_event_groups reach its target value -# yet. But when T1 later re-did mark_start_commit(), it failed to send a wakeup -# to T3. - ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=3; -SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; ---source include/start_slave.inc - ---connection server_1 -SET @old_format= @@SESSION.binlog_format; -SET binlog_format= STATEMENT; -# This debug_sync will linger on and be used to control T3 later. -INSERT INTO t1 VALUES (foo(50, - "rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", - "rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); ---save_master_pos ---connection server_2 -# Wait for the debug_sync point for T3 to be set. But let the preparation -# transaction remain hanging, so that T1 and T2 will be scheduled for the -# remaining two worker threads. -SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; - ---connection server_1 -INSERT INTO t2 VALUES (foo(50, - "rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", - "rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); ---save_master_pos - ---connection server_2 -SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; -# T1 has now done mark_start_commit(). It will later do a rollback and retry. - ---connection server_1 -# Use a MyISAM table for T2 and T3, so they do not trigger the -# rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event. -INSERT INTO t1 VALUES (foo(51, - "rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", - "rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); - ---connection server_2 -SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; -# T2 has now started running, but has not yet done mark_start_commit() -SET DEBUG_SYNC= "now SIGNAL t1_cont1"; -SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; -# T1 has now done unmark_start_commit() in preparation for its retry. - ---connection server_1 -INSERT INTO t1 VALUES (52); -SET BINLOG_FORMAT= @old_format; -SELECT * FROM t2 WHERE a>=50 ORDER BY a; -SELECT * FROM t1 WHERE a>=50 ORDER BY a; - ---connection server_2 -# Let the preparation transaction complete, so that the same worker thread -# can continue with the transaction T3. -SET DEBUG_SYNC= "now SIGNAL prep_cont"; -SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; -# T3 has now gone to wait for T2 to start committing -SET DEBUG_SYNC= "now SIGNAL t2_cont1"; -SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; -# T2 has now done mark_start_commit(). -# Let things run, and check that T3 does not get deadlocked. -SET DEBUG_SYNC= "now SIGNAL t1_cont2"; ---sync_with_master - ---connection server_1 ---save_master_pos ---connection server_2 ---sync_with_master -SELECT * FROM t2 WHERE a>=50 ORDER BY a; -SELECT * FROM t1 WHERE a>=50 ORDER BY a; -SET DEBUG_SYNC="reset"; - -# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** MDEV-7326 Server deadlock in connection with parallel replication *** -# Similar to the previous test, but with T2 and T3 in the same GCO. -# We use three transactions, T1 in one group commit and T2/T3 in another. -# T1 does mark_start_commit(), then gets a deadlock error. -# T2 wakes up and starts running -# T1 does unmark_start_commit() -# T3 goes to wait for T1 to start its commit -# T2 does mark_start_commit() -# The bug was that at this point, T3 got deadlocked. T2 increments the -# count_committing_event_groups but does not signal T3, as they are in -# the same GCO. Then later when T1 increments, it would also not signal -# T3, because now the count_committing_event_groups is not equal to the -# wait_count of T3 (it is one larger). - ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=3; -SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; ---source include/start_slave.inc - ---connection server_1 -SET @old_format= @@SESSION.binlog_format; -SET binlog_format= STATEMENT; -# This debug_sync will linger on and be used to control T3 later. -INSERT INTO t1 VALUES (foo(60, - "rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", - "rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); ---save_master_pos ---connection server_2 -# Wait for the debug_sync point for T3 to be set. But let the preparation -# transaction remain hanging, so that T1 and T2 will be scheduled for the -# remaining two worker threads. -SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; - ---connection server_1 -INSERT INTO t2 VALUES (foo(60, - "rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", - "rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); ---save_master_pos - ---connection server_2 -SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; -# T1 has now done mark_start_commit(). It will later do a rollback and retry. - -# Do T2 and T3 in a single group commit. -# Use a MyISAM table for T2 and T3, so they do not trigger the -# rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event. ---connection con_temp3 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; -SET binlog_format=statement; -send INSERT INTO t1 VALUES (foo(61, - "rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", - "rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued1'; - ---connection con_temp4 -SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; -send INSERT INTO t6 VALUES (62); - ---connection server_1 -SET debug_sync='now WAIT_FOR master_queued2'; -SET debug_sync='now SIGNAL master_cont1'; - ---connection con_temp3 -REAP; ---connection con_temp4 -REAP; - ---connection server_1 -SET debug_sync='RESET'; -SET BINLOG_FORMAT= @old_format; -SELECT * FROM t2 WHERE a>=60 ORDER BY a; -SELECT * FROM t1 WHERE a>=60 ORDER BY a; -SELECT * FROM t6 WHERE a>=60 ORDER BY a; - ---connection server_2 -SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; -# T2 has now started running, but has not yet done mark_start_commit() -SET DEBUG_SYNC= "now SIGNAL t1_cont1"; -SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; -# T1 has now done unmark_start_commit() in preparation for its retry. - ---connection server_2 -# Let the preparation transaction complete, so that the same worker thread -# can continue with the transaction T3. -SET DEBUG_SYNC= "now SIGNAL prep_cont"; -SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; -# T3 has now gone to wait for T2 to start committing -SET DEBUG_SYNC= "now SIGNAL t2_cont1"; -SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; -# T2 has now done mark_start_commit(). -# Let things run, and check that T3 does not get deadlocked. -SET DEBUG_SYNC= "now SIGNAL t1_cont2"; ---sync_with_master - ---connection server_1 ---save_master_pos ---connection server_2 ---sync_with_master -SELECT * FROM t2 WHERE a>=60 ORDER BY a; -SELECT * FROM t1 WHERE a>=60 ORDER BY a; -SELECT * FROM t6 WHERE a>=60 ORDER BY a; -SET DEBUG_SYNC="reset"; - -# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; -SET GLOBAL slave_parallel_threads=0; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - ---echo *** MDEV-7335: Potential parallel slave deadlock with specific binlog corruption *** - ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=1; -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,slave_discard_xid_for_gtid_0_x_1000"; - ---connection server_1 -INSERT INTO t2 VALUES (101); -INSERT INTO t2 VALUES (102); -INSERT INTO t2 VALUES (103); -INSERT INTO t2 VALUES (104); -INSERT INTO t2 VALUES (105); -# Inject a partial event group (missing XID at the end). The bug was that such -# partial group was not handled appropriately, leading to server deadlock. -SET gtid_seq_no=1000; -INSERT INTO t2 VALUES (106); -INSERT INTO t2 VALUES (107); -INSERT INTO t2 VALUES (108); -INSERT INTO t2 VALUES (109); -INSERT INTO t2 VALUES (110); -INSERT INTO t2 VALUES (111); -INSERT INTO t2 VALUES (112); -INSERT INTO t2 VALUES (113); -INSERT INTO t2 VALUES (114); -INSERT INTO t2 VALUES (115); -INSERT INTO t2 VALUES (116); -INSERT INTO t2 VALUES (117); -INSERT INTO t2 VALUES (118); -INSERT INTO t2 VALUES (119); -INSERT INTO t2 VALUES (120); -INSERT INTO t2 VALUES (121); -INSERT INTO t2 VALUES (122); -INSERT INTO t2 VALUES (123); -INSERT INTO t2 VALUES (124); -INSERT INTO t2 VALUES (125); -INSERT INTO t2 VALUES (126); -INSERT INTO t2 VALUES (127); -INSERT INTO t2 VALUES (128); -INSERT INTO t2 VALUES (129); -INSERT INTO t2 VALUES (130); ---source include/save_master_gtid.inc - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc -# The partial event group (a=106) should be rolled back and thus missing. -SELECT * FROM t2 WHERE a >= 100 ORDER BY a; - ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - ---echo *** MDEV-6676 - test syntax of @@slave_parallel_mode *** ---connection server_2 - ---let $status_items= Parallel_Mode ---source include/show_slave_status.inc ---source include/stop_slave.inc -SET GLOBAL slave_parallel_mode='aggressive'; ---let $status_items= Parallel_Mode ---source include/show_slave_status.inc -SET GLOBAL slave_parallel_mode='conservative'; ---let $status_items= Parallel_Mode ---source include/show_slave_status.inc - - ---echo *** MDEV-6676 - test that empty parallel_mode does not replicate in parallel *** ---connection server_1 -INSERT INTO t2 VALUES (1040); ---source include/save_master_gtid.inc - ---connection server_2 -SET GLOBAL slave_parallel_mode='none'; -# Test that we do not use parallel apply, by injecting an unconditional -# crash in the parallel apply code. -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,slave_crash_if_parallel_apply"; ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc -SELECT * FROM t2 WHERE a >= 1040 ORDER BY a; ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; - - ---echo *** MDEV-6676 - test disabling domain-based parallel replication *** ---connection server_1 -# Let's do a bunch of transactions that will conflict if run out-of-order in -# domain-based parallel replication mode. -SET gtid_domain_id = 1; -INSERT INTO t2 VALUES (1041); -INSERT INTO t2 VALUES (1042); -INSERT INTO t2 VALUES (1043); -INSERT INTO t2 VALUES (1044); -INSERT INTO t2 VALUES (1045); -INSERT INTO t2 VALUES (1046); -DELETE FROM t2 WHERE a >= 1041; -SET gtid_domain_id = 2; -INSERT INTO t2 VALUES (1041); -INSERT INTO t2 VALUES (1042); -INSERT INTO t2 VALUES (1043); -INSERT INTO t2 VALUES (1044); -INSERT INTO t2 VALUES (1045); -INSERT INTO t2 VALUES (1046); -SET gtid_domain_id = 0; ---source include/save_master_gtid.inc ---connection server_2 -SET GLOBAL slave_parallel_mode=minimal; ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc -SELECT * FROM t2 WHERE a >= 1040 ORDER BY a; ---source include/stop_slave.inc -SET GLOBAL slave_parallel_mode='conservative'; ---source include/start_slave.inc - - ---echo *** MDEV-7847: "Slave worker thread retried transaction 10 time(s) in vain, giving up", followed by replication hanging *** ---echo *** MDEV-7882: Excessive transaction retry in parallel replication *** - ---connection server_1 -CREATE TABLE t7 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; -CREATE TABLE t8 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; ---save_master_pos - ---connection server_2 ---sync_with_master ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=40; -SELECT @old_retries:=@@GLOBAL.slave_transaction_retries; -SET GLOBAL slave_transaction_retries= 5; - - -# Using dbug error injection, we artificially create event groups with a lot of -# conflicting transactions in each event group. The bugs were originally seen -# "in the wild" with transactions that did not conflict on the master, and only -# conflicted very rarely on the slave (maybe some edge case with InnoDB btree -# page splits or something like that). The event groups here loosely reflect -# the structure of the original failure's group commits. - - ---connection server_1 -INSERT INTO t7 VALUES (1,1), (2,2), (3,3), (4,4), (5,5); -SET @old_dbug= @@SESSION.debug_dbug; -SET @commit_id= 42; -SET SESSION debug_dbug="+d,binlog_force_commit_id"; -INSERT INTO t8 VALUES (1,1); -INSERT INTO t8 VALUES (2,2); -INSERT INTO t8 VALUES (3,3); -INSERT INTO t8 VALUES (4,4); -INSERT INTO t8 VALUES (5,5); -INSERT INTO t8 VALUES (6,6); -INSERT INTO t8 VALUES (7,7); -INSERT INTO t8 VALUES (8,8); - -UPDATE t7 SET b=9 WHERE a=3; -UPDATE t7 SET b=10 WHERE a=3; -UPDATE t7 SET b=11 WHERE a=3; - -INSERT INTO t8 VALUES (12,12); -INSERT INTO t8 VALUES (13,13); - -UPDATE t7 SET b=14 WHERE a=3; -UPDATE t7 SET b=15 WHERE a=3; - -INSERT INTO t8 VALUES (16,16); - -UPDATE t7 SET b=17 WHERE a=3; - -INSERT INTO t8 VALUES (18,18); -INSERT INTO t8 VALUES (19,19); - -UPDATE t7 SET b=20 WHERE a=3; - -INSERT INTO t8 VALUES (21,21); - -UPDATE t7 SET b=22 WHERE a=3; - -INSERT INTO t8 VALUES (23,24); -INSERT INTO t8 VALUES (24,24); - -UPDATE t7 SET b=25 WHERE a=3; - -INSERT INTO t8 VALUES (26,26); - -UPDATE t7 SET b=27 WHERE a=3; - -BEGIN; -INSERT INTO t8 VALUES (28,28); -INSERT INTO t8 VALUES (29,28), (30,28); -INSERT INTO t8 VALUES (31,28); -INSERT INTO t8 VALUES (32,28); -INSERT INTO t8 VALUES (33,28); -INSERT INTO t8 VALUES (34,28); -INSERT INTO t8 VALUES (35,28); -INSERT INTO t8 VALUES (36,28); -INSERT INTO t8 VALUES (37,28); -INSERT INTO t8 VALUES (38,28); -INSERT INTO t8 VALUES (39,28); -INSERT INTO t8 VALUES (40,28); -INSERT INTO t8 VALUES (41,28); -INSERT INTO t8 VALUES (42,28); -COMMIT; - - -SET @commit_id=43; -INSERT INTO t8 VALUES (43,43); -INSERT INTO t8 VALUES (44,44); - -UPDATE t7 SET b=45 WHERE a=3; - -INSERT INTO t8 VALUES (46,46); -INSERT INTO t8 VALUES (47,47); - -UPDATE t7 SET b=48 WHERE a=3; - -INSERT INTO t8 VALUES (49,49); -INSERT INTO t8 VALUES (50,50); - - -SET @commit_id=44; -INSERT INTO t8 VALUES (51,51); -INSERT INTO t8 VALUES (52,52); - -UPDATE t7 SET b=53 WHERE a=3; - -INSERT INTO t8 VALUES (54,54); -INSERT INTO t8 VALUES (55,55); - -UPDATE t7 SET b=56 WHERE a=3; - -INSERT INTO t8 VALUES (57,57); - -UPDATE t7 SET b=58 WHERE a=3; - -INSERT INTO t8 VALUES (58,58); -INSERT INTO t8 VALUES (59,59); -INSERT INTO t8 VALUES (60,60); -INSERT INTO t8 VALUES (61,61); - -UPDATE t7 SET b=62 WHERE a=3; - -INSERT INTO t8 VALUES (63,63); -INSERT INTO t8 VALUES (64,64); -INSERT INTO t8 VALUES (65,65); -INSERT INTO t8 VALUES (66,66); - -UPDATE t7 SET b=67 WHERE a=3; - -INSERT INTO t8 VALUES (68,68); - -UPDATE t7 SET b=69 WHERE a=3; -UPDATE t7 SET b=70 WHERE a=3; -UPDATE t7 SET b=71 WHERE a=3; - -INSERT INTO t8 VALUES (72,72); - -UPDATE t7 SET b=73 WHERE a=3; -UPDATE t7 SET b=74 WHERE a=3; -UPDATE t7 SET b=75 WHERE a=3; -UPDATE t7 SET b=76 WHERE a=3; - -INSERT INTO t8 VALUES (77,77); - -UPDATE t7 SET b=78 WHERE a=3; - -INSERT INTO t8 VALUES (79,79); - -UPDATE t7 SET b=80 WHERE a=3; - -INSERT INTO t8 VALUES (81,81); - -UPDATE t7 SET b=82 WHERE a=3; - -INSERT INTO t8 VALUES (83,83); - -UPDATE t7 SET b=84 WHERE a=3; - - -SET @commit_id=45; -INSERT INTO t8 VALUES (85,85); -UPDATE t7 SET b=86 WHERE a=3; -INSERT INTO t8 VALUES (87,87); - - -SET @commit_id=46; -INSERT INTO t8 VALUES (88,88); -INSERT INTO t8 VALUES (89,89); -INSERT INTO t8 VALUES (90,90); - -SET SESSION debug_dbug=@old_dbug; - -INSERT INTO t8 VALUES (91,91); -INSERT INTO t8 VALUES (92,92); -INSERT INTO t8 VALUES (93,93); -INSERT INTO t8 VALUES (94,94); -INSERT INTO t8 VALUES (95,95); -INSERT INTO t8 VALUES (96,96); -INSERT INTO t8 VALUES (97,97); -INSERT INTO t8 VALUES (98,98); -INSERT INTO t8 VALUES (99,99); - - -SELECT * FROM t7 ORDER BY a; -SELECT * FROM t8 ORDER BY a; ---source include/save_master_gtid.inc - - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc -SELECT * FROM t7 ORDER BY a; -SELECT * FROM t8 ORDER BY a; - ---source include/stop_slave.inc -SET GLOBAL slave_transaction_retries= @old_retries; -SET GLOBAL slave_parallel_threads=10; ---source include/start_slave.inc - - ---echo *** MDEV-7888: ANALYZE TABLE does wakeup_subsequent_commits(), causing wrong binlog order and parallel replication hang *** - ---connection server_2 ---source include/stop_slave.inc -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug= '+d,inject_analyze_table_sleep'; - ---connection server_1 -# Inject two group commits. The bug was that ANALYZE TABLE would call -# wakeup_subsequent_commits() too early, allowing the following transaction -# in the same group to run ahead and binlog and free the GCO. Then we get -# wrong binlog order and later access freed GCO, which causes lost wakeup -# of following GCO and thus replication hang. -# We injected a small sleep in ANALYZE to make the race easier to hit (this -# can only cause false negatives in versions with the bug, not false positives, -# so sleep is ok here. And it's in general not possible to trigger reliably -# the race with debug_sync, since the bugfix makes the race impossible). - -SET @old_dbug= @@SESSION.debug_dbug; -SET SESSION debug_dbug="+d,binlog_force_commit_id"; - -# Group commit with cid=10000, two event groups. -SET @commit_id= 10000; -ANALYZE TABLE t2; -INSERT INTO t3 VALUES (120, 0); - -# Group commit with cid=10001, one event group. -SET @commit_id= 10001; -INSERT INTO t3 VALUES (121, 0); - -SET SESSION debug_dbug=@old_dbug; - -SELECT * FROM t3 WHERE a >= 120 ORDER BY a; ---source include/save_master_gtid.inc - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc - -SELECT * FROM t3 WHERE a >= 120 ORDER BY a; - ---source include/stop_slave.inc -SET GLOBAL debug_dbug= @old_dbug; ---source include/start_slave.inc - - ---echo *** MDEV-7929: record_gtid() for non-transactional event group calls wakeup_subsequent_commits() too early, causing slave hang. *** - ---connection server_2 ---source include/stop_slave.inc -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug= '+d,inject_record_gtid_serverid_100_sleep'; - ---connection server_1 -# Inject two group commits. The bug was that record_gtid for a -# non-transactional event group would commit its own transaction, which would -# cause ha_commit_trans() to call wakeup_subsequent_commits() too early. This -# in turn lead to access to freed group_commit_orderer object, losing a wakeup -# and causing slave threads to hang. -# We inject a small sleep in the corresponding record_gtid() to make the race -# easier to hit. - -SET @old_dbug= @@SESSION.debug_dbug; -SET SESSION debug_dbug="+d,binlog_force_commit_id"; - -# Group commit with cid=10010, two event groups. -SET @old_server_id= @@SESSION.server_id; -SET SESSION server_id= 100; -SET @commit_id= 10010; -ALTER TABLE t1 COMMENT "Hulubulu!"; -SET SESSION server_id= @old_server_id; -INSERT INTO t3 VALUES (130, 0); - -# Group commit with cid=10011, one event group. -SET @commit_id= 10011; -INSERT INTO t3 VALUES (131, 0); - -SET SESSION debug_dbug=@old_dbug; - -SELECT * FROM t3 WHERE a >= 130 ORDER BY a; ---source include/save_master_gtid.inc - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc - -SELECT * FROM t3 WHERE a >= 130 ORDER BY a; - ---source include/stop_slave.inc -SET GLOBAL debug_dbug= @old_dbug; ---source include/start_slave.inc - - ---echo *** MDEV-8031: Parallel replication stops on "connection killed" error (probably incorrectly handled deadlock kill) *** - ---connection server_1 -INSERT INTO t3 VALUES (201,0), (202,0); ---source include/save_master_gtid.inc - ---connection server_2 ---source include/sync_with_master_gtid.inc ---source include/stop_slave.inc -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug= '+d,inject_mdev8031'; - ---connection server_1 -# We artificially create a situation that hopefully resembles the original -# bug which was only seen "in the wild", and only once. -# Setup a fake group commit with lots of conflicts that will lead to deadloc -# kill. The slave DBUG injection causes the slave to be deadlock killed at -# a particular point during the retry, and then later do a small sleep at -# another critical point where the prior transaction then has a chance to -# complete. Finally an extra KILL check catches an unhandled, lingering -# deadlock kill. So rather artificial, but at least it exercises the -# relevant code paths. -SET @old_dbug= @@SESSION.debug_dbug; -SET SESSION debug_dbug="+d,binlog_force_commit_id"; - -SET @commit_id= 10200; -INSERT INTO t3 VALUES (203, 1); -INSERT INTO t3 VALUES (204, 1); -INSERT INTO t3 VALUES (205, 1); -UPDATE t3 SET b=b+1 WHERE a=201; -UPDATE t3 SET b=b+1 WHERE a=201; -UPDATE t3 SET b=b+1 WHERE a=201; -UPDATE t3 SET b=b+1 WHERE a=202; -UPDATE t3 SET b=b+1 WHERE a=202; -UPDATE t3 SET b=b+1 WHERE a=202; -UPDATE t3 SET b=b+1 WHERE a=202; -UPDATE t3 SET b=b+1 WHERE a=203; -UPDATE t3 SET b=b+1 WHERE a=203; -UPDATE t3 SET b=b+1 WHERE a=204; -UPDATE t3 SET b=b+1 WHERE a=204; -UPDATE t3 SET b=b+1 WHERE a=204; -UPDATE t3 SET b=b+1 WHERE a=203; -UPDATE t3 SET b=b+1 WHERE a=205; -UPDATE t3 SET b=b+1 WHERE a=205; -SET SESSION debug_dbug=@old_dbug; - -SELECT * FROM t3 WHERE a>=200 ORDER BY a; ---source include/save_master_gtid.inc - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc - -SELECT * FROM t3 WHERE a>=200 ORDER BY a; ---source include/stop_slave.inc -SET GLOBAL debug_dbug= @old_dbug; ---source include/start_slave.inc - - ---echo *** Check getting deadlock killed inside open_binlog() during retry. *** - ---connection server_2 ---source include/stop_slave.inc -SET @old_dbug= @@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug= '+d,inject_retry_event_group_open_binlog_kill'; -SET @old_max= @@GLOBAL.max_relay_log_size; -SET GLOBAL max_relay_log_size= 4096; - ---connection server_1 -SET @old_dbug= @@SESSION.debug_dbug; -SET SESSION debug_dbug="+d,binlog_force_commit_id"; - ---let $large= `SELECT REPEAT("*", 8192)` -SET @commit_id= 10210; ---echo Omit long queries that cause relaylog rotations and transaction retries... ---disable_query_log -eval UPDATE t3 SET b=b+1 WHERE a=201 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=201 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=201 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=202 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=203 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=203 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=204 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=204 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=204 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=203 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=205 /* $large */; -eval UPDATE t3 SET b=b+1 WHERE a=205 /* $large */; ---enable_query_log -SET SESSION debug_dbug=@old_dbug; - -SELECT * FROM t3 WHERE a>=200 ORDER BY a; ---source include/save_master_gtid.inc - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc - -SELECT * FROM t3 WHERE a>=200 ORDER BY a; ---source include/stop_slave.inc -SET GLOBAL debug_dbug= @old_debg; -SET GLOBAL max_relay_log_size= @old_max; ---source include/start_slave.inc - - ---echo *** MDEV-8302: Duplicate key with parallel replication *** - ---connection server_2 ---source include/stop_slave.inc -/* Inject a small sleep which makes the race easier to hit. */ -SET @old_dbug=@@GLOBAL.debug_dbug; -SET GLOBAL debug_dbug="+d,inject_mdev8302"; - - ---connection server_1 -INSERT INTO t7 VALUES (100,1), (101,2), (102,3), (103,4), (104,5); - -# Artificially create a bunch of group commits with conflicting transactions. -# The bug happened when T1 and T2 was in one group commit, and T3 was in the -# following group commit. T2 is a DELETE of a row with same primary key as a -# row that T3 inserts. T1 and T2 can conflict, causing T2 to be deadlock -# killed after starting to commit. The bug was that T2 could roll back before -# doing unmark_start_commit(); this could allow T3 to run before the retry -# of T2, causing duplicate key violation. - -SET @old_dbug= @@SESSION.debug_dbug; -SET @commit_id= 20000; -SET SESSION debug_dbug="+d,binlog_force_commit_id"; - ---let $n = 100 ---disable_query_log -while ($n) -{ - eval UPDATE t7 SET b=b+1 WHERE a=100+($n MOD 5); - eval DELETE FROM t7 WHERE a=100+($n MOD 5); - - SET @commit_id = @commit_id + 1; - eval INSERT INTO t7 VALUES (100+($n MOD 5), $n); - SET @commit_id = @commit_id + 1; - dec $n; -} ---enable_query_log -SET SESSION debug_dbug=@old_dbug; - - -SELECT * FROM t7 ORDER BY a; ---source include/save_master_gtid.inc - - ---connection server_2 ---source include/start_slave.inc ---source include/sync_with_master_gtid.inc -SELECT * FROM t7 ORDER BY a; - ---source include/stop_slave.inc -SET GLOBAL debug_dbug=@old_dbug; ---source include/start_slave.inc - - - ---echo *** MDEV-8725: Assertion on ROLLBACK statement in the binary log *** ---connection server_1 -# Inject an event group terminated by ROLLBACK, by mixing MyISAM and InnoDB -# in a transaction. The bug was an assertion on the ROLLBACK due to -# mark_start_commit() being already called. ---disable_warnings -BEGIN; -INSERT INTO t2 VALUES (2000); -INSERT INTO t1 VALUES (2000); -INSERT INTO t2 VALUES (2001); -ROLLBACK; ---enable_warnings -SELECT * FROM t1 WHERE a>=2000 ORDER BY a; -SELECT * FROM t2 WHERE a>=2000 ORDER BY a; ---source include/save_master_gtid.inc - ---connection server_2 ---source include/sync_with_master_gtid.inc -SELECT * FROM t1 WHERE a>=2000 ORDER BY a; -SELECT * FROM t2 WHERE a>=2000 ORDER BY a; - - -# Clean up. ---connection server_2 ---source include/stop_slave.inc -SET GLOBAL slave_parallel_threads=@old_parallel_threads; ---source include/start_slave.inc -SET DEBUG_SYNC= 'RESET'; - ---connection server_1 -DROP function foo; -DROP TABLE t1,t2,t3,t4,t5,t6,t7,t8; -SET DEBUG_SYNC= 'RESET'; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_parallel.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test b/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test index 83d847318d86c..9e93b0b56e9bc 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_show_binlog_events_purge_logs.test @@ -1,34 +1 @@ -# BUG#13979418: SHOW BINLOG EVENTS MAY CRASH THE SERVER -# -# The function mysql_show_binlog_events has a local stack variable -# 'LOG_INFO linfo;', which is assigned to thd->current_linfo, however -# this variable goes out of scope and is destroyed before clean -# thd->current_linfo. -# -# This test case runs SHOW BINLOG EVENTS and FLUSH LOGS to make sure -# that with the fix local variable linfo is valid along all -# mysql_show_binlog_events function scope. -# ---source include/have_debug_sync.inc ---source include/master-slave.inc - ---echo [connection slave] ---connection slave -SET DEBUG_SYNC= 'after_show_binlog_events SIGNAL on_show_binlog_events WAIT_FOR end'; ---send SHOW BINLOG EVENTS - ---connection slave1 ---echo [connection slave1] -SET DEBUG_SYNC= 'now WAIT_FOR on_show_binlog_events'; -FLUSH LOGS; -SET DEBUG_SYNC= 'now SIGNAL end'; - ---echo [connection slave] ---connection slave ---disable_result_log ---reap ---enable_result_log -SET DEBUG_SYNC= 'RESET'; - ---connection master ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_parallel_show_binlog_events_purge_logs.inc diff --git a/mysql-test/suite/rpl/t/rpl_relayrotate.test b/mysql-test/suite/rpl/t/rpl_relayrotate.test index 4c0840446ecf1..5e3bcdcd711f0 100644 --- a/mysql-test/suite/rpl/t/rpl_relayrotate.test +++ b/mysql-test/suite/rpl/t/rpl_relayrotate.test @@ -1,12 +1 @@ -####################################################### -# Wrapper for rpl_relayrotate.test to allow multi # -# Engines to reuse test code. By JBM 2006-02-15 # -####################################################### --- source include/have_innodb.inc -# Slow test, don't run during staging part --- source include/not_staging.inc --- source include/master-slave.inc - -let $engine_type=innodb; --- source extra/rpl_tests/rpl_relayrotate.test ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_relayrotate.inc diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync.test b/mysql-test/suite/rpl/t/rpl_semi_sync.test index a9ea498503978..d5f80619aeb43 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync.test @@ -1,574 +1 @@ -source include/have_semisync.inc; -source include/not_embedded.inc; -source include/have_innodb.inc; -source include/master-slave.inc; - -let $engine_type= InnoDB; -#let $engine_type= MyISAM; - -# Suppress warnings that might be generated during the test -connection master; -call mtr.add_suppression("Timeout waiting for reply of binlog"); -call mtr.add_suppression("Read semi-sync reply"); -call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); -connection slave; -call mtr.add_suppression("Master server does not support semi-sync"); -call mtr.add_suppression("Semi-sync slave .* reply"); -call mtr.add_suppression("Slave SQL.*Request to stop slave SQL Thread received while applying a group that has non-transactional changes; waiting for completion of the group"); -connection master; - -# wait for dying connections (if any) to disappear -let $wait_condition= select count(*) = 0 from information_schema.processlist where command='killed'; ---source include/wait_condition.inc - -# After fix of BUG#45848, semi-sync slave should not create any extra -# connections on master, save the count of connections before start -# semi-sync slave for comparison below. -let $_connections_normal_slave= query_get_value(SHOW STATUS LIKE 'Threads_connected', Value, 1); - ---echo # ---echo # Uninstall semi-sync plugins on master and slave ---echo # -connection slave; -source include/stop_slave.inc; -reset slave; -set global rpl_semi_sync_master_enabled= 0; -set global rpl_semi_sync_slave_enabled= 0; - -connection master; -reset master; -set global rpl_semi_sync_master_enabled= 0; -set global rpl_semi_sync_slave_enabled= 0; - ---echo # ---echo # Main test of semi-sync replication start here ---echo # - -connection master; -echo [ on master ]; - -set global rpl_semi_sync_master_timeout= 60000; # 60s - -echo [ default state of semi-sync on master should be OFF ]; -show variables like 'rpl_semi_sync_master_enabled'; - -echo [ enable semi-sync on master ]; -set global rpl_semi_sync_master_enabled = 1; -show variables like 'rpl_semi_sync_master_enabled'; - -echo [ status of semi-sync on master should be ON even without any semi-sync slaves ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_yes_tx'; - ---echo # ---echo # BUG#45672 Semisync repl: ActiveTranx:insert_tranx_node: transaction node allocation failed ---echo # BUG#45673 Semisynch reports correct operation even if no slave is connected ---echo # - -# BUG#45672 When semi-sync is enabled on master, it would allocate -# transaction node even without semi-sync slave connected, and would -# finally result in transaction node allocation error. -# -# Semi-sync master will pre-allocate 'max_connections' transaction -# nodes, so here we do more than that much transactions to check if it -# will fail or not. -# select @@global.max_connections + 1; -let $i= `select @@global.max_connections + 1`; -disable_query_log; -eval create table t1 (a int) engine=$engine_type; -while ($i) -{ - eval insert into t1 values ($i); - dec $i; -} -drop table t1; -enable_query_log; - -# BUG#45673 -echo [ status of semi-sync on master should be OFF ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; ---replace_result 305 304 -show status like 'Rpl_semi_sync_master_yes_tx'; - -# reset master to make sure the following test will start with a clean environment -reset master; - -connection slave; -echo [ on slave ]; - -echo [ default state of semi-sync on slave should be OFF ]; -show variables like 'rpl_semi_sync_slave_enabled'; - -echo [ enable semi-sync on slave ]; -set global rpl_semi_sync_slave_enabled = 1; -show variables like 'rpl_semi_sync_slave_enabled'; -source include/start_slave.inc; - -connection master; -echo [ on master ]; - -# NOTE: Rpl_semi_sync_master_client will only be updated when -# semi-sync slave has started binlog dump request -let $status_var= Rpl_semi_sync_master_clients; -let $status_var_value= 1; -source include/wait_for_status_var.inc; - -echo [ initial master state after the semi-sync slave connected ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; - -replace_result $engine_type ENGINE_TYPE; -eval create table t1(a int) engine = $engine_type; - -echo [ master state after CREATE TABLE statement ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; - -# After fix of BUG#45848, semi-sync slave should not create any extra -# connections on master. -let $_connections_semisync_slave= query_get_value(SHOW STATUS LIKE 'Threads_connected', Value, 1); -replace_result $_connections_normal_slave CONNECTIONS_NORMAL_SLAVE $_connections_semisync_slave CONNECTIONS_SEMISYNC_SLAVE; -eval select $_connections_semisync_slave - $_connections_normal_slave as 'Should be 0'; - -echo [ insert records to table ]; -insert t1 values (10); -insert t1 values (9); -insert t1 values (8); -insert t1 values (7); -insert t1 values (6); -insert t1 values (5); -insert t1 values (4); -insert t1 values (3); -insert t1 values (2); -insert t1 values (1); - -echo [ master status after inserts ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; - -sync_slave_with_master; -echo [ on slave ]; - -echo [ slave status after replicated inserts ]; -show status like 'Rpl_semi_sync_slave_status'; - -select count(distinct a) from t1; -select min(a) from t1; -select max(a) from t1; - ---echo ---echo # BUG#50157 ---echo # semi-sync replication crashes when replicating a transaction which ---echo # include 'CREATE TEMPORARY TABLE `MyISAM_t` SELECT * FROM `Innodb_t` ; - -connection master; -echo [ on master ]; -SET SESSION AUTOCOMMIT= 0; -CREATE TABLE t2(c1 INT) ENGINE=innodb; -sync_slave_with_master; - -connection master; -BEGIN; ---echo ---echo # Even though it is in a transaction, this statement is binlogged into binlog ---echo # file immediately. ---disable_warnings -CREATE TEMPORARY TABLE t3 SELECT c1 FROM t2 where 1=1; ---enable_warnings ---echo ---echo # These statements will not be binlogged until the transaction is committed -INSERT INTO t2 VALUES(11); -INSERT INTO t2 VALUES(22); -COMMIT; - -DROP TABLE t2, t3; -SET SESSION AUTOCOMMIT= 1; -sync_slave_with_master; - - ---echo # ---echo # Test semi-sync master will switch OFF after one transaction ---echo # timeout waiting for slave reply. ---echo # -connection slave; -source include/stop_slave.inc; - -connection master; -echo [ on master ]; -set global rpl_semi_sync_master_timeout= 5000; - -# The first semi-sync check should be on because after slave stop, -# there are no transactions on the master. -echo [ master status should be ON ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; ---replace_result 305 304 -show status like 'Rpl_semi_sync_master_yes_tx'; -show status like 'Rpl_semi_sync_master_clients'; - -echo [ semi-sync replication of these transactions will fail ]; -insert into t1 values (500); - -# Wait for the semi-sync replication of this transaction to timeout -let $status_var= Rpl_semi_sync_master_status; -let $status_var_value= OFF; -source include/wait_for_status_var.inc; - -# The second semi-sync check should be off because one transaction -# times out during waiting. -echo [ master status should be OFF ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; ---replace_result 305 304 -show status like 'Rpl_semi_sync_master_yes_tx'; - -# Semi-sync status on master is now OFF, so all these transactions -# will be replicated asynchronously. -delete from t1 where a=10; -delete from t1 where a=9; -delete from t1 where a=8; -delete from t1 where a=7; -delete from t1 where a=6; -delete from t1 where a=5; -delete from t1 where a=4; -delete from t1 where a=3; -delete from t1 where a=2; -delete from t1 where a=1; - -insert into t1 values (100); - -echo [ master status should be OFF ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; ---replace_result 305 304 -show status like 'Rpl_semi_sync_master_yes_tx'; - ---echo # ---echo # Test semi-sync status on master will be ON again when slave catches up ---echo # - -# Save the master position for later use. -save_master_pos; - -connection slave; -echo [ on slave ]; - -echo [ slave status should be OFF ]; -show status like 'Rpl_semi_sync_slave_status'; -source include/start_slave.inc; -sync_with_master; - -echo [ slave status should be ON ]; -show status like 'Rpl_semi_sync_slave_status'; - -select count(distinct a) from t1; -select min(a) from t1; -select max(a) from t1; - -connection master; -echo [ on master ]; - -# The master semi-sync status should be on again after slave catches up. -echo [ master status should be ON again after slave catches up ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; ---replace_result 305 304 -show status like 'Rpl_semi_sync_master_yes_tx'; -show status like 'Rpl_semi_sync_master_clients'; - ---echo # ---echo # Test disable/enable master semi-sync on the fly. ---echo # - -drop table t1; -sync_slave_with_master; -echo [ on slave ]; - -source include/stop_slave.inc; - ---echo # ---echo # Flush status ---echo # -connection master; -echo [ Semi-sync master status variables before FLUSH STATUS ]; -SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; -SHOW STATUS LIKE 'Rpl_semi_sync_master_yes_tx'; -# Do not write the FLUSH STATUS to binlog, to make sure we'll get a -# clean status after this. -FLUSH NO_WRITE_TO_BINLOG STATUS; -echo [ Semi-sync master status variables after FLUSH STATUS ]; -SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; -SHOW STATUS LIKE 'Rpl_semi_sync_master_yes_tx'; - -connection master; -echo [ on master ]; - -source include/show_master_logs.inc; -show variables like 'rpl_semi_sync_master_enabled'; - -echo [ disable semi-sync on the fly ]; -set global rpl_semi_sync_master_enabled=0; -show variables like 'rpl_semi_sync_master_enabled'; -show status like 'Rpl_semi_sync_master_status'; - -echo [ enable semi-sync on the fly ]; -set global rpl_semi_sync_master_enabled=1; -show variables like 'rpl_semi_sync_master_enabled'; -show status like 'Rpl_semi_sync_master_status'; - ---echo # ---echo # Test RESET MASTER/SLAVE ---echo # - -connection slave; -echo [ on slave ]; - -source include/start_slave.inc; - -connection master; -echo [ on master ]; - -replace_result $engine_type ENGINE_TYPE; -eval create table t1 (a int) engine = $engine_type; -drop table t1; - -##show status like 'Rpl_semi_sync_master_status'; - -sync_slave_with_master; ---replace_column 2 # -show status like 'Rpl_relay%'; - -echo [ test reset master ]; -connection master; -echo [ on master]; - -reset master; - -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; - -connection slave; -echo [ on slave ]; - -source include/stop_slave.inc; -reset slave; - -# Kill the dump thread on master for previous slave connection and -# wait for it to exit -connection master; -let $_tid= `select id from information_schema.processlist where command = 'Binlog Dump' limit 1`; -if ($_tid) -{ - --replace_result $_tid _tid - eval kill query $_tid; - - # After dump thread exit, Rpl_semi_sync_master_clients will be 0 - let $status_var= Rpl_semi_sync_master_clients; - let $status_var_value= 0; - source include/wait_for_status_var.inc; -} - -connection slave; -source include/start_slave.inc; - -connection master; -echo [ on master ]; - -# Wait for dump thread to start, Rpl_semi_sync_master_clients will be -# 1 after dump thread started. -let $status_var= Rpl_semi_sync_master_clients; -let $status_var_value= 1; -source include/wait_for_status_var.inc; - -replace_result $engine_type ENGINE_TYPE; -eval create table t1 (a int) engine = $engine_type; -insert into t1 values (1); -insert into t1 values (2), (3); - -sync_slave_with_master; -echo [ on slave ]; - -select * from t1; - -connection master; -echo [ on master ]; - -echo [ master semi-sync status should be ON ]; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; - ---echo # ---echo # Start semi-sync replication without SUPER privilege ---echo # -connection slave; -source include/stop_slave.inc; -reset slave; -connection master; -echo [ on master ]; -reset master; - -# Kill the dump thread on master for previous slave connection and wait for it to exit -let $_tid= `select id from information_schema.processlist where command = 'Binlog Dump' limit 1`; -if ($_tid) -{ - --replace_result $_tid _tid - eval kill query $_tid; - - # After dump thread exit, Rpl_semi_sync_master_clients will be 0 - let $status_var= Rpl_semi_sync_master_clients; - let $status_var_value= 0; - source include/wait_for_status_var.inc; -} - -# Do not binlog the following statement because it will generate -# different events for ROW and STATEMENT format -set sql_log_bin=0; -grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; -flush privileges; -set sql_log_bin=1; -connection slave; -echo [ on slave ]; -grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; -flush privileges; -change master to master_user='rpl',master_password='rpl_password'; -source include/start_slave.inc; -show status like 'Rpl_semi_sync_slave_status'; -connection master; -echo [ on master ]; - -# Wait for the semi-sync binlog dump thread to start -let $status_var= Rpl_semi_sync_master_clients; -let $status_var_value= 1; -source include/wait_for_status_var.inc; -echo [ master semi-sync should be ON ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; -insert into t1 values (4); -insert into t1 values (5); -echo [ master semi-sync should be ON ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; -show status like 'Rpl_semi_sync_master_no_tx'; -show status like 'Rpl_semi_sync_master_yes_tx'; - ---echo # ---echo # Test semi-sync slave connect to non-semi-sync master ---echo # - -# Disable semi-sync on master -connection slave; -echo [ on slave ]; -source include/stop_slave.inc; -SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; - -connection master; -echo [ on master ]; - -# Kill the dump thread on master for previous slave connection and wait for it to exit -let $_tid= `select id from information_schema.processlist where command = 'Binlog Dump' limit 1`; -if ($_tid) -{ - --replace_result $_tid _tid - eval kill query $_tid; - - # After dump thread exit, Rpl_semi_sync_master_clients will be 0 - let $status_var= Rpl_semi_sync_master_clients; - let $status_var_value= 0; - source include/wait_for_status_var.inc; -} - -echo [ Semi-sync status on master should be ON ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; -set global rpl_semi_sync_master_enabled= 0; - -connection slave; -echo [ on slave ]; -SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; -source include/start_slave.inc; -connection master; -echo [ on master ]; -insert into t1 values (8); -let $status_var= Rpl_semi_sync_master_clients; -let $status_var_value= 1; -source include/wait_for_status_var.inc; -echo [ master semi-sync clients should be 1, status should be OFF ]; -show status like 'Rpl_semi_sync_master_clients'; -show status like 'Rpl_semi_sync_master_status'; -sync_slave_with_master; -echo [ on slave ]; -show status like 'Rpl_semi_sync_slave_status'; - -# Uninstall semi-sync plugin on master -connection slave; -source include/stop_slave.inc; -connection master; -echo [ on master ]; -set global rpl_semi_sync_master_enabled= 0; - -connection slave; -echo [ on slave ]; -SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; -source include/start_slave.inc; - -connection master; -echo [ on master ]; -insert into t1 values (10); -sync_slave_with_master; - ---echo # ---echo # Test non-semi-sync slave connect to semi-sync master ---echo # - -connection master; -set global rpl_semi_sync_master_timeout= 5000; # 5s -set global rpl_semi_sync_master_enabled= 1; - -connection slave; -echo [ on slave ]; -source include/stop_slave.inc; -SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; - -echo [ uninstall semi-sync slave plugin ]; -set global rpl_semi_sync_slave_enabled= 0; - -echo [ reinstall semi-sync slave plugin and disable semi-sync ]; -SHOW VARIABLES LIKE 'rpl_semi_sync_slave_enabled'; -SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; -source include/start_slave.inc; -SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; - ---echo # ---echo # Clean up ---echo # - -connection slave; -source include/stop_slave.inc; -set global rpl_semi_sync_slave_enabled= 0; - -connection master; -set global rpl_semi_sync_master_enabled= 0; - -connection slave; -change master to master_user='root',master_password=''; -source include/start_slave.inc; - -connection master; -drop table t1; -sync_slave_with_master; - -connection master; -drop user rpl@127.0.0.1; -flush privileges; -set global rpl_semi_sync_master_timeout= default; ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_semi_sync.inc diff --git a/mysql-test/suite/rpl/t/rpl_skip_replication.test b/mysql-test/suite/rpl/t/rpl_skip_replication.test index f815554d4afc4..c57256780a475 100644 --- a/mysql-test/suite/rpl/t/rpl_skip_replication.test +++ b/mysql-test/suite/rpl/t/rpl_skip_replication.test @@ -1,377 +1 @@ ---source include/master-slave.inc ---source include/have_innodb.inc - -connection slave; -# Test that SUPER is required to change @@replicate_events_marked_for_skip. -CREATE USER 'nonsuperuser'@'127.0.0.1'; -GRANT ALTER,CREATE,DELETE,DROP,EVENT,INSERT,PROCESS,REPLICATION SLAVE, - SELECT,UPDATE ON *.* TO 'nonsuperuser'@'127.0.0.1'; -connect(nonpriv, 127.0.0.1, nonsuperuser,, test, $SLAVE_MYPORT,); -connection nonpriv; ---error ER_SPECIFIC_ACCESS_DENIED_ERROR -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; -disconnect nonpriv; -connection slave; -DROP USER'nonsuperuser'@'127.0.0.1'; - -SELECT @@global.replicate_events_marked_for_skip; ---error ER_SLAVE_MUST_STOP -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; -SELECT @@global.replicate_events_marked_for_skip; -STOP SLAVE; ---error ER_GLOBAL_VARIABLE -SET SESSION replicate_events_marked_for_skip=FILTER_ON_MASTER; -SELECT @@global.replicate_events_marked_for_skip; -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; -SELECT @@global.replicate_events_marked_for_skip; -START SLAVE; - -connection master; -SELECT @@skip_replication; ---error ER_LOCAL_VARIABLE -SET GLOBAL skip_replication=1; -SELECT @@skip_replication; - -CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=myisam; -CREATE TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=innodb; -INSERT INTO t1(a) VALUES (1); -INSERT INTO t2(a) VALUES (1); - - -# Test that master-side filtering works. -SET skip_replication=1; - -CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; -INSERT INTO t1(a) VALUES (2); -INSERT INTO t2(a) VALUES (2); - -# Inject a rotate event in the binlog stream sent to slave (otherwise we will -# fail sync_slave_with_master as the last event on the master is not present -# on the slave). -FLUSH NO_WRITE_TO_BINLOG LOGS; - -sync_slave_with_master; -connection slave; -SHOW TABLES; -SELECT * FROM t1; -SELECT * FROM t2; - -connection master; -DROP TABLE t3; - -FLUSH NO_WRITE_TO_BINLOG LOGS; -sync_slave_with_master; - - -# Test that slave-side filtering works. -connection slave; -STOP SLAVE; -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; -START SLAVE; - -connection master; -SET skip_replication=1; -CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; -INSERT INTO t1(a) VALUES (3); -INSERT INTO t2(a) VALUES (3); - -# Inject a rotate event in the binlog stream sent to slave (otherwise we will -# fail sync_slave_with_master as the last event on the master is not present -# on the slave). -FLUSH NO_WRITE_TO_BINLOG LOGS; - -sync_slave_with_master; -connection slave; -SHOW TABLES; -SELECT * FROM t1; -SELECT * FROM t2; - -connection master; -DROP TABLE t3; - -FLUSH NO_WRITE_TO_BINLOG LOGS; -sync_slave_with_master; -connection slave; -STOP SLAVE; -SET GLOBAL replicate_events_marked_for_skip=REPLICATE; -START SLAVE; - - -# Test that events with @@skip_replication=1 are not filtered when filtering is -# not set on slave. -connection master; -SET skip_replication=1; -CREATE TABLE t3 (a INT PRIMARY KEY, b INT) ENGINE=myisam; -INSERT INTO t3(a) VALUES(2); -sync_slave_with_master; -connection slave; -SELECT * FROM t3; -connection master; -DROP TABLE t3; - -# -# Test that the slave will preserve the @@skip_replication flag in its -# own binlog. -# - -TRUNCATE t1; -sync_slave_with_master; -connection slave; -RESET MASTER; - -connection master; -SET skip_replication=0; -INSERT INTO t1 VALUES (1,0); -SET skip_replication=1; -INSERT INTO t1 VALUES (2,0); -SET skip_replication=0; -INSERT INTO t1 VALUES (3,0); - -sync_slave_with_master; -connection slave; -# Since slave has @@replicate_events_marked_for_skip=REPLICATE, it should have -# applied all events. -SELECT * FROM t1 ORDER by a; - -STOP SLAVE; -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; -let $SLAVE_DATADIR= `select @@datadir`; - -connection master; -TRUNCATE t1; - -# Now apply the slave binlog to the master, to check that both the slave -# and mysqlbinlog will preserve the @@skip_replication flag. ---exec $MYSQL_BINLOG $SLAVE_DATADIR/slave-bin.000001 > $MYSQLTEST_VARDIR/tmp/rpl_skip_replication.binlog ---exec $MYSQL test < $MYSQLTEST_VARDIR/tmp/rpl_skip_replication.binlog - -# The master should have all three events. -SELECT * FROM t1 ORDER by a; - -# The slave should be missing event 2, which is marked with the -# @@skip_replication flag. - -connection slave; -START SLAVE; - -connection master; -sync_slave_with_master; - -connection slave; -SELECT * FROM t1 ORDER by a; - -# -# Test that @@sql_slave_skip_counter does not count skipped @@skip_replication -# events. -# - -connection master; -TRUNCATE t1; - -sync_slave_with_master; -connection slave; -STOP SLAVE; -# We will skip two INSERTs (in addition to any skipped due to -# @@skip_replication). Since from 5.5 every statement is wrapped in -# BEGIN ... END, we need to skip 6 events for this. -SET GLOBAL sql_slave_skip_counter=6; -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; -START SLAVE; - -connection master; -# Need to fix @@binlog_format to get consistent event count. -SET @old_binlog_format= @@binlog_format; -SET binlog_format= statement; -SET skip_replication=0; -INSERT INTO t1 VALUES (1,5); -SET skip_replication=1; -INSERT INTO t1 VALUES (2,5); -SET skip_replication=0; -INSERT INTO t1 VALUES (3,5); -INSERT INTO t1 VALUES (4,5); -SET binlog_format= @old_binlog_format; - -sync_slave_with_master; -connection slave; - -# The slave should have skipped the first three inserts (number 1 and 3 due -# to @@sql_slave_skip_counter=2, number 2 due to -# @@replicate_events_marked_for_skip=FILTER_ON_SLAVE). So only number 4 -# should be left. -SELECT * FROM t1; - - -# -# Check that BINLOG statement preserves the @@skip_replication flag. -# -connection slave; -# Need row @@binlog_format for BINLOG statements containing row events. ---source include/stop_slave.inc -SET @old_slave_binlog_format= @@global.binlog_format; -SET GLOBAL binlog_format= row; ---source include/start_slave.inc - -connection master; -TRUNCATE t1; - -SET @old_binlog_format= @@binlog_format; -SET binlog_format= row; -# Format description log event. -BINLOG 'wlZOTw8BAAAA8QAAAPUAAAAAAAQANS41LjIxLU1hcmlhREItZGVidWctbG9nAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAEzgNAAgAEgAEBAQEEgAA2QAEGggAAAAICAgCAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAA371saA=='; -# INSERT INTO t1 VALUES (1,8) # with @@skip_replication=1 -BINLOG 'wlZOTxMBAAAAKgAAAGMBAAAAgCkAAAAAAAEABHRlc3QAAnQxAAIDAwAC -wlZOTxcBAAAAJgAAAIkBAAAAgCkAAAAAAAEAAv/8AQAAAAgAAAA='; -# INSERT INTO t1 VALUES (2,8) # with @@skip_replication=0 -BINLOG 'wlZOTxMBAAAAKgAAADwCAAAAACkAAAAAAAEABHRlc3QAAnQxAAIDAwAC -wlZOTxcBAAAAJgAAAGICAAAAACkAAAAAAAEAAv/8AgAAAAgAAAA='; -SET binlog_format= @old_binlog_format; - -SELECT * FROM t1 ORDER BY a; -sync_slave_with_master; -connection slave; -# Slave should have only the second insert, the first should be ignored due to -# the @@skip_replication flag. -SELECT * FROM t1 ORDER by a; - ---source include/stop_slave.inc -SET GLOBAL binlog_format= @old_slave_binlog_format; ---source include/start_slave.inc - - -# Test that it is not possible to change @@skip_replication inside a -# transaction or statement, thereby replicating only parts of statements -# or transactions. -connection master; -SET skip_replication=0; - -BEGIN; ---error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET skip_replication=0; ---error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET skip_replication=1; -ROLLBACK; -SET skip_replication=1; -BEGIN; ---error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET skip_replication=0; ---error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET skip_replication=1; -COMMIT; -SET autocommit=0; -INSERT INTO t2(a) VALUES(100); ---error ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET skip_replication=1; -ROLLBACK; -SET autocommit=1; - -SET skip_replication=1; ---delimiter | -CREATE FUNCTION foo (x INT) RETURNS INT BEGIN SET SESSION skip_replication=x; RETURN x; END| -CREATE PROCEDURE bar(x INT) BEGIN SET SESSION skip_replication=x; END| -CREATE FUNCTION baz (x INT) RETURNS INT BEGIN CALL bar(x); RETURN x; END| ---delimiter ; ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -SELECT foo(0); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -SELECT baz(0); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET @a= foo(1); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -SET @a= baz(1); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -UPDATE t2 SET b=foo(0); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -UPDATE t2 SET b=baz(0); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -INSERT INTO t1 VALUES (101, foo(1)); ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_SKIP_REPLICATION -INSERT INTO t1 VALUES (101, baz(0)); -SELECT @@skip_replication; -CALL bar(0); -SELECT @@skip_replication; -CALL bar(1); -SELECT @@skip_replication; -DROP FUNCTION foo; -DROP PROCEDURE bar; -DROP FUNCTION baz; - - -# Test that master-side filtering happens on the master side, and that -# slave-side filtering happens on the slave. - -# First test that events do not reach the slave when master-side filtering -# is configured. Do this by replicating first with only the IO thread running -# and master-side filtering; then change to no filtering and start the SQL -# thread. This should still skip the events, as master-side filtering -# means the events never reached the slave. -connection master; -SET skip_replication= 0; -TRUNCATE t1; -sync_slave_with_master; -connection slave; -STOP SLAVE; -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_MASTER; -START SLAVE IO_THREAD; -connection master; -SET skip_replication= 1; -INSERT INTO t1(a) VALUES (1); -SET skip_replication= 0; -INSERT INTO t1(a) VALUES (2); ---source include/save_master_pos.inc -connection slave; ---source include/sync_io_with_master.inc -STOP SLAVE IO_THREAD; -SET GLOBAL replicate_events_marked_for_skip=REPLICATE; -START SLAVE; -connection master; -sync_slave_with_master; -connection slave; -# Now only the second insert of (2) should be visible, as the first was -# filtered on the master, so even though the SQL thread ran without skipping -# events, it will never see the event in the first place. -SELECT * FROM t1; - -# Now tests that when slave-side filtering is configured, events _do_ reach -# the slave. -connection master; -SET skip_replication= 0; -TRUNCATE t1; -sync_slave_with_master; -connection slave; -STOP SLAVE; -SET GLOBAL replicate_events_marked_for_skip=FILTER_ON_SLAVE; -START SLAVE IO_THREAD; -connection master; -SET skip_replication= 1; -INSERT INTO t1(a) VALUES (1); -SET skip_replication= 0; -INSERT INTO t1(a) VALUES (2); ---source include/save_master_pos.inc -connection slave; ---source include/sync_io_with_master.inc -STOP SLAVE IO_THREAD; -SET GLOBAL replicate_events_marked_for_skip=REPLICATE; -START SLAVE; -connection master; -sync_slave_with_master; -connection slave; -# Now both inserts should be visible. Since filtering was configured to be -# slave-side, the event is in the relay log, and when the SQL thread ran we -# had disabled filtering again. -SELECT * FROM t1 ORDER BY a; - - -# Clean up. -connection master; -SET skip_replication=0; -DROP TABLE t1,t2; -connection slave; -STOP SLAVE; -SET GLOBAL replicate_events_marked_for_skip=REPLICATE; -START SLAVE; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_skip_replication.inc diff --git a/mysql-test/suite/rpl/t/rpl_special_charset.test b/mysql-test/suite/rpl/t/rpl_special_charset.test index 8ccb1b4183f31..6f196005711b4 100644 --- a/mysql-test/suite/rpl/t/rpl_special_charset.test +++ b/mysql-test/suite/rpl/t/rpl_special_charset.test @@ -1,26 +1 @@ -################################################################################ -# Bug#19855907 IO THREAD AUTHENTICATION ISSUE WITH SOME CHARACTER SETS -# Problem: IO thread fails to connect to master if servers are configured with -# special character sets like utf16, utf32, ucs2. -# -# Analysis: MySQL server does not support few special character sets like -# utf16,utf32 and ucs2 as "client's character set"(eg: utf16,utf32, ucs2). -# When IO thread is trying to connect to Master, it sets server's character -# set as client's character set. When Slave server is started with these -# special character sets, IO thread (a connection to Master) fails because -# of the above said reason. -# -# Fix: If server's character set is not supported as client's character set, -# then set default's client character set(latin1) as client's character set. -############################################################################### ---source include/master-slave.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); -CREATE TABLE t1(i VARCHAR(20)); -INSERT INTO t1 VALUES (0xFFFF); ---sync_slave_with_master ---let diff_tables=master:t1, slave:t1 ---source include/diff_tables.inc -# Cleanup ---connection master -DROP TABLE t1; ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_special_charset.inc diff --git a/mysql-test/suite/rpl/t/rpl_sporadic_master.test b/mysql-test/suite/rpl/t/rpl_sporadic_master.test index 592d13e67b0fe..0a75698204760 100644 --- a/mysql-test/suite/rpl/t/rpl_sporadic_master.test +++ b/mysql-test/suite/rpl/t/rpl_sporadic_master.test @@ -1,26 +1 @@ -# test to see if replication can continue when master sporadically fails on -# COM_BINLOG_DUMP and additionally limits the number of events per dump - -source include/master-slave.inc; - -create table t2(n int); -create table t1(n int not null auto_increment primary key); -insert into t1 values (NULL),(NULL); -truncate table t1; -# We have to use 4 in the following to make this test work with all table types -insert into t1 values (4),(NULL); -sync_slave_with_master; ---source include/stop_slave.inc ---source include/start_slave.inc -connection master; -insert into t1 values (NULL),(NULL); -flush logs; -truncate table t1; -insert into t1 values (10),(NULL),(NULL),(NULL),(NULL),(NULL); -sync_slave_with_master; -select * from t1 ORDER BY n; -connection master; -drop table t1,t2; -sync_slave_with_master; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_sporadic_master.inc diff --git a/mysql-test/suite/rpl/t/rpl_ssl.test b/mysql-test/suite/rpl/t/rpl_ssl.test index 9e42764715b6e..883b367e9f26a 100644 --- a/mysql-test/suite/rpl/t/rpl_ssl.test +++ b/mysql-test/suite/rpl/t/rpl_ssl.test @@ -1,111 +1 @@ -source include/have_ssl_communication.inc; -source include/master-slave.inc; - -# create a user for replication that requires ssl encryption -connection master; -create user replssl@localhost; -grant replication slave on *.* to replssl@localhost require ssl; -create table t1 (t int auto_increment, KEY(t)); - -sync_slave_with_master; - -# Set slave to use SSL for connection to master -stop slave; ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR -eval change master to - master_user='replssl', - master_password='', - master_ssl=1, - master_ssl_ca ='$MYSQL_TEST_DIR/std_data/cacert.pem', - master_ssl_cert='$MYSQL_TEST_DIR/std_data/client-cert.pem', - master_ssl_key='$MYSQL_TEST_DIR/std_data/client-key.pem'; -start slave; - -# Switch to master and insert one record, then sync it to slave -connection master; -insert into t1 values(1); -sync_slave_with_master; - -# The record should now be on slave -select * from t1; - -# The slave is synced and waiting/reading from master -# SHOW SLAVE STATUS will show "Waiting for master to send event" -let $status_items= Master_SSL_Allowed, Master_SSL_CA_Path, Master_SSL_CA_File, Master_SSL_Cert, Master_SSL_Key; -source include/show_slave_status.inc; -source include/check_slave_is_running.inc; - -# Stop the slave, as reported in bug#21871 it would hang -STOP SLAVE; - -select * from t1; - -# Do the same thing a number of times -disable_query_log; -disable_result_log; -# 2007-11-27 mats Bug #32756 Starting and stopping the slave in a loop can lose rows -# After discussions with Engineering, I'm disabling this part of the test to avoid it causing -# red trees. -disable_parsing; -let $i= 100; -while ($i) -{ - start slave; - connection master; - insert into t1 values (NULL); - select * from t1; # Some variance - connection slave; - select * from t1; # Some variance - stop slave; - dec $i; -} -enable_parsing; -START SLAVE; -enable_query_log; -enable_result_log; -connection master; -# INSERT one more record to make sure -# the sync has something to do -insert into t1 values (NULL); -let $master_count= `select count(*) from t1`; - -sync_slave_with_master; ---source include/wait_for_slave_to_start.inc -source include/show_slave_status.inc; -source include/check_slave_is_running.inc; - -let $slave_count= `select count(*) from t1`; - -if ($slave_count != $master_count) -{ - echo master and slave differed in number of rows; - echo master: $master_count; - echo slave: $slave_count; - - connection master; - echo === master ===; - select count(*) t1; - select * from t1; - connection slave; - echo === slave ===; - select count(*) t1; - select * from t1; - query_vertical show slave status; -} - -connection master; -drop user replssl@localhost; -drop table t1; -sync_slave_with_master; - ---source include/stop_slave.inc -CHANGE MASTER TO - master_user = 'root', - master_ssl = 0, - master_ssl_ca = '', - master_ssl_cert = '', - master_ssl_key = ''; - ---echo End of 5.0 tests ---let $rpl_only_running_threads= 1 ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_ssl.inc diff --git a/mysql-test/suite/rpl/t/rpl_stm_relay_ign_space.test b/mysql-test/suite/rpl/t/rpl_stm_relay_ign_space.test index db6e6bd14bfc6..f72300ee2ded1 100644 --- a/mysql-test/suite/rpl/t/rpl_stm_relay_ign_space.test +++ b/mysql-test/suite/rpl/t/rpl_stm_relay_ign_space.test @@ -1,101 +1 @@ -# -# BUG#12400313 / BUG#64503 test case -# -# -# Description -# ----------- -# -# This test case starts the slave server with: -# --relay-log-space-limit=8192 --relay-log-purge --max-relay-log-size=4096 -# -# Then it issues some queries that will cause the slave to reach -# relay-log-space-limit. We lock the table so that the SQL thread is -# not able to purge the log and then we issue some more statements. -# -# The purpose is to show that the IO thread will honor the limits -# while the SQL thread is not able to purge the relay logs, which did -# not happen before this patch. In addition we assert that while -# ignoring the limit (SQL thread needs to rotate before purging), the -# IO thread does not do it in an uncontrolled manner. - ---source include/have_binlog_format_statement.inc ---source include/master-slave.inc ---source include/have_innodb.inc - ---disable_query_log -CREATE TABLE t1 (c1 TEXT) engine=InnoDB; - -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); - ---sync_slave_with_master - -# wait for the SQL thread to sleep ---let $show_statement= SHOW PROCESSLIST ---let $field= State ---let $condition= = 'Slave has read all relay log; waiting for the slave I/O thread to update it' ---source include/wait_show_condition.inc - -# now the io thread has set rli->ignore_space_limit -# lets lock the table so that once the SQL thread awakes -# it blocks there and does not set rli->ignore_space_limit -# back to zero -LOCK TABLE t1 WRITE; - -# now issue more statements that will overflow the -# rli->log_space_limit (in this case ~10K) ---connection master - -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); -INSERT INTO t1 VALUES ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'); - ---connection slave - -# ASSERT that the IO thread waits for the SQL thread to release some -# space before continuing ---let $show_statement= SHOW PROCESSLIST ---let $field= State ---let $condition= LIKE 'Waiting for %' -# before the patch (IO would have transfered everything) -#--let $condition= = 'Waiting for master to send event' -# after the patch (now it waits for space to be freed) -#--let $condition= = 'Waiting for the slave SQL thread to free enough relay log space' ---source include/wait_show_condition.inc - -# without the patch we can uncomment the following two lines and -# watch the IO thread synchronize with the master, thus writing -# relay logs way over the space limit -#--connection master -#--source include/sync_slave_io_with_master.inc - -## ASSERT that the IO thread has honored the limit+few bytes required to be able to purge ---let $relay_log_space_while_sql_is_executing = query_get_value(SHOW SLAVE STATUS, Relay_Log_Space, 1) ---let $relay_log_space_limit = query_get_value(SHOW VARIABLES LIKE "relay_log_space_limit", Value, 1) ---let $assert_text= Assert that relay log space is close to the limit ---let $assert_cond= $relay_log_space_while_sql_is_executing <= $relay_log_space_limit * 1.15 ---source include/assert.inc - -# unlock the table and let SQL thread continue applying events -UNLOCK TABLES; - ---connection master ---sync_slave_with_master ---let $diff_tables=master:test.t1,slave:test.t1 ---source include/diff_tables.inc - ---connection master -DROP TABLE t1; ---enable_query_log ---sync_slave_with_master - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_stm_relay_ign_space.inc diff --git a/mysql-test/suite/rpl/t/rpl_switch_stm_row_mixed.test b/mysql-test/suite/rpl/t/rpl_switch_stm_row_mixed.test index 575fdb2e89dac..cd826c6be1ee6 100644 --- a/mysql-test/suite/rpl/t/rpl_switch_stm_row_mixed.test +++ b/mysql-test/suite/rpl/t/rpl_switch_stm_row_mixed.test @@ -1,625 +1 @@ -# -# rpl_switch_stm_row_mixed tests covers -# -# - Master is switching explicitly between STATEMENT, ROW, and MIXED -# binlog format showing when it is possible and when not. -# - Master switching from MIXED to RBR implicitly listing all use -# cases, e.g a query invokes UUID(), thereafter to serve as the -# definition of MIXED binlog format -# - correctness of execution - - --- source include/have_binlog_format_mixed_or_row.inc --- source include/master-slave.inc - -# Since this test generates row-based events in the binary log, the -# slave SQL thread cannot be in STATEMENT mode to execute this test, -# so we only execute it for MIXED and ROW as default value of -# BINLOG_FORMAT. - -connection slave; - -connection master; ---disable_warnings -drop database if exists mysqltest1; -create database mysqltest1; ---enable_warnings -use mysqltest1; - -# Save binlog format -set @my_binlog_format= @@global.binlog_format; - -# play with switching -set session binlog_format=mixed; -show session variables like "binlog_format%"; -set session binlog_format=statement; -show session variables like "binlog_format%"; -set session binlog_format=row; -show session variables like "binlog_format%"; - -set global binlog_format=DEFAULT; -show global variables like "binlog_format%"; -set global binlog_format=MIXED; -show global variables like "binlog_format%"; -set global binlog_format=STATEMENT; -show global variables like "binlog_format%"; -set global binlog_format=ROW; -show global variables like "binlog_format%"; -show session variables like "binlog_format%"; -select @@global.binlog_format, @@session.binlog_format; - -CREATE TABLE t1 (a varchar(100)); - -prepare stmt1 from 'insert into t1 select concat(UUID(),?)'; -set @string="emergency_1_"; -insert into t1 values("work_2_"); -execute stmt1 using @string; -deallocate prepare stmt1; - -prepare stmt1 from 'insert into t1 select ?'; -insert into t1 values(concat(UUID(),"work_3_")); -execute stmt1 using @string; -deallocate prepare stmt1; - -insert into t1 values(concat("for_4_",UUID())); -insert into t1 select "yesterday_5_"; - -# verify that temp tables prevent a switch to SBR -create temporary table tmp(a char(100)); -insert into tmp values("see_6_"); ---error ER_TEMP_TABLE_PREVENTS_SWITCH_OUT_OF_RBR -set binlog_format=statement; -insert into t1 select * from tmp; -drop temporary table tmp; - -# Now we go to SBR -set binlog_format=statement; -show global variables like "binlog_format%"; -show session variables like "binlog_format%"; -select @@global.binlog_format, @@session.binlog_format; -set global binlog_format=statement; -show global variables like "binlog_format%"; -show session variables like "binlog_format%"; -select @@global.binlog_format, @@session.binlog_format; - -prepare stmt1 from 'insert into t1 select ?'; -set @string="emergency_7_"; -insert into t1 values("work_8_"); -execute stmt1 using @string; -deallocate prepare stmt1; - -prepare stmt1 from 'insert into t1 select ?'; -insert into t1 values("work_9_"); -execute stmt1 using @string; -deallocate prepare stmt1; - -insert into t1 values("for_10_"); -insert into t1 select "yesterday_11_"; - -# test statement (is not default after wl#3368) -set binlog_format=statement; -select @@global.binlog_format, @@session.binlog_format; -set global binlog_format=statement; -select @@global.binlog_format, @@session.binlog_format; - -prepare stmt1 from 'insert into t1 select ?'; -set @string="emergency_12_"; -insert into t1 values("work_13_"); -execute stmt1 using @string; -deallocate prepare stmt1; - -prepare stmt1 from 'insert into t1 select ?'; -insert into t1 values("work_14_"); -execute stmt1 using @string; -deallocate prepare stmt1; - -insert into t1 values("for_15_"); -insert into t1 select "yesterday_16_"; - -# and now the mixed mode - -set global binlog_format=mixed; -select @@global.binlog_format, @@session.binlog_format; -set binlog_format=default; -select @@global.binlog_format, @@session.binlog_format; - -prepare stmt1 from 'insert into t1 select concat(UUID(),?)'; -set @string="emergency_17_"; -insert into t1 values("work_18_"); -execute stmt1 using @string; -deallocate prepare stmt1; - -prepare stmt1 from 'insert into t1 select ?'; -insert into t1 values(concat(UUID(),"work_19_")); -execute stmt1 using @string; -deallocate prepare stmt1; - -insert into t1 values(concat("for_20_",UUID())); -insert into t1 select "yesterday_21_"; - -prepare stmt1 from 'insert into t1 select ?'; -insert into t1 values(concat(UUID(),"work_22_")); -execute stmt1 using @string; -deallocate prepare stmt1; - -insert into t1 values(concat("for_23_",UUID())); -insert into t1 select "yesterday_24_"; - -# Test of CREATE TABLE SELECT - -create table t2 ENGINE=MyISAM select rpad(UUID(),100,' '); -create table t3 select 1 union select UUID(); ---disable_warnings -create table t4 select * from t1 where 3 in (select 1 union select 2 union select UUID() union select 3); ---enable_warnings -create table t5 select * from t1 where 3 in (select 1 union select 2 union select curdate() union select 3); -# what if UUID() is first: ---disable_warnings -insert into t5 select UUID() from t1 where 3 in (select 1 union select 2 union select 3 union select * from t4); ---enable_warnings - -# inside a stored procedure - -delimiter |; -create procedure foo() -begin -insert into t1 values("work_25_"); -insert into t1 values(concat("for_26_",UUID())); -insert into t1 select "yesterday_27_"; -end| -create procedure foo2() -begin -insert into t1 values(concat("emergency_28_",UUID())); -insert into t1 values("work_29_"); -insert into t1 values(concat("for_30_",UUID())); -set session binlog_format=row; # accepted for stored procs -insert into t1 values("more work_31_"); -set session binlog_format=mixed; -end| -create function foo3() returns bigint unsigned -begin - set session binlog_format=row; # rejected for stored funcs - insert into t1 values("alarm"); - return 100; -end| -create procedure foo4(x varchar(100)) -begin -insert into t1 values(concat("work_250_",x)); -insert into t1 select "yesterday_270_"; -end| -delimiter ;| -call foo(); -call foo2(); -call foo4("hello"); -call foo4(UUID()); -call foo4("world"); - -# test that can't SET in a stored function ---error ER_STORED_FUNCTION_PREVENTS_SWITCH_BINLOG_FORMAT -select foo3(); -select * from t1 where a="alarm"; - -# Tests of stored functions/triggers/views for BUG#20930 "Mixed -# binlogging mode does not work with stored functions, triggers, -# views" - -# Function which calls procedure -drop function foo3; -delimiter |; -create function foo3() returns bigint unsigned -begin - insert into t1 values("foo3_32_"); - call foo(); - return 100; -end| -delimiter ;| -insert into t2 select foo3(); - -prepare stmt1 from 'insert into t2 select foo3()'; -execute stmt1; -execute stmt1; -deallocate prepare stmt1; - -# Test if stored function calls stored function which calls procedure -# which requires row-based. - -delimiter |; -create function foo4() returns bigint unsigned -begin - insert into t2 select foo3(); - return 100; -end| -delimiter ;| -select foo4(); - -prepare stmt1 from 'select foo4()'; -execute stmt1; -execute stmt1; -deallocate prepare stmt1; - -# A simple stored function -delimiter |; -create function foo5() returns bigint unsigned -begin - insert into t2 select UUID(); - return 100; -end| -delimiter ;| -select foo5(); - -prepare stmt1 from 'select foo5()'; -execute stmt1; -execute stmt1; -deallocate prepare stmt1; - -# A simple stored function where UUID() is in the argument -delimiter |; -create function foo6(x varchar(100)) returns bigint unsigned -begin - insert into t2 select x; - return 100; -end| -delimiter ;| -select foo6("foo6_1_"); -select foo6(concat("foo6_2_",UUID())); - -prepare stmt1 from 'select foo6(concat("foo6_3_",UUID()))'; -execute stmt1; -execute stmt1; -deallocate prepare stmt1; - - -# Test of views using UUID() - -create view v1 as select uuid(); -create table t11 (data varchar(255)); -insert into t11 select * from v1; -# Test of querying INFORMATION_SCHEMA which parses the view's body, -# to verify that it binlogs statement-based (is not polluted by -# the parsing of the view's body). -insert into t11 select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='mysqltest1' and TABLE_NAME IN ('v1','t11'); -prepare stmt1 from "insert into t11 select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='mysqltest1' and TABLE_NAME IN ('v1','t11')"; -execute stmt1; -execute stmt1; -deallocate prepare stmt1; - -# Test of triggers with UUID() -delimiter |; -create trigger t11_bi before insert on t11 for each row -begin - set NEW.data = concat(NEW.data,UUID()); -end| -delimiter ;| -insert into t11 values("try_560_"); - -# Test that INSERT DELAYED works in mixed mode (BUG#20649) -insert delayed into t2 values("delay_1_"); -insert delayed into t2 values(concat("delay_2_",UUID())); -insert delayed into t2 values("delay_6_"); - -# Test for BUG#20633 (INSERT DELAYED RAND()/user_variable does not -# replicate fine in statement-based ; we test that in mixed mode it -# works). -insert delayed into t2 values(rand()); -set @a=2.345; -insert delayed into t2 values(@a); - -# With INSERT DELAYED, rows are written to the binlog after they are -# written to the table. Therefore, it is not enough to wait until the -# rows make it to t2 on the master (the rows may not be in the binlog -# at that time, and may still not be in the binlog when -# sync_slave_with_master is later called). Instead, we wait until the -# rows make it to t2 on the slave. We first call -# sync_slave_with_master, so that we are sure that t2 has been created -# on the slave. -sync_slave_with_master; -let $wait_condition= SELECT COUNT(*) = 19 FROM mysqltest1.t2; ---source include/wait_condition.inc -connection master; - -# If you want to do manual testing of the mixed mode regarding UDFs (not -# testable automatically as quite platform- and compiler-dependent), -# you just need to set the variable below to 1, and to -# "make udf_example.so" in sql/, and to copy sql/udf_example.so to -# MYSQL_TEST_DIR/lib/mysql. -let $you_want_to_test_UDF=0; -if ($you_want_to_test_UDF) -{ - CREATE FUNCTION metaphon RETURNS STRING SONAME 'udf_example.so'; - prepare stmt1 from 'insert into t1 select metaphon(?)'; - set @string="emergency_133_"; - insert into t1 values("work_134_"); - execute stmt1 using @string; - deallocate prepare stmt1; - prepare stmt1 from 'insert into t1 select ?'; - insert into t1 values(metaphon("work_135_")); - execute stmt1 using @string; - deallocate prepare stmt1; - insert into t1 values(metaphon("for_136_")); - insert into t1 select "yesterday_137_"; - create table t6 select metaphon("for_138_"); - create table t7 select 1 union select metaphon("for_139_"); - create table t8 select * from t1 where 3 in (select 1 union select 2 union select metaphon("for_140_") union select 3); - create table t9 select * from t1 where 3 in (select 1 union select 2 union select curdate() union select 3); -} - -create table t20 select * from t1; # save for comparing later -create table t21 select * from t2; -create table t22 select * from t3; -drop table t1,t2,t3; - -# This tests the fix to -# BUG#19630 stored function inserting into two auto_increment breaks statement-based binlog -# We verify that under the mixed binlog mode, a stored function -# modifying at least two tables having an auto_increment column, -# is binlogged row-based. Indeed in statement-based binlogging, -# only the auto_increment value generated for the first table -# is recorded in the binlog, the value generated for the 2nd table -# lacking. - -create table t1 (a int primary key auto_increment, b varchar(100)); -create table t2 (a int primary key auto_increment, b varchar(100)); -create table t3 (b varchar(100)); -delimiter |; -create function f (x varchar(100)) returns int deterministic -begin - insert into t1 values(null,x); - insert into t2 values(null,x); - return 1; -end| -delimiter ;| -select f("try_41_"); -# Two operations which compensate each other except that their net -# effect is that they advance the auto_increment counter of t2 on slave: -sync_slave_with_master; -use mysqltest1; -insert into t2 values(2,null),(3,null),(4,null); -delete from t2 where a>=2; - -connection master; -# this is the call which didn't replicate well -select f("try_42_"); -sync_slave_with_master; - -# now use prepared statement and test again, just to see that the RBB -# mode isn't set at PREPARE but at EXECUTE. - -insert into t2 values(3,null),(4,null); -delete from t2 where a>=3; - -connection master; -prepare stmt1 from 'select f(?)'; -set @string="try_43_"; -insert into t1 values(null,"try_44_"); # should be SBB -execute stmt1 using @string; # should be RBB -deallocate prepare stmt1; -sync_slave_with_master; - -# verify that if only one table has auto_inc, it does not trigger RBB -# (we'll check in binlog further below) - -connection master; -create table t12 select * from t1; # save for comparing later -drop table t1; -create table t1 (a int, b varchar(100), key(a)); -select f("try_45_"); - -# restore table's key -create table t13 select * from t1; -drop table t1; -create table t1 (a int primary key auto_increment, b varchar(100)); - -# now test if it's two functions, each of them inserts in one table - -drop function f; -# we need a unique key to have sorting of rows by mysqldump -create table t14 (unique (a)) select * from t2; -truncate table t2; -delimiter |; -create function f1 (x varchar(100)) returns int deterministic -begin - insert into t1 values(null,x); - return 1; -end| -create function f2 (x varchar(100)) returns int deterministic -begin - insert into t2 values(null,x); - return 1; -end| -delimiter ;| -select f1("try_46_"),f2("try_47_"); - -sync_slave_with_master; -insert into t2 values(2,null),(3,null),(4,null); -delete from t2 where a>=2; - -connection master; -# Test with SELECT and INSERT -select f1("try_48_"),f2("try_49_"); -insert into t3 values(concat("try_50_",f1("try_51_"),f2("try_52_"))); -sync_slave_with_master; - -# verify that if f2 does only read on an auto_inc table, this does not -# switch to RBB -connection master; -drop function f2; -delimiter |; -create function f2 (x varchar(100)) returns int deterministic -begin - declare y int; - insert into t1 values(null,x); - set y = (select count(*) from t2); - return y; -end| -delimiter ;| -select f1("try_53_"),f2("try_54_"); -sync_slave_with_master; - -# And now, a normal statement with a trigger (no stored functions) - -connection master; -drop function f2; -delimiter |; -create trigger t1_bi before insert on t1 for each row -begin - insert into t2 values(null,"try_55_"); -end| -delimiter ;| -insert into t1 values(null,"try_56_"); -# and now remove one auto_increment and verify SBB -alter table t1 modify a int, drop primary key; -insert into t1 values(null,"try_57_"); -sync_slave_with_master; - -# Test for BUG#20499 "mixed mode with temporary table breaks binlog" -# Slave used to have only 2 rows instead of 3. -connection master; -CREATE TEMPORARY TABLE t15 SELECT UUID(); -create table t16 like t15; -INSERT INTO t16 SELECT * FROM t15; -# we'll verify that this one is done RBB -insert into t16 values("try_65_"); -drop table t15; -# we'll verify that this one is done SBB -insert into t16 values("try_66_"); -sync_slave_with_master; - -# and now compare: - -connection master; - -# first check that data on master is sensible -select count(*) from t1; -select count(*) from t2; -select count(*) from t3; -select count(*) from t4; -select count(*) from t5; -select count(*) from t11; -select count(*) from t20; -select count(*) from t21; -select count(*) from t22; -select count(*) from t12; -select count(*) from t13; -select count(*) from t14; -select count(*) from t16; -if ($you_want_to_test_UDF) -{ - select count(*) from t6; - select count(*) from t7; - select count(*) from t8; - select count(*) from t9; -} - -sync_slave_with_master; - -# -# Bug#20863 If binlog format is changed between update and unlock of -# tables, wrong binlog -# - -connection master; -DROP TABLE IF EXISTS t11; -SET SESSION BINLOG_FORMAT=STATEMENT; -CREATE TABLE t11 (song VARCHAR(255)); -LOCK TABLES t11 WRITE; -SET SESSION BINLOG_FORMAT=ROW; -INSERT INTO t11 VALUES('Several Species of Small Furry Animals Gathered Together in a Cave and Grooving With a Pict'); -SET SESSION BINLOG_FORMAT=STATEMENT; -INSERT INTO t11 VALUES('Careful With That Axe, Eugene'); -UNLOCK TABLES; - ---query_vertical SELECT * FROM t11 -sync_slave_with_master; -USE mysqltest1; ---query_vertical SELECT * FROM t11 - -connection master; -DROP TABLE IF EXISTS t12; -SET SESSION BINLOG_FORMAT=MIXED; -CREATE TABLE t12 (data LONG); -LOCK TABLES t12 WRITE; -INSERT INTO t12 VALUES(UUID()); -UNLOCK TABLES; -sync_slave_with_master; - -# -# BUG#28086: SBR of USER() becomes corrupted on slave -# - -connection master; - -# Just to get something that is non-trivial, albeit still simple, we -# stuff the result of USER() and CURRENT_USER() into a variable. ---delimiter $$ -CREATE FUNCTION my_user() - RETURNS CHAR(64) -BEGIN - DECLARE user CHAR(64); - SELECT USER() INTO user; - RETURN user; -END $$ ---delimiter ; - ---delimiter $$ -CREATE FUNCTION my_current_user() - RETURNS CHAR(64) -BEGIN - DECLARE user CHAR(64); - SELECT CURRENT_USER() INTO user; - RETURN user; -END $$ ---delimiter ; - -DROP TABLE IF EXISTS t13; -CREATE TABLE t13 (data CHAR(64)); -INSERT INTO t13 VALUES (USER()); -INSERT INTO t13 VALUES (my_user()); -INSERT INTO t13 VALUES (CURRENT_USER()); -INSERT INTO t13 VALUES (my_current_user()); - -sync_slave_with_master; - -# as we're using UUID we don't SELECT but use "diff" like in rpl_row_UUID ---exec $MYSQL_DUMP --compact --order-by-primary --skip-extended-insert --no-create-info mysqltest1 > $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql ---exec $MYSQL_DUMP_SLAVE --compact --order-by-primary --skip-extended-insert --no-create-info mysqltest1 > $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_slave.sql - -# Let's compare. Note: If they match test will pass, if they do not match -# the test will show that the diff statement failed and not reject file -# will be created. You will need to go to the mysql-test dir and diff -# the files your self to see what is not matching - -diff_files $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_slave.sql; - -connection master; - -# Now test that mysqlbinlog works fine on a binlog generated by the -# mixed mode - -# BUG#11312 "DELIMITER is not written to the binary log that causes -# syntax error" makes that mysqlbinlog will fail if we pass it the -# text of queries; this forces us to use --base64-output here. - -# BUG#20929 "BINLOG command causes invalid free plus assertion -# failure" makes mysqld segfault when receiving --base64-output - -# So I can't enable this piece of test -# SIGH - -if ($enable_when_11312_or_20929_fixed) -{ ---exec $MYSQL_BINLOG --base64-output $MYSQLTEST_VARDIR/log/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_mixed.sql -drop database mysqltest1; ---exec $MYSQL < $MYSQLTEST_VARDIR/tmp/mysqlbinlog_mixed.sql ---exec $MYSQL_DUMP --compact --order-by-primary --skip-extended-insert --no-create-info mysqltest1 > $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql -# the old mysqldump output on slave is the same as what it was on -# master before restoring on master. -diff_files $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_master.sql $MYSQLTEST_VARDIR/tmp/rpl_switch_stm_row_mixed_slave.sql; -} - -drop database mysqltest1; -sync_slave_with_master; - -connection master; -# Restore binlog format setting -set global binlog_format =@my_binlog_format; ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_switch_stm_row_mixed.inc diff --git a/mysql-test/suite/rpl/t/rpl_sync.test b/mysql-test/suite/rpl/t/rpl_sync.test index 820ec19925f3d..ec98a344282f1 100644 --- a/mysql-test/suite/rpl/t/rpl_sync.test +++ b/mysql-test/suite/rpl/t/rpl_sync.test @@ -1,153 +1,2 @@ -######################################################################################## -# This test verifies the options --sync-relay-log-info and --relay-log-recovery by -# crashing the slave in two different situations: -# (case-1) - Corrupt the relay log with changes which were not processed by -# the SQL Thread and crashes it. -# (case-2) - Corrupt the master.info with wrong coordinates and crashes it. -# -# Case 1: -# 1 - Stops the SQL Thread -# 2 - Inserts new records into the master. -# 3 - Corrupts the relay-log.bin* which most likely has such changes. -# 4 - Crashes the slave -# 5 - Verifies if the slave is sync with the master which means that the information -# loss was circumvented by the recovery process. -# -# Case 2: -# 1 - Stops the SQL/IO Threads -# 2 - Inserts new records into the master. -# 3 - Corrupts the master.info with wrong coordinates. -# 4 - Crashes the slave -# 5 - Verifies if the slave is sync with the master which means that the information -# loss was circumvented by the recovery process. -######################################################################################## - -######################################################################################## -# Configuring the environment -######################################################################################## ---echo =====Configuring the enviroment=======; ---source include/master-slave.inc ---source include/not_embedded.inc ---source include/not_valgrind.inc ---source include/have_debug.inc ---source include/have_innodb.inc ---source include/not_crashrep.inc - -call mtr.add_suppression('Attempting backtrace'); -call mtr.add_suppression("Recovery from master pos .* and file master-bin.000001"); -# Use innodb so we do not get "table should be repaired" issues. -ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; -flush tables; -CREATE TABLE t1(a INT, PRIMARY KEY(a)) engine=innodb; - -insert into t1(a) values(1); -insert into t1(a) values(2); -insert into t1(a) values(3); - -######################################################################################## -# Case 1: Corrupt a relay-log.bin* -######################################################################################## ---echo =====Inserting data on the master but without the SQL Thread being running=======; -sync_slave_with_master; - -connection slave; -let $MYSQLD_SLAVE_DATADIR= `select @@datadir`; ---replace_result $MYSQLD_SLAVE_DATADIR MYSQLD_SLAVE_DATADIR ---copy_file $MYSQLD_SLAVE_DATADIR/master.info $MYSQLD_SLAVE_DATADIR/master.backup ---source include/stop_slave_sql.inc - -connection master; -insert into t1(a) values(4); -insert into t1(a) values(5); -insert into t1(a) values(6); - ---echo =====Removing relay log files and crashing/recoverying the slave=======; -connection slave; ---source include/stop_slave_io.inc - -let $file= query_get_value("SHOW SLAVE STATUS", Relay_Log_File, 1); - ---let FILE_TO_CORRUPT= $MYSQLD_SLAVE_DATADIR/$file -perl; -$file= $ENV{'FILE_TO_CORRUPT'}; -open(FILE, ">$file") || die "Unable to open $file."; -truncate(FILE,0); -print FILE "failure"; -close ($file); -EOF - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect -SET SESSION debug_dbug="d,crash_before_rotate_relaylog"; ---error 2013 -FLUSH LOGS; - ---let $rpl_server_number= 2 ---source include/rpl_reconnect.inc - ---echo =====Dumping and comparing tables=======; ---source include/start_slave.inc - -connection master; -sync_slave_with_master; - -let $diff_tables=master:t1,slave:t1; -source include/diff_tables.inc; - -######################################################################################## -# Case 2: Corrupt a master.info -######################################################################################## ---echo =====Corrupting the master.info=======; -connection slave; ---source include/stop_slave.inc - -connection master; -FLUSH LOGS; - -insert into t1(a) values(7); -insert into t1(a) values(8); -insert into t1(a) values(9); - -connection slave; -let MYSQLD_SLAVE_DATADIR=`select @@datadir`; - ---perl -use strict; -use warnings; -my $src= "$ENV{'MYSQLD_SLAVE_DATADIR'}/master.backup"; -my $dst= "$ENV{'MYSQLD_SLAVE_DATADIR'}/master.info"; -open(FILE, "<", $src) or die; -my @content= ; -close FILE; -open(FILE, ">", $dst) or die; -binmode FILE; -print FILE @content; -close FILE; -EOF - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect -SET SESSION debug_dbug="d,crash_before_rotate_relaylog"; ---error 2013 -FLUSH LOGS; - ---let $rpl_server_number= 2 ---source include/rpl_reconnect.inc - ---echo =====Dumping and comparing tables=======; ---source include/start_slave.inc - -connection master; -sync_slave_with_master; - -let $diff_tables=master:t1,slave:t1; -source include/diff_tables.inc; - -######################################################################################## -# Clean up -######################################################################################## ---echo =====Clean up=======; -connection master; -drop table t1; - ---remove_file $MYSQLD_SLAVE_DATADIR/master.backup ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_sync.inc diff --git a/mysql-test/suite/rpl/t/rpl_temporal_format_default_to_default.test b/mysql-test/suite/rpl/t/rpl_temporal_format_default_to_default.test index af4ce104af40d..99a70e011c47f 100644 --- a/mysql-test/suite/rpl/t/rpl_temporal_format_default_to_default.test +++ b/mysql-test/suite/rpl/t/rpl_temporal_format_default_to_default.test @@ -1,76 +1 @@ ---source include/master-slave.inc - -if ($force_master_mysql56_temporal_format) -{ - connection master; - eval SET @@global.mysql56_temporal_format=$force_master_mysql56_temporal_format; -} - -if ($force_slave_mysql56_temporal_format) -{ - connection slave; - eval SET @@global.mysql56_temporal_format=$force_slave_mysql56_temporal_format; -} - -connection master; -SELECT @@global.mysql56_temporal_format AS on_master; -connection slave; -SELECT @@global.mysql56_temporal_format AS on_slave; -connection master; - -CREATE TABLE t1 -( - c0 TIME(0), - c1 TIME(1), - c2 TIME(2), - c3 TIME(3), - c4 TIME(4), - c5 TIME(5), - c6 TIME(6) -); -CREATE TABLE t2 -( - c0 TIMESTAMP(0), - c1 TIMESTAMP(1), - c2 TIMESTAMP(2), - c3 TIMESTAMP(3), - c4 TIMESTAMP(4), - c5 TIMESTAMP(5), - c6 TIMESTAMP(6) -); - -CREATE TABLE t3 -( - c0 DATETIME(0), - c1 DATETIME(1), - c2 DATETIME(2), - c3 DATETIME(3), - c4 DATETIME(4), - c5 DATETIME(5), - c6 DATETIME(6) -); -INSERT INTO t1 VALUES ('01:01:01','01:01:01.1','01:01:01.11','01:01:01.111','01:01:01.1111','01:01:01.11111','01:01:01.111111'); -INSERT INTO t2 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); -INSERT INTO t3 VALUES ('2001-01-01 01:01:01','2001-01-01 01:01:01.1','2001-01-01 01:01:01.11','2001-01-01 01:01:01.111','2001-01-01 01:01:01.1111','2001-01-01 01:01:01.11111','2001-01-01 01:01:01.111111'); -SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES -WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; -sync_slave_with_master; - -connection slave; ---query_vertical SELECT * FROM t1; ---query_vertical SELECT * FROM t2; ---query_vertical SELECT * FROM t3; -SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH,DATA_LENGTH FROM INFORMATION_SCHEMA.TABLES -WHERE TABLE_NAME RLIKE 't[1-3]' ORDER BY TABLE_NAME; - -connection master; -DROP TABLE t1; -DROP TABLE t2; -DROP TABLE t3; - -connection slave; -SET @@global.mysql56_temporal_format=DEFAULT; -connection master; -SET @@global.mysql56_temporal_format=DEFAULT; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_temporal_format_default_to_default.inc diff --git a/mysql-test/suite/rpl/t/rpl_typeconv.test b/mysql-test/suite/rpl/t/rpl_typeconv.test index 59d75dd47f520..4dbfc27d088e5 100644 --- a/mysql-test/suite/rpl/t/rpl_typeconv.test +++ b/mysql-test/suite/rpl/t/rpl_typeconv.test @@ -1,72 +1 @@ ---source include/have_binlog_format_row.inc ---source include/master-slave.inc - -connection slave; -set @saved_slave_type_conversions = @@global.slave_type_conversions; -CREATE TABLE type_conversions ( - TestNo INT AUTO_INCREMENT PRIMARY KEY, - Source TEXT, - Target TEXT, - Flags TEXT, - On_Master TEXT, - On_Slave TEXT, - Expected TEXT, - Compare INT, - Error TEXT); - -SELECT @@global.slave_type_conversions; -SET GLOBAL SLAVE_TYPE_CONVERSIONS=''; -SELECT @@global.slave_type_conversions; -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_NON_LOSSY'; -SELECT @@global.slave_type_conversions; -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY'; -SELECT @@global.slave_type_conversions; -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY'; -SELECT @@global.slave_type_conversions; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY,NONEXISTING_BIT'; -SELECT @@global.slave_type_conversions; - -# Checking strict interpretation of type conversions -connection slave; -SET GLOBAL SLAVE_TYPE_CONVERSIONS=''; -source extra/rpl_tests/type_conversions.test; - -# Checking lossy integer type conversions -connection slave; -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_NON_LOSSY'; -source extra/rpl_tests/type_conversions.test; - -# Checking non-lossy integer type conversions -connection slave; -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY'; -source extra/rpl_tests/type_conversions.test; - -# Checking all type conversions -connection slave; -SET GLOBAL SLAVE_TYPE_CONVERSIONS='ALL_LOSSY,ALL_NON_LOSSY'; -source extra/rpl_tests/type_conversions.test; - -connection slave; ---echo **** Result of conversions **** -disable_query_log; -SELECT RPAD(Source, 15, ' ') AS Source_Type, - RPAD(Target, 15, ' ') AS Target_Type, - RPAD(Flags, 25, ' ') AS All_Type_Conversion_Flags, - IF(Compare IS NULL AND Error IS NOT NULL, '', - IF(Compare, '', - CONCAT("'", On_Slave, "' != '", Expected, "'"))) - AS Value_On_Slave - FROM type_conversions; -enable_query_log; -DROP TABLE type_conversions; - -call mtr.add_suppression("Slave SQL.*Column 1 of table .test.t1. cannot be converted from type.* error.* 1677"); - -connection master; -DROP TABLE t1; -sync_slave_with_master; - -set global slave_type_conversions = @saved_slave_type_conversions; - ---source include/rpl_end.inc +--source extra/rpl_tests/rpl_typeconv.inc diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 8c79ab0f29641..afd7a3c42c7ce 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -89,6 +89,10 @@ binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- +binlog_encryption.* : Added in 10.1.20 + +#---------------------------------------------------------------- + connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results connect.jdbc : New test, added on 2016-07-15 connect.jdbc-new : New test, added on 2016-07-14 From d036be7218fd9fb8b2233a121281a849bb53d4c0 Mon Sep 17 00:00:00 2001 From: Ronak Jain Date: Tue, 6 Dec 2016 02:29:52 +0530 Subject: [PATCH 238/295] fixes MDEV-11354 twin include --- sql/hostname.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hostname.cc b/sql/hostname.cc index 6aef84f1b26c4..39e4b34d615b1 100644 --- a/sql/hostname.cc +++ b/sql/hostname.cc @@ -28,7 +28,6 @@ #include "sql_priv.h" #include "unireg.h" // SPECIAL_NO_HOST_CACHE #include "hostname.h" -#include "unireg.h" #ifndef __WIN__ #include // getservbyname, servent #endif From 46dee0d1848e24e39fc236e78a438efa22356f8d Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 5 Dec 2016 16:50:12 +0400 Subject: [PATCH 239/295] MDEV-10717 Assertion `!null_value' failed in virtual bool Item::send(Protocol*, String*) The problem was that null_value was not set to "false" on a well-formed row. If an ill-formed row was followed by a well-forned row, null_value remained "true" in the call of Item::send() for the well-formed row. --- mysql-test/r/ctype_utf8.result | 26 ++++++++++++++++++++++++++ mysql-test/t/ctype_utf8.test | 14 ++++++++++++++ sql/item.cc | 1 + 3 files changed, 41 insertions(+) diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index 121168c2a2a5c..294c2cb2be13b 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -5869,5 +5869,31 @@ SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65537) AS data) AS sub; len 131074 # +# MDEV-10717 Assertion `!null_value' failed in virtual bool Item::send(Protocol*, String*) +# +CREATE TABLE t1 (i INT, KEY(i)); +INSERT INTO t1 VALUES (20081205),(20050327); +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +HEX(i) HEX(CHAR(i USING utf8)) +131F197 0131 +1326A35 01326A35 +Warnings: +Warning 1300 Invalid utf8 character string: 'F197' +SET sql_mode='STRICT_ALL_TABLES'; +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +HEX(i) HEX(CHAR(i USING utf8)) +131F197 NULL +1326A35 01326A35 +Warnings: +Warning 1300 Invalid utf8 character string: 'F197' +SELECT CHAR(i USING utf8) FROM t1; +CHAR(i USING utf8) +### +### +Warnings: +### 1300 Invalid utf8 character string: 'F197' +SET sql_mode=DEFAULT; +DROP TABLE t1; +# # End of 5.5 tests # diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index d6fdc6c6a2ced..75581ede8fa3a 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1681,6 +1681,20 @@ SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65535) AS data) AS sub; SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65536) AS data) AS sub; SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65537) AS data) AS sub; +--echo # +--echo # MDEV-10717 Assertion `!null_value' failed in virtual bool Item::send(Protocol*, String*) +--echo # +CREATE TABLE t1 (i INT, KEY(i)); +INSERT INTO t1 VALUES (20081205),(20050327); +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +SET sql_mode='STRICT_ALL_TABLES'; +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +# Avoid garbage in the output +--replace_column 1 ### +SELECT CHAR(i USING utf8) FROM t1; +SET sql_mode=DEFAULT; +DROP TABLE t1; + --echo # --echo # End of 5.5 tests --echo # diff --git a/sql/item.cc b/sql/item.cc index 3448b23664050..ede1df6aa9b95 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5743,6 +5743,7 @@ String *Item::check_well_formed_result(String *str, bool send_error) uint wlen= cs->cset->well_formed_len(cs, str->ptr(), str->ptr() + str->length(), str->length(), &well_formed_error); + null_value= false; if (wlen < str->length()) { THD *thd= current_thd; From 952856c810c7a44678960a455062531279ddf113 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 3 Dec 2016 20:26:42 +0100 Subject: [PATCH 240/295] MDEV-11288 Server crashes in Binlog_crypt_data::init trying to feed encrypted log without decryption capabilities --- .../suite/binlog_encryption/disabled.def | 1 - ...rypted_master_switch_to_unencrypted.result | 85 +++++++++++++++++++ ...ncrypted_master_switch_to_unencrypted.test | 20 +++-- sql/encryption.cc | 5 ++ sql/sql_repl.cc | 24 +++--- 5 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result diff --git a/mysql-test/suite/binlog_encryption/disabled.def b/mysql-test/suite/binlog_encryption/disabled.def index 4fe25e7112586..27743e3e8ff30 100644 --- a/mysql-test/suite/binlog_encryption/disabled.def +++ b/mysql-test/suite/binlog_encryption/disabled.def @@ -1,4 +1,3 @@ -encrypted_master_switch_to_unencrypted : MDEV-11288 - server crash binlog_incident : MDEV-11319 - mysqlbinlog crash or failure encrypted_master_lost_key : MDEV-11323 - unspecified behavior for IO thread rpl_checksum_cache : MDEV-11486 - sporadic failure in IO thread diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result new file mode 100644 index 0000000000000..ec7bc4c79d6e6 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result @@ -0,0 +1,85 @@ +################# +# Initialization +################# +include/rpl_init.inc [topology=1->2] +connection server_2; +include/stop_slave.inc +##################################################### +# Part 1: unencrypted master +##################################################### +connection server_1; +call mtr.add_suppression("Got fatal error 1236 from master when reading data from binary log: 'Could not decrypt binlog: encryption key error;"); +CREATE TABLE table1_no_encryption ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table1_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; +NOT FOUND /table1_no_encryption/ in master-bin.0* +##################################################### +# Part 2: restart master, now with binlog encryption +##################################################### +connection default; +connection server_1; +CREATE TABLE table2_to_encrypt ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table2_to_encrypt VALUES (NULL,NOW(),'data_to_encrypt'); +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +FLUSH BINARY LOGS; +SET binlog_format=ROW; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +INSERT INTO table2_to_encrypt SELECT NULL,NOW(),b FROM table2_to_encrypt; +NOT FOUND /table2_to_encrypt/ in master-bin.0* +##################################################### +# Part 3: restart master again without encryption +##################################################### +connection default; +connection server_1; +CREATE TABLE table3_no_encryption ( +pk INT AUTO_INCREMENT PRIMARY KEY, +ts TIMESTAMP NULL, +b BLOB +) ENGINE=MyISAM; +INSERT INTO table3_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; +INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; +##################################################### +# Check: resume replication and check how it goes +##################################################### +connection server_2; +start slave; +include/wait_for_slave_io_error.inc [errno=1236] +connection server_2; +connection server_2; +connection server_2; +SHOW TABLES; +Tables_in_test +table1_no_encryption +include/stop_slave.inc +reset slave; +########## +# Cleanup +########## +connection server_1; +reset master; +SELECT COUNT(*) FROM table1_no_encryption; +COUNT(*) +8 +SELECT COUNT(*) FROM table2_to_encrypt; +COUNT(*) +8 +SELECT COUNT(*) FROM table3_no_encryption; +COUNT(*) +4 +DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption; +connection server_2; +include/start_slave.inc +include/rpl_end.inc diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test index 70133e30b6953..3f4289dcb4e64 100644 --- a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test @@ -35,6 +35,8 @@ --connection server_1 +call mtr.add_suppression("Got fatal error 1236 from master when reading data from binary log: 'Could not decrypt binlog: encryption key error;"); + CREATE TABLE table1_no_encryption ( pk INT AUTO_INCREMENT PRIMARY KEY, ts TIMESTAMP NULL, @@ -107,29 +109,31 @@ INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; --echo ##################################################### --connection server_2 ---disable_connect_log ---source include/start_slave.inc ---enable_connect_log ---sync_with_master +start slave; +--let slave_io_errno=1236 +--source include/wait_for_slave_io_error.inc --sorted_result SHOW TABLES; +--disable_connect_log +--source include/stop_slave.inc +--enable_connect_log +reset slave; + --echo ########## --echo # Cleanup --echo ########## --connection server_1 +reset master; SELECT COUNT(*) FROM table1_no_encryption; SELECT COUNT(*) FROM table2_to_encrypt; SELECT COUNT(*) FROM table3_no_encryption; DROP TABLE table1_no_encryption, table2_to_encrypt, table3_no_encryption; ---save_master_pos - --connection server_2 ---sync_with_master - --disable_connect_log +--source include/start_slave.inc --source include/rpl_end.inc diff --git a/sql/encryption.cc b/sql/encryption.cc index 52aaef896ddba..a92296e8b668d 100644 --- a/sql/encryption.cc +++ b/sql/encryption.cc @@ -29,6 +29,10 @@ uint no_key(uint) { return ENCRYPTION_KEY_VERSION_INVALID; } +uint zero_size(uint,uint) +{ + return 0; +} static int ctx_init(void *ctx, const unsigned char* key, unsigned int klen, const unsigned char* iv, unsigned int ivlen, int flags, @@ -97,6 +101,7 @@ int finalize_encryption_plugin(st_plugin_int *plugin) encryption_handler.encryption_key_get_func= (uint (*)(uint, uint, uchar*, uint*))no_key; encryption_handler.encryption_key_get_latest_version_func= no_key; + encryption_handler.encryption_ctx_size_func= zero_size; if (plugin && plugin->plugin->deinit && plugin->plugin->deinit(NULL)) { diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 572d08399d5db..0dd4c59ce5607 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2212,6 +2212,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, THD *thd= info->thd; String *packet= info->packet; Log_event_type event_type; + DBUG_ENTER("send_format_descriptor_event"); /** * 1) reset fdev before each log-file @@ -2226,12 +2227,12 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, { info->errmsg= "Out of memory initializing format_description event"; info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG; - return 1; + DBUG_RETURN(1); } /* reset transmit packet for the event read from binary log file */ if (reset_transmit_packet(info, info->flags, &ev_offset, &info->errmsg)) - return 1; + DBUG_RETURN(1); /* Try to find a Format_description_log_event at the beginning of @@ -2247,7 +2248,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, if (error) { set_read_error(info, error); - return 1; + DBUG_RETURN(1); } event_type= (Log_event_type)((uchar)(*packet)[LOG_EVENT_OFFSET+ev_offset]); @@ -2268,7 +2269,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, sql_print_warning("Failed to find format descriptor event in " "start of binlog: %s", info->log_file_name); - return 1; + DBUG_RETURN(1); } info->current_checksum_alg= get_checksum_alg(packet->ptr() + ev_offset, @@ -2288,7 +2289,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, sql_print_warning("Master is configured to log replication events " "with checksum, but will not send such events to " "slaves that cannot process them"); - return 1; + DBUG_RETURN(1); } uint ev_len= packet->length() - ev_offset; @@ -2302,7 +2303,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG; info->errmsg= "Corrupt Format_description event found " "or out-of-memory"; - return 1; + DBUG_RETURN(1); } delete info->fdev; info->fdev= tmp; @@ -2361,7 +2362,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, { info->errmsg= "Failed on my_net_write()"; info->error= ER_UNKNOWN_ERROR; - return 1; + DBUG_RETURN(1); } /* @@ -2381,7 +2382,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, if (error) { set_read_error(info, error); - return 1; + DBUG_RETURN(1); } event_type= (Log_event_type)((uchar)(*packet)[LOG_EVENT_OFFSET]); @@ -2393,14 +2394,15 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, if (!sele) { info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG; - return 1; + DBUG_RETURN(1); } if (info->fdev->start_decryption(sele)) { info->error= ER_MASTER_FATAL_ERROR_READING_BINLOG; info->errmsg= "Could not decrypt binlog: encryption key error"; - return 1; + delete sele; + DBUG_RETURN(1); } delete sele; } @@ -2416,7 +2418,7 @@ static int send_format_descriptor_event(binlog_send_info *info, IO_CACHE *log, /** all done */ - return 0; + DBUG_RETURN(0); } static bool should_stop(binlog_send_info *info) From b5aa0f437fc595d508b9eb5d36185fd8cbaa62eb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 3 Dec 2016 20:34:50 +0100 Subject: [PATCH 241/295] MDEV-11319 mysqlbinlog crashes or fails with out of memory while reading some encrypted binlogs support encrypted binlogs. Not decryption, but at least recognizing that event are encrypted and prining them as such --- client/mysqlbinlog.cc | 12 ++++++++++- .../binlog_encryption/mysqlbinlog.result | 6 ++++++ .../suite/binlog_encryption/mysqlbinlog.test | 21 +++++++++++++++++++ sql/log_event.cc | 13 ++++++++++-- sql/log_event.h | 7 +++++-- 5 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 mysql-test/suite/binlog_encryption/mysqlbinlog.result create mode 100644 mysql-test/suite/binlog_encryption/mysqlbinlog.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index da41268ee1978..e27a4a8e4e33e 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1257,6 +1257,9 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, goto err; break; } + case START_ENCRYPTION_EVENT: + glob_description_event->start_decryption((Start_encryption_log_event*)ev); + /* fall through */ default: print_skip_replication_statement(print_event_info, ev); ev->print(result_file, print_event_info); @@ -2837,9 +2840,16 @@ void *sql_alloc(size_t size) return alloc_root(&s_mem_root, size); } +uint dummy1() { return 1; } struct encryption_service_st encryption_handler= { - 0, 0, 0, 0, 0, 0, 0 + (uint(*)(uint))dummy1, + (uint(*)(uint, uint, uchar*, uint*))dummy1, + (uint(*)(uint, uint))dummy1, + (int (*)(void*, const uchar*, uint, const uchar*, uint, int, uint, uint))dummy1, + (int (*)(void*, const uchar*, uint, uchar*, uint*))dummy1, + (int (*)(void*, uchar*, uint*))dummy1, + (uint (*)(uint, uint, uint))dummy1 }; /* diff --git a/mysql-test/suite/binlog_encryption/mysqlbinlog.result b/mysql-test/suite/binlog_encryption/mysqlbinlog.result new file mode 100644 index 0000000000000..71758f7d6e71b --- /dev/null +++ b/mysql-test/suite/binlog_encryption/mysqlbinlog.result @@ -0,0 +1,6 @@ +RESET MASTER; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +REPLACE INTO t1 VALUES (4); +DROP TABLE t1; +FLUSH LOGS; diff --git a/mysql-test/suite/binlog_encryption/mysqlbinlog.test b/mysql-test/suite/binlog_encryption/mysqlbinlog.test new file mode 100644 index 0000000000000..b80388aaa4561 --- /dev/null +++ b/mysql-test/suite/binlog_encryption/mysqlbinlog.test @@ -0,0 +1,21 @@ +source include/have_log_bin.inc; +source include/have_debug.inc; + +let datadir=`select @@datadir`; +RESET MASTER; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2),(3); +REPLACE INTO t1 VALUES (4); +DROP TABLE t1; +FLUSH LOGS; + +let filename= master-bin.000001; +let local=$datadir/$filename; +let remote=--read-from-remote-server --protocol=tcp --host=127.0.0.1 --port=$MASTER_MYPORT -uroot $filename; +let outfile=$MYSQLTEST_VARDIR/tmp/binlog_enc.sql; +--error 1 +exec $MYSQL_BINLOG $local > $outfile; +exec $MYSQL_BINLOG $local --force-read >> $outfile; +exec $MYSQL_BINLOG $remote >> $outfile; +remove_file $outfile; + diff --git a/sql/log_event.cc b/sql/log_event.cc index de6528638f02e..ced262657fbae 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1514,6 +1514,10 @@ Log_event* Log_event::read_log_event(IO_CACHE* file, mysql_mutex_t* log_lock, if (error) { DBUG_ASSERT(!res); +#ifdef MYSQL_CLIENT + if (force_opt) + DBUG_RETURN(new Unknown_log_event()); +#endif if (event.length() >= OLD_HEADER_LEN) sql_print_error("Error in Log_event::read_log_event(): '%s'," " data_len: %lu, event_type: %d", error, @@ -8182,8 +8186,13 @@ void Unknown_log_event::print(FILE* file_arg, PRINT_EVENT_INFO* print_event_info if (print_event_info->short_form) return; - print_header(&cache, print_event_info, FALSE); - my_b_printf(&cache, "\n# %s", "Unknown event\n"); + if (what != ENCRYPTED) + { + print_header(&cache, print_event_info, FALSE); + my_b_printf(&cache, "\n# Unknown event\n"); + } + else + my_b_printf(&cache, "# Encrypted event\n"); } #endif diff --git a/sql/log_event.h b/sql/log_event.h index bc850c22a146d..90900f63533b5 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -1149,7 +1149,7 @@ class Log_event return thd ? thd->db : 0; } #else - Log_event() : temp_buf(0), flags(0) {} + Log_event() : temp_buf(0), when(0), flags(0) {} ha_checksum crc; /* print*() functions are used by mysqlbinlog */ virtual void print(FILE* file, PRINT_EVENT_INFO* print_event_info) = 0; @@ -3677,6 +3677,7 @@ class Execute_load_query_log_event: public Query_log_event class Unknown_log_event: public Log_event { public: + enum { UNKNOWN, ENCRYPTED } what; /* Even if this is an unknown event, we still pass description_event to Log_event's ctor, this way we can extract maximum information from the @@ -3684,8 +3685,10 @@ class Unknown_log_event: public Log_event */ Unknown_log_event(const char* buf, const Format_description_log_event *description_event): - Log_event(buf, description_event) + Log_event(buf, description_event), what(UNKNOWN) {} + /* constructor for hopelessly corrupted events */ + Unknown_log_event(): Log_event(), what(ENCRYPTED) {} ~Unknown_log_event() {} void print(FILE* file, PRINT_EVENT_INFO* print_event_info); Log_event_type get_type_code() { return UNKNOWN_EVENT;} From 5142cd55f492a2c9a2e308e168c86278765f95c3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 4 Dec 2016 21:19:32 +0100 Subject: [PATCH 242/295] MDEV-11052 mariadb-service-convert does not work after upgrading to 10.1.18 mysqld_safe: don't close stdout and stderr if --dry-run --- scripts/mysqld_safe.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index d4d2a0a6c3724..0a9f411717c8d 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -738,10 +738,6 @@ else logging=syslog fi -# close stdout and stderr, everything goes to $logging now -exec 1>&- -exec 2>&- - USER_OPTION="" if test -w / -o "$USER" = "root" then @@ -987,9 +983,14 @@ do done cmd="$cmd $args" [ $dry_run -eq 1 ] && return + # Avoid 'nohup: ignoring input' warning test -n "$NOHUP_NICENESS" && cmd="$cmd < /dev/null" +# close stdout and stderr, everything goes to $logging now +exec 1>&- +exec 2>&- + log_notice "Starting $MYSQLD daemon with databases from $DATADIR" # variable to track the current number of "fast" (a.k.a. subsecond) restarts From 76546a099c24a06ad89238f61435051a4663b112 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 5 Dec 2016 15:51:24 +0100 Subject: [PATCH 243/295] MDEV-10382 Using systemd, mariadb doesn't restart on crashes when crashing on a signal, don't exit(), but re-signal it, so that the caller could check WIFSIGNALED() --- sql/signal_handler.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index f72eb6767438a..6593c3f3dbc94 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -75,7 +75,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) if (segfaulted) { my_safe_printf_stderr("Fatal " SIGNAL_FMT " while backtracing\n", sig); - _exit(1); /* Quit without running destructors */ + goto end; } segfaulted = 1; @@ -301,9 +301,11 @@ extern "C" sig_handler handle_fatal_signal(int sig) #ifndef __WIN__ /* Quit, without running destructors (etc.) + Use a signal, because the parent (systemd) can check that with WIFSIGNALED On Windows, do not terminate, but pass control to exception filter. */ - _exit(1); // Using _exit(), since exit() is not async signal safe + signal(sig, SIG_DFL); + kill(getpid(), sig); #else return; #endif From 74d52ded721d8d39bfb25d18a0b0b8ea8a7af884 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 5 Dec 2016 22:29:25 +0100 Subject: [PATCH 244/295] fix binlog_encryption.binlog_incident test --- .../binlog_tests}/binlog_incident-master.opt | 0 mysql-test/extra/rpl_tests/multisource.inc | 11 +---------- mysql-test/include/binlog_start_pos.inc | 4 ++-- .../suite/binlog_encryption/binlog_incident.result | 2 +- .../binlog_encryption/binlog_row_annotate.result | 1 - mysql-test/suite/binlog_encryption/disabled.def | 1 - 6 files changed, 4 insertions(+), 15 deletions(-) rename mysql-test/{suite/binlog/t => extra/binlog_tests}/binlog_incident-master.opt (100%) diff --git a/mysql-test/suite/binlog/t/binlog_incident-master.opt b/mysql-test/extra/binlog_tests/binlog_incident-master.opt similarity index 100% rename from mysql-test/suite/binlog/t/binlog_incident-master.opt rename to mysql-test/extra/binlog_tests/binlog_incident-master.opt diff --git a/mysql-test/extra/rpl_tests/multisource.inc b/mysql-test/extra/rpl_tests/multisource.inc index 83a7e6ad259d6..2842aff4ef180 100644 --- a/mysql-test/extra/rpl_tests/multisource.inc +++ b/mysql-test/extra/rpl_tests/multisource.inc @@ -4,20 +4,11 @@ # Please check all dependent tests after modifying it # # Usage: -# --let $binlog_extra_length= X # optional, default 0 # --source extra/rpl_tests/multisource.inc # # By default, the script expects the length of the 2nd binary log to be # $binlog_start_pos + length(Gtid_list event) + 2 x length(Binlog_checkpoint event) # Some tests can have specific configuration which would change it, -# e.g. for encrypted binlogs there will be additional event -# Start_encryption of the length of 36. -# binlog_extra_length should compensate for the difference. - -if (!$binlog_extra_length) -{ - --let $binlog_extra_length= 0 -} # # Test basic replication functionality @@ -244,7 +235,7 @@ flush logs; --connection master1 purge binary logs to 'master-bin.000002'; # Additional events: 39 (Gtid_list) + 2 x 40 (Binlog_checkpoint) = 119 -let filesize=`select $binlog_start_pos+119+$binlog_extra_length`; +let filesize=`select $binlog_start_pos+119`; --replace_result $filesize filesize show binary logs; insert into t1 (f1) values ('four'); diff --git a/mysql-test/include/binlog_start_pos.inc b/mysql-test/include/binlog_start_pos.inc index a187e18b3a4cb..1fe2dd8d1715c 100644 --- a/mysql-test/include/binlog_start_pos.inc +++ b/mysql-test/include/binlog_start_pos.inc @@ -21,8 +21,8 @@ # ############################################################################## -let $binlog_start_pos=249; --disable_query_log -SET @binlog_start_pos=249; +set @binlog_start_pos=249 + @@encrypt_binlog * 36; --enable_query_log +let $binlog_start_pos=`select @binlog_start_pos`; diff --git a/mysql-test/suite/binlog_encryption/binlog_incident.result b/mysql-test/suite/binlog_encryption/binlog_incident.result index 6118f066f4adc..7a555743723df 100644 --- a/mysql-test/suite/binlog_encryption/binlog_incident.result +++ b/mysql-test/suite/binlog_encryption/binlog_incident.result @@ -10,4 +10,4 @@ REPLACE INTO t1 VALUES (4); DROP TABLE t1; FLUSH LOGS; Contain RELOAD DATABASE -0 +1 diff --git a/mysql-test/suite/binlog_encryption/binlog_row_annotate.result b/mysql-test/suite/binlog_encryption/binlog_row_annotate.result index d32b80b12497f..41721441a7261 100644 --- a/mysql-test/suite/binlog_encryption/binlog_row_annotate.result +++ b/mysql-test/suite/binlog_encryption/binlog_row_annotate.result @@ -8,7 +8,6 @@ ##################################################################################### show binlog events in 'master-bin.000001' from ; Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000001 # Start_encryption 1 # master-bin.000001 # Gtid_list 1 # [] master-bin.000001 # Binlog_checkpoint 1 # master-bin.000001 master-bin.000001 # Gtid 1 # GTID 0-1-1 diff --git a/mysql-test/suite/binlog_encryption/disabled.def b/mysql-test/suite/binlog_encryption/disabled.def index 27743e3e8ff30..b7a26a8343f3b 100644 --- a/mysql-test/suite/binlog_encryption/disabled.def +++ b/mysql-test/suite/binlog_encryption/disabled.def @@ -1,3 +1,2 @@ -binlog_incident : MDEV-11319 - mysqlbinlog crash or failure encrypted_master_lost_key : MDEV-11323 - unspecified behavior for IO thread rpl_checksum_cache : MDEV-11486 - sporadic failure in IO thread From 3ada31696918de27669b1ceb035131000dc53267 Mon Sep 17 00:00:00 2001 From: iangilfillan Date: Tue, 6 Dec 2016 13:18:48 +0200 Subject: [PATCH 245/295] Update mysqldump man page --- man/mysqldump.1 | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/man/mysqldump.1 b/man/mysqldump.1 index be5b855e3faa7..6495619f4e056 100644 --- a/man/mysqldump.1 +++ b/man/mysqldump.1 @@ -753,6 +753,29 @@ suppresses date printing\& .sp -1 .IP \(bu 2.3 .\} +.\" mysqldump: dump-slave option +.\" dump-slave option: mysqldump +\fB\-\-dump\-slave[=\fR\fB\fIvalue\fR\fR\fB]\fR +.sp +Used for producing a dump file from a replication slave server that can be used to set up another slave server +with the same master\&. Causes the binary log position and filename of the master to be appended to the dumped +data output\&. Setting the value to 1 (the default) will print it as a CHANGE MASTER command in the dumped data +output; if set to 2, that command will be prefixed with a comment symbol\&. This option will turn +\-\-lock\-all\-tables on, unless \-\-single-transaction is specified too (in which case a global read lock is only +taken a short time at the beginning of the dump \- don't forget to read about \-\-single-transaction below)\&. In +all cases any action on logs will happen at the exact moment of the dump\&. Option automatically turns +\-\-lock\-tables off\&. Using this option causes mysqldump to stop the slave SQL thread before beginning the dump, +and restart it again after completion\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} .\" mysqldump: events option .\" events option: mysqldump \fB\-\-events\fR, From f988bcecfde819d3d3d2d7227789491fcc0ee430 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 21 Sep 2016 18:36:34 +0200 Subject: [PATCH 246/295] MDEV-10776: Server crash on query Exclude untouched in prepare phese subqueries from the select/unit tree because they became unreachable by execution. --- mysql-test/r/subselect.result | 11 +++++++++++ mysql-test/r/subselect_no_mat.result | 11 +++++++++++ mysql-test/r/subselect_no_opts.result | 11 +++++++++++ mysql-test/r/subselect_no_scache.result | 11 +++++++++++ mysql-test/r/subselect_no_semijoin.result | 11 +++++++++++ mysql-test/t/subselect.test | 14 ++++++++++++++ sql/item_subselect.cc | 22 +++++++++++++++++++--- sql/sql_lex.cc | 20 ++++++++++++++++++-- 8 files changed, 106 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 16ab4cf9d82db..0a599a64f80d1 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7094,3 +7094,14 @@ a 0 DROP TABLE t1; SET SESSION big_tables=0; +# +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index aed4fb39a84ca..b819b1e4ef9dc 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7091,6 +7091,17 @@ a 0 DROP TABLE t1; SET SESSION big_tables=0; +# +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; set optimizer_switch=default; select @@optimizer_switch like '%materialization=on%'; @@optimizer_switch like '%materialization=on%' diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index e1f08537788d8..e1001a5165828 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7089,4 +7089,15 @@ a 0 DROP TABLE t1; SET SESSION big_tables=0; +# +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; set @optimizer_switch_for_subselect_test=null; diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index b5fa7b1dd8fc2..e175e7e007280 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7100,6 +7100,17 @@ a 0 DROP TABLE t1; SET SESSION big_tables=0; +# +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; set optimizer_switch=default; select @@optimizer_switch like '%subquery_cache=on%'; @@optimizer_switch like '%subquery_cache=on%' diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index b815559433c41..a211d498762e9 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7089,5 +7089,16 @@ a 0 DROP TABLE t1; SET SESSION big_tables=0; +# +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; set @optimizer_switch_for_subselect_test=null; set @join_cache_level_for_subselect_test=NULL; diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index b802761aff368..77b6c6c55822f 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -5974,3 +5974,17 @@ INSERT INTO t1 VALUES(0),(0),(0); SELECT * FROM t1 WHERE a IN(SELECT MIN(a) FROM t1); DROP TABLE t1; SET SESSION big_tables=0; + + +--echo # +--echo # MDEV-10776: Server crash on query +--echo # +create table t1 (field1 int); + +insert into t1 values (1); + +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); + +drop table t1; diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 3727711a395cc..6427b0ecae407 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -209,6 +209,7 @@ bool Item_subselect::select_transformer(JOIN *join) { DBUG_ENTER("Item_subselect::select_transformer"); + DBUG_ASSERT(thd == join->thd); DBUG_RETURN(false); } @@ -579,7 +580,7 @@ bool Item_subselect::is_expensive() examined_rows+= cur_join->get_examined_rows(); } - + // here we are sure that subquery is optimized so thd is set return (examined_rows > thd->variables.expensive_subquery_limit); } @@ -643,6 +644,7 @@ bool Item_subselect::exec() subselect_engine *org_engine= engine; DBUG_ENTER("Item_subselect::exec"); + DBUG_ASSERT(fixed); /* Do not execute subselect in case of a fatal error @@ -688,6 +690,7 @@ int Item_in_subselect::optimize(double *out_rows, double *cost) { int res; DBUG_ENTER("Item_in_subselect::optimize"); + DBUG_ASSERT(fixed); SELECT_LEX *save_select= thd->lex->current_select; JOIN *join= unit->first_select()->join; @@ -802,6 +805,7 @@ bool Item_in_subselect::expr_cache_is_needed(THD *thd) bool Item_in_subselect::exec() { DBUG_ENTER("Item_in_subselect::exec"); + DBUG_ASSERT(fixed); /* Initialize the cache of the left predicate operand. This has to be done as late as now, because Cached_item directly contains a resolved field (not @@ -856,6 +860,7 @@ table_map Item_subselect::used_tables() const bool Item_subselect::const_item() const { + DBUG_ASSERT(thd); return (thd->lex->context_analysis_only ? FALSE : forced_const || const_item_cache); @@ -1049,10 +1054,11 @@ Item_singlerow_subselect::select_transformer(JOIN *join) DBUG_ENTER("Item_singlerow_subselect::select_transformer"); if (changed) DBUG_RETURN(false); + DBUG_ASSERT(join->thd == thd); SELECT_LEX *select_lex= join->select_lex; Query_arena *arena= thd->stmt_arena; - + if (!select_lex->master_unit()->is_union() && !select_lex->table_list.elements && select_lex->item_list.elements == 1 && @@ -1717,6 +1723,7 @@ Item_in_subselect::single_value_transformer(JOIN *join) { SELECT_LEX *select_lex= join->select_lex; DBUG_ENTER("Item_in_subselect::single_value_transformer"); + DBUG_ASSERT(thd == join->thd); /* Check that the right part of the subselect contains no more than one @@ -1829,9 +1836,9 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join) if (!test_strategy(SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE)) DBUG_RETURN(false); Item **place= optimizer->arguments() + 1; - THD *thd= join->thd; SELECT_LEX *select_lex= join->select_lex; Item *subs; + DBUG_ASSERT(thd == join->thd); /* */ @@ -1938,6 +1945,7 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join) bool Item_in_subselect::fix_having(Item *having, SELECT_LEX *select_lex) { bool fix_res= 0; + DBUG_ASSERT(thd); if (!having->fixed) { select_lex->having_fix_field= 1; @@ -2000,6 +2008,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, Item **having_item) { SELECT_LEX *select_lex= join->select_lex; + DBUG_ASSERT(thd == join->thd); /* The non-transformed HAVING clause of 'join' may be stored in two ways during JOIN::optimize: this->tmp_having= this->having; this->having= 0; @@ -2136,6 +2145,7 @@ Item_in_subselect::row_value_transformer(JOIN *join) uint cols_num= left_expr->cols(); DBUG_ENTER("Item_in_subselect::row_value_transformer"); + DBUG_ASSERT(thd == join->thd); // psergey: duplicated_subselect_card_check if (select_lex->item_list.elements != cols_num) @@ -2248,6 +2258,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, !select_lex->table_list.elements); DBUG_ENTER("Item_in_subselect::create_row_in_to_exists_cond"); + DBUG_ASSERT(thd == join->thd); *where_item= NULL; *having_item= NULL; @@ -2473,6 +2484,7 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg) Item *having_item= join_arg->in_to_exists_having; DBUG_ENTER("Item_in_subselect::inject_in_to_exists_cond"); + DBUG_ASSERT(thd == join_arg->thd); if (where_item) { @@ -2561,6 +2573,7 @@ Item_in_subselect::select_in_like_transformer(JOIN *join) bool result; DBUG_ENTER("Item_in_subselect::select_in_like_transformer"); + DBUG_ASSERT(thd == join->thd); /* IN/SOME/ALL/ANY subqueries aren't support LIMIT clause. Without it @@ -2762,6 +2775,7 @@ bool Item_in_subselect::setup_mat_engine() subselect_single_select_engine *select_engine; DBUG_ENTER("Item_in_subselect::setup_mat_engine"); + DBUG_ASSERT(thd); /* The select_engine (that executes transformed IN=>EXISTS subselects) is @@ -2800,6 +2814,7 @@ bool Item_in_subselect::setup_mat_engine() bool Item_in_subselect::init_left_expr_cache() { JOIN *outer_join; + DBUG_ASSERT(thd); outer_join= unit->outer_select()->join; /* @@ -2826,6 +2841,7 @@ bool Item_in_subselect::init_left_expr_cache() bool Item_in_subselect::init_cond_guards() { + DBUG_ASSERT(thd); uint cols_num= left_expr->cols(); if (!abort_on_null && left_expr->maybe_null && !pushed_cond_guards) { diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 2462f0fea1764..f2e7b4f7c3a8d 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -3486,12 +3486,28 @@ bool st_select_lex::add_index_hint (THD *thd, char *str, uint length) bool st_select_lex::optimize_unflattened_subqueries(bool const_only) { - for (SELECT_LEX_UNIT *un= first_inner_unit(); un; un= un->next_unit()) + SELECT_LEX_UNIT *next_unit= NULL; + for (SELECT_LEX_UNIT *un= first_inner_unit(); + un; + un= next_unit ? next_unit : un->next_unit()) { Item_subselect *subquery_predicate= un->item; - + next_unit= NULL; + if (subquery_predicate) { + if (!subquery_predicate->fixed) + { + /* + This subquery was excluded as part of some expression so it is + invisible from all prepared expression. + */ + next_unit= un->next_unit(); + un->exclude_level(); + if (next_unit) + continue; + break; + } if (subquery_predicate->substype() == Item_subselect::IN_SUBS) { Item_in_subselect *in_subs= (Item_in_subselect*) subquery_predicate; From 035a5ac62a0215c2f6e3e363331e3e984d780138 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 26 Sep 2016 18:15:11 +0200 Subject: [PATCH 247/295] MDEV-10713: signal 11 error on multi-table update - crash in handler::increment_statistics or in make_select or assertion failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS))) Move expression execution out of Item constructor. --- mysql-test/r/sp.result | 38 ++++++++++++++++++++++++++++++++ mysql-test/t/sp.test | 50 ++++++++++++++++++++++++++++++++++++++++++ sql/item_strfunc.cc | 14 +++++++++++- sql/item_strfunc.h | 28 ++++++++++------------- 4 files changed, 113 insertions(+), 17 deletions(-) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index 6214fbcde3510..d15031989bf56 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8016,4 +8016,42 @@ Warnings: Error 1329 No data - zero rows fetched, selected, or processed DROP PROCEDURE p1; DROP TABLE t1; +# +# MDEV-10713: signal 11 error on multi-table update - crash in +# handler::increment_statistics or in make_select or assertion +# failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS))) +# +CREATE TABLE `t1` ( +`CLOSE_YN` varchar(10) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +CREATE TABLE `t2` ( +`ap_close_to` varchar(8) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +CREATE FUNCTION `f1`(`P_DC_CD` VARBINARY(50), `P_SYS_DATE` DATETIME) RETURNS datetime +DETERMINISTIC +SQL SECURITY INVOKER +BEGIN +DECLARE V_SYS_DATE DATETIME; +SELECT now() AS LOC_DATE INTO V_SYS_DATE ; +RETURN v_sys_date ; +END $$ +update t1 S +JOIN +( +SELECT CASE +WHEN DATE_FORMAT( f1('F01', NOW()) , '%Y%m%d') <= CLOSE_YMD +THEN '99991231' + ELSE '' END ACCOUNT_APPLY_YYYYMMDD +FROM ( +select case +when 'AP'='AP' + then ap_close_to +end AS CLOSE_YMD +from t2 +) A +) X +SET S.CLOSE_YN = '' +where 1=1; +drop function if exists f1; +drop table t1,t2; # End of 5.5 test diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index ecb1240865478..b56ab6c3b1178 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9302,4 +9302,54 @@ CALL p1; DROP PROCEDURE p1; DROP TABLE t1; +--echo # +--echo # MDEV-10713: signal 11 error on multi-table update - crash in +--echo # handler::increment_statistics or in make_select or assertion +--echo # failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS))) +--echo # + +CREATE TABLE `t1` ( + `CLOSE_YN` varchar(10) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; + + +CREATE TABLE `t2` ( + `ap_close_to` varchar(8) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; + + +--delimiter $$ + +CREATE FUNCTION `f1`(`P_DC_CD` VARBINARY(50), `P_SYS_DATE` DATETIME) RETURNS datetime + DETERMINISTIC + SQL SECURITY INVOKER +BEGIN + DECLARE V_SYS_DATE DATETIME; + SELECT now() AS LOC_DATE INTO V_SYS_DATE ; + RETURN v_sys_date ; +END $$ + +--delimiter ; + +update t1 S +JOIN +( + SELECT CASE + WHEN DATE_FORMAT( f1('F01', NOW()) , '%Y%m%d') <= CLOSE_YMD + THEN '99991231' + ELSE '' END ACCOUNT_APPLY_YYYYMMDD + FROM ( + select case + when 'AP'='AP' + then ap_close_to + end AS CLOSE_YMD + from t2 + ) A +) X +SET S.CLOSE_YN = '' +where 1=1; + +drop function if exists f1; +drop table t1,t2; + --echo # End of 5.5 test diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 94370d45cef6e..93e2edf9975de 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -2951,7 +2951,19 @@ String *Item_func_conv::val_str(String *str) String *Item_func_conv_charset::val_str(String *str) { DBUG_ASSERT(fixed == 1); - if (use_cached_value) + if (cached_value == CONST_WILL_BE_CACHED) + { + uint errors= 0; + String tmp, *str= args[0]->val_str(&tmp); + if (!str || str_value.copy(str->ptr(), str->length(), + str->charset(), conv_charset, &errors)) + null_value= 1; + cached_value= CACHED; + str_value.mark_as_const(); + safe= (errors == 0); + is_expensive_cache= 0; + } + if (cached_value == CACHED) return null_value ? 0 : &str_value; String *arg= args[0]->val_str(str); uint dummy_errors; diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 7606c2815484f..459dc5af34e94 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -817,31 +817,26 @@ class Item_func_quote :public Item_str_func class Item_func_conv_charset :public Item_str_func { - bool use_cached_value; String tmp_value; + enum state_of_cache { NOT_CONST, CONST_WILL_BE_CACHED, CACHED }; + enum state_of_cache cached_value; public: bool safe; CHARSET_INFO *conv_charset; // keep it public - Item_func_conv_charset(Item *a, CHARSET_INFO *cs) :Item_str_func(a) - { conv_charset= cs; use_cached_value= 0; safe= 0; } - Item_func_conv_charset(Item *a, CHARSET_INFO *cs, bool cache_if_const) - :Item_str_func(a) + Item_func_conv_charset(Item *a, CHARSET_INFO *cs) :Item_str_func(a), + cached_value(NOT_CONST), safe(0), conv_charset(cs) + {} + Item_func_conv_charset(Item *a, CHARSET_INFO *cs, bool cache_if_const) + :Item_str_func(a), conv_charset(cs) { - conv_charset= cs; - if (cache_if_const && args[0]->const_item() && !args[0]->is_expensive()) + if (cache_if_const && args[0]->const_item()) { - uint errors= 0; - String tmp, *str= args[0]->val_str(&tmp); - if (!str || str_value.copy(str->ptr(), str->length(), - str->charset(), conv_charset, &errors)) - null_value= 1; - use_cached_value= 1; - str_value.mark_as_const(); - safe= (errors == 0); + is_expensive_cache= MY_TEST(args[0]->is_expensive()); + cached_value= CONST_WILL_BE_CACHED; } else { - use_cached_value= 0; + cached_value= NOT_CONST; /* Conversion from and to "binary" is safe. Conversion to Unicode is safe. @@ -892,6 +887,7 @@ class Item_func_conv_charset :public Item_str_func void fix_length_and_dec(); const char *func_name() const { return "convert"; } virtual void print(String *str, enum_query_type query_type); + virtual bool const_item() const { return cached_value != NOT_CONST; } }; class Item_func_set_collation :public Item_str_func From d67ef7a2fb3b52b3f61ce71dfe23cf4d610afc3c Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Mon, 5 Dec 2016 17:37:54 +0100 Subject: [PATCH 248/295] MDEV-10663: Use of Inline table columns in HAVING clause throws 1463 Error check for VIEW/DERIVED fields --- mysql-test/r/derived.result | 65 ++++++++++++++++++++++++++++++++ mysql-test/r/group_by.result | 14 +++++++ mysql-test/t/derived.test | 47 +++++++++++++++++++++++ mysql-test/t/group_by.test | 12 ++++++ sql/item.cc | 72 +++++++++++++++++++++--------------- 5 files changed, 181 insertions(+), 29 deletions(-) diff --git a/mysql-test/r/derived.result b/mysql-test/r/derived.result index 2c316a829a763..a4d474c9cdfde 100644 --- a/mysql-test/r/derived.result +++ b/mysql-test/r/derived.result @@ -924,3 +924,68 @@ id select_type table type possible_keys key key_len ref rows Extra 3 DERIVED NULL NULL NULL NULL NULL NULL NULL no matching row in const table 2 DERIVED NULL NULL NULL NULL NULL NULL NULL no matching row in const table DROP TABLES t1,t2; +# +# MDEV-10663: Use of Inline table columns in HAVING clause +# throws 1463 Error +# +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; +CREATE TABLE `example1463` ( +`Customer` varchar(255) NOT NULL, +`DeliveryStatus` varchar(255) NOT NULL, +`OrderSize` int(11) NOT NULL +); +INSERT INTO example1463 VALUES ('Charlie', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Success', 110); +INSERT INTO example1463 VALUES ('Charlie', 'Failed', 200); +INSERT INTO example1463 VALUES ('David', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Unknown', 100); +INSERT INTO example1463 VALUES ('Edward', 'Success', 150); +INSERT INTO example1463 VALUES ('Edward', 'Pending', 150); +SELECT Customer, Success, SUM(OrderSize) +FROM (SELECT Customer, +CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, +OrderSize +FROM example1463) as subQ +GROUP BY Success, Customer +WITH ROLLUP; +Customer Success SUM(OrderSize) +Charlie No 200 +David No 100 +Edward No 150 +NULL No 450 +Charlie Yes 100 +David Yes 210 +Edward Yes 150 +NULL Yes 460 +NULL NULL 910 +SELECT Customer, Success, SUM(OrderSize) +FROM (SELECT Customer, +CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, +OrderSize +FROM example1463) as subQ +GROUP BY Success, Customer; +Customer Success SUM(OrderSize) +Charlie No 200 +David No 100 +Edward No 150 +Charlie Yes 100 +David Yes 210 +Edward Yes 150 +SELECT Customer, Success, SUM(OrderSize) +FROM (SELECT Customer, +CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, +OrderSize +FROM example1463) as subQ +GROUP BY Success, Customer +HAVING Success IS NOT NULL; +Customer Success SUM(OrderSize) +Charlie No 200 +David No 100 +Edward No 150 +Charlie Yes 100 +David Yes 210 +Edward Yes 150 +DROP TABLE example1463; +set sql_mode= @save_sql_mode; +# end of 5.5 diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result index 0b4973cc35b10..262bb2ebd849a 100644 --- a/mysql-test/r/group_by.result +++ b/mysql-test/r/group_by.result @@ -2517,3 +2517,17 @@ MAX(i) c 0 bar 7 foo drop table t1,t2; +# +# ONLY_FULL_GROUP_BY references +# +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY'; +create table t1 (a int, b int); +select a+b as x from t1 group by x having x > 1; +x +select a as x from t1 group by x having x > 1; +x +select a from t1 group by a having a > 1; +a +drop table t1; +set sql_mode= @save_sql_mode; diff --git a/mysql-test/t/derived.test b/mysql-test/t/derived.test index e10349b451c22..e8a6ac34392d5 100644 --- a/mysql-test/t/derived.test +++ b/mysql-test/t/derived.test @@ -796,3 +796,50 @@ A.DIVISION=C1.DIVISION AND A.RECEIVABLE_GROUP=C1.RECEIVABLE_GROUP AND A.CREDIT_L ORDER BY TOTAL DESC; DROP TABLES t1,t2; + +--echo # +--echo # MDEV-10663: Use of Inline table columns in HAVING clause +--echo # throws 1463 Error +--echo # + +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; + +CREATE TABLE `example1463` ( + `Customer` varchar(255) NOT NULL, + `DeliveryStatus` varchar(255) NOT NULL, + `OrderSize` int(11) NOT NULL +); +INSERT INTO example1463 VALUES ('Charlie', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Success', 110); +INSERT INTO example1463 VALUES ('Charlie', 'Failed', 200); +INSERT INTO example1463 VALUES ('David', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Unknown', 100); +INSERT INTO example1463 VALUES ('Edward', 'Success', 150); +INSERT INTO example1463 VALUES ('Edward', 'Pending', 150); + +SELECT Customer, Success, SUM(OrderSize) + FROM (SELECT Customer, + CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, + OrderSize + FROM example1463) as subQ + GROUP BY Success, Customer + WITH ROLLUP; +SELECT Customer, Success, SUM(OrderSize) + FROM (SELECT Customer, + CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, + OrderSize + FROM example1463) as subQ + GROUP BY Success, Customer; +SELECT Customer, Success, SUM(OrderSize) + FROM (SELECT Customer, + CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, + OrderSize + FROM example1463) as subQ + GROUP BY Success, Customer + HAVING Success IS NOT NULL; + +DROP TABLE example1463; +set sql_mode= @save_sql_mode; + +--echo # end of 5.5 diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test index c94d27b1d16db..4162e9c67a19e 100644 --- a/mysql-test/t/group_by.test +++ b/mysql-test/t/group_by.test @@ -1691,6 +1691,18 @@ SELECT MAX(i), c FROM t1 WHERE c != 'qux' AND ( SELECT SUM(j) FROM t1, t2 ) IS NOT NULL GROUP BY c; drop table t1,t2; +--echo # +--echo # ONLY_FULL_GROUP_BY references +--echo # + +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY'; +create table t1 (a int, b int); +select a+b as x from t1 group by x having x > 1; +select a as x from t1 group by x having x > 1; +select a from t1 group by a having a > 1; +drop table t1; +set sql_mode= @save_sql_mode; # # End of MariaDB 5.5 tests # diff --git a/sql/item.cc b/sql/item.cc index ede1df6aa9b95..53666aaf83da2 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -4556,8 +4556,6 @@ static Item** find_field_in_group_list(Item *find_item, ORDER *group_list) const char *field_name; ORDER *found_group= NULL; int found_match_degree= 0; - Item_ident *cur_field; - int cur_match_degree= 0; char name_buff[SAFE_NAME_LEN+1]; if (find_item->type() == Item::FIELD_ITEM || @@ -4582,54 +4580,70 @@ static Item** find_field_in_group_list(Item *find_item, ORDER *group_list) for (ORDER *cur_group= group_list ; cur_group ; cur_group= cur_group->next) { - if ((*(cur_group->item))->real_item()->type() == Item::FIELD_ITEM) + int cur_match_degree= 0; + + /* SELECT list element with explicit alias */ + if ((*(cur_group->item))->name && + !(*(cur_group->item))->is_autogenerated_name && + !my_strcasecmp(system_charset_info, + (*(cur_group->item))->name, field_name)) { - cur_field= (Item_ident*) *cur_group->item; - cur_match_degree= 0; - - DBUG_ASSERT(cur_field->field_name != 0); + ++cur_match_degree; + } + /* Reference on the field or view/derived field. */ + else if ((*(cur_group->item))->type() == Item::FIELD_ITEM || + (*(cur_group->item))->type() == Item::REF_ITEM ) + { + Item_ident *cur_field= (Item_ident*) *cur_group->item; + const char *l_db_name= cur_field->db_name; + const char *l_table_name= cur_field->table_name; + const char *l_field_name= cur_field->field_name; + + DBUG_ASSERT(l_field_name != 0); if (!my_strcasecmp(system_charset_info, - cur_field->field_name, field_name)) + l_field_name, field_name)) ++cur_match_degree; else continue; - if (cur_field->table_name && table_name) + if (l_table_name && table_name) { /* If field_name is qualified by a table name. */ - if (my_strcasecmp(table_alias_charset, cur_field->table_name, table_name)) + if (my_strcasecmp(table_alias_charset, l_table_name, table_name)) /* Same field names, different tables. */ return NULL; ++cur_match_degree; - if (cur_field->db_name && db_name) + if (l_db_name && db_name) { /* If field_name is also qualified by a database name. */ - if (strcmp(cur_field->db_name, db_name)) + if (strcmp(l_db_name, db_name)) /* Same field names, different databases. */ return NULL; ++cur_match_degree; } } + } + else + continue; - if (cur_match_degree > found_match_degree) - { - found_match_degree= cur_match_degree; - found_group= cur_group; - } - else if (found_group && (cur_match_degree == found_match_degree) && - ! (*(found_group->item))->eq(cur_field, 0)) - { - /* - If the current resolve candidate matches equally well as the current - best match, they must reference the same column, otherwise the field - is ambiguous. - */ - my_error(ER_NON_UNIQ_ERROR, MYF(0), - find_item->full_name(), current_thd->where); - return NULL; - } + if (cur_match_degree > found_match_degree) + { + found_match_degree= cur_match_degree; + found_group= cur_group; + } + else if (found_group && (cur_match_degree == found_match_degree) && + !(*(found_group->item))->eq((*(cur_group->item)), 0)) + { + /* + If the current resolve candidate matches equally well as the current + best match, they must reference the same column, otherwise the field + is ambiguous. + */ + my_error(ER_NON_UNIQ_ERROR, MYF(0), + find_item->full_name(), current_thd->where); + return NULL; } } From 63edd271673dbd0e6b0d3fffc13f0ab63d5e0631 Mon Sep 17 00:00:00 2001 From: iangilfillan Date: Wed, 7 Dec 2016 11:58:40 +0200 Subject: [PATCH 249/295] Update mysqld_multi man page --- man/mysqld_multi.1 | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/man/mysqld_multi.1 b/man/mysqld_multi.1 index d40ecfff3db29..bb0da44bc7e70 100644 --- a/man/mysqld_multi.1 +++ b/man/mysqld_multi.1 @@ -1,6 +1,6 @@ '\" t .\" -.TH "\FBMYSQLD_MULTI\FR" "1" "14/12/2015" "MariaDB 10\&.1" "MariaDB Database System" +.TH "\FBMYSQLD_MULTI\FR" "1" "7 December 2016" "MariaDB 10\&.1" "MariaDB Database System" .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- @@ -419,6 +419,21 @@ Be more verbose\&. .sp Display version information and exit\&. .RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysqld_multi: wsrep-new-cluster option +.\" wsrep-new-cluster option: mysqld_multi +\fB\-\-wsrep\-new\-cluster\fR +.sp +Bootstrap a cluster\&. +.RE .PP Some notes about \fBmysqld_multi\fR: @@ -653,7 +668,7 @@ user = jani .SH "COPYRIGHT" .br .PP -Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2015 MariaDB Foundation +Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2016 MariaDB Foundation .PP This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. .PP From 1d702ff07c53770f45086c514f34bf6ec4e8b299 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Wed, 7 Dec 2016 14:42:08 +0400 Subject: [PATCH 250/295] MDEV-8329 MariaDB crashes when replicate_wild_ignore_table is set to NULL. Rpl_filter::parse_filter_rule() made NULL-safe. --- mysql-test/suite/sys_vars/r/replicate_do_db_basic.result | 4 ++++ mysql-test/suite/sys_vars/r/replicate_do_table_basic.result | 4 ++++ mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result | 4 ++++ .../suite/sys_vars/r/replicate_ignore_table_basic.result | 4 ++++ .../suite/sys_vars/r/replicate_wild_do_table_basic.result | 4 ++++ .../suite/sys_vars/r/replicate_wild_ignore_table_basic.result | 4 ++++ mysql-test/suite/sys_vars/t/replicate_do_db_basic.test | 3 +++ mysql-test/suite/sys_vars/t/replicate_do_table_basic.test | 3 +++ mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test | 3 +++ mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test | 3 +++ .../suite/sys_vars/t/replicate_wild_do_table_basic.test | 3 +++ .../suite/sys_vars/t/replicate_wild_ignore_table_basic.test | 3 +++ sql/rpl_filter.cc | 3 +++ 13 files changed, 45 insertions(+) diff --git a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result index b964d3d14a11c..a05b85a9bfda8 100644 --- a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result @@ -33,5 +33,9 @@ SET @@GLOBAL.replicate_do_db=""; SELECT @@GLOBAL.replicate_do_db; @@GLOBAL.replicate_do_db +SET @@GLOBAL.replicate_do_db=null; +SELECT @@GLOBAL.replicate_do_db; +@@GLOBAL.replicate_do_db + # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result index fac237228ac83..e67b1eeca0162 100644 --- a/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_do_table=""; SELECT @@GLOBAL.replicate_do_table; @@GLOBAL.replicate_do_table +SET @@GLOBAL.replicate_do_table=null; +SELECT @@GLOBAL.replicate_do_table; +@@GLOBAL.replicate_do_table + # Cleanup. SET @@GLOBAL.replicate_do_table = @save_replicate_do_table; diff --git a/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result b/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result index c4d7a37321e78..c7ff697b34f8a 100644 --- a/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result @@ -33,5 +33,9 @@ SET @@GLOBAL.replicate_ignore_db=""; SELECT @@GLOBAL.replicate_ignore_db; @@GLOBAL.replicate_ignore_db +SET @@GLOBAL.replicate_ignore_db=null; +SELECT @@GLOBAL.replicate_ignore_db; +@@GLOBAL.replicate_ignore_db + # Cleanup. SET @@GLOBAL.replicate_ignore_db = @save_replicate_ignore_db; diff --git a/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result index bc463d0731984..db97ce14c931b 100644 --- a/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_ignore_table=""; SELECT @@GLOBAL.replicate_ignore_table; @@GLOBAL.replicate_ignore_table +SET @@GLOBAL.replicate_ignore_table=null; +SELECT @@GLOBAL.replicate_ignore_table; +@@GLOBAL.replicate_ignore_table + # Cleanup. SET @@GLOBAL.replicate_ignore_table = @save_replicate_ignore_table; diff --git a/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result index 5647cc964fba0..8c55103080f52 100644 --- a/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_wild_do_table=""; SELECT @@GLOBAL.replicate_wild_do_table; @@GLOBAL.replicate_wild_do_table +SET @@GLOBAL.replicate_wild_do_table=null; +SELECT @@GLOBAL.replicate_wild_do_table; +@@GLOBAL.replicate_wild_do_table + # Cleanup. SET @@GLOBAL.replicate_wild_do_table = @save_replicate_wild_do_table; diff --git a/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result index c6829b792a433..0f46ce3880556 100644 --- a/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_wild_ignore_table=""; SELECT @@GLOBAL.replicate_wild_ignore_table; @@GLOBAL.replicate_wild_ignore_table +SET @@GLOBAL.replicate_wild_ignore_table=null; +SELECT @@GLOBAL.replicate_wild_ignore_table; +@@GLOBAL.replicate_wild_ignore_table + # Cleanup. SET @@GLOBAL.replicate_wild_ignore_table = @save_replicate_wild_ignore_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test index ccf50b1d6ab48..59d0176add2e9 100644 --- a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test @@ -35,5 +35,8 @@ SELECT @@GLOBAL.replicate_do_db; SET @@GLOBAL.replicate_do_db=""; SELECT @@GLOBAL.replicate_do_db; +SET @@GLOBAL.replicate_do_db=null; +SELECT @@GLOBAL.replicate_do_db; + --echo # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test index f3b1585613e77..346bdf3b038aa 100644 --- a/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_do_table; SET @@GLOBAL.replicate_do_table=""; SELECT @@GLOBAL.replicate_do_table; +SET @@GLOBAL.replicate_do_table=null; +SELECT @@GLOBAL.replicate_do_table; + --echo # Cleanup. SET @@GLOBAL.replicate_do_table = @save_replicate_do_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test b/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test index 3a0bc88109aae..376397d1635d0 100644 --- a/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test @@ -35,5 +35,8 @@ SELECT @@GLOBAL.replicate_ignore_db; SET @@GLOBAL.replicate_ignore_db=""; SELECT @@GLOBAL.replicate_ignore_db; +SET @@GLOBAL.replicate_ignore_db=null; +SELECT @@GLOBAL.replicate_ignore_db; + --echo # Cleanup. SET @@GLOBAL.replicate_ignore_db = @save_replicate_ignore_db; diff --git a/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test index aebe90732d2ec..56cf7f17c7f59 100644 --- a/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_ignore_table; SET @@GLOBAL.replicate_ignore_table=""; SELECT @@GLOBAL.replicate_ignore_table; +SET @@GLOBAL.replicate_ignore_table=null; +SELECT @@GLOBAL.replicate_ignore_table; + --echo # Cleanup. SET @@GLOBAL.replicate_ignore_table = @save_replicate_ignore_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test index b96a62f8dd197..832d3397f893c 100644 --- a/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_wild_do_table; SET @@GLOBAL.replicate_wild_do_table=""; SELECT @@GLOBAL.replicate_wild_do_table; +SET @@GLOBAL.replicate_wild_do_table=null; +SELECT @@GLOBAL.replicate_wild_do_table; + --echo # Cleanup. SET @@GLOBAL.replicate_wild_do_table = @save_replicate_wild_do_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test index 2900deab4d168..5cb1ff6c82047 100644 --- a/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_wild_ignore_table; SET @@GLOBAL.replicate_wild_ignore_table=""; SELECT @@GLOBAL.replicate_wild_ignore_table; +SET @@GLOBAL.replicate_wild_ignore_table=null; +SELECT @@GLOBAL.replicate_wild_ignore_table; + --echo # Cleanup. SET @@GLOBAL.replicate_wild_ignore_table = @save_replicate_wild_ignore_table; diff --git a/sql/rpl_filter.cc b/sql/rpl_filter.cc index 9103cad337dd0..6205c253ffc91 100644 --- a/sql/rpl_filter.cc +++ b/sql/rpl_filter.cc @@ -279,6 +279,9 @@ Rpl_filter::parse_filter_rule(const char* spec, Add_filter add) int status= 0; char *arg, *ptr, *pstr; + if (!spec) + return false; + if (! (ptr= my_strdup(spec, MYF(MY_WME)))) return true; From f5e0522d9261408aab8f7736ff9ab116b1e4358f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 7 Dec 2016 13:06:14 +0100 Subject: [PATCH 251/295] MDEV-10388 MariaDB 10.1.x keeps (deleted) ML* files in tmpdir after LOAD DATA completes truncate unused IO_CACHE backing store files in binlog_cache_data to release the disk space they were occupying --- sql/log.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/log.cc b/sql/log.cc index 077bbf5e71186..dc87c39730cce 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -263,6 +263,9 @@ class binlog_cache_data { compute_statistics(); truncate(0); + if(cache_log.file != -1) + my_chsize(cache_log.file, 0, 0, MYF(MY_WME)); + changes_to_non_trans_temp_table_flag= FALSE; incident= FALSE; before_stmt_pos= MY_OFF_T_UNDEF; From c32d3e16f37b0b3581ee03d4b3681add890112da Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 7 Dec 2016 18:05:13 +0400 Subject: [PATCH 252/295] MDEV-10787 Assertion `ltime->neg == 0' failed in void date_to_datetime(MYSQL_TIME*) --- mysql-test/r/func_time.result | 68 +++++++++++++++++++++++++++++++++++ mysql-test/t/func_time.test | 25 +++++++++++++ sql/sql_time.cc | 5 ++- 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 85ba5d73f3642..243a2b1f6d413 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -2778,5 +2778,73 @@ Warning 1292 Truncated incorrect time value: '-1441:00:00' Warning 1292 Truncated incorrect time value: '-1441:00:00' Warning 1292 Truncated incorrect time value: '-1441:00:00' # +# MDEV-10787 Assertion `ltime->neg == 0' failed in void date_to_datetime(MYSQL_TIME*) +# +CREATE TABLE t1 (d DATE); +INSERT INTO t1 VALUES ('2005-07-20'),('2012-12-21'); +SELECT REPLACE( ADDDATE( d, INTERVAL 0.6732771076944444 HOUR_SECOND ), '2', 'x' ) FROM t1; +REPLACE( ADDDATE( d, INTERVAL 0.6732771076944444 HOUR_SECOND ), '2', 'x' ) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT REPLACE( ADDDATE( d, INTERVAL '0.6732771076944444' HOUR_SECOND ), '2', 'x' ) FROM t1; +REPLACE( ADDDATE( d, INTERVAL '0.6732771076944444' HOUR_SECOND ), '2', 'x' ) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT CAST(ADDDATE( d, INTERVAL 6732771076944444 SECOND) AS CHAR) FROM t1; +CAST(ADDDATE( d, INTERVAL 6732771076944444 SECOND) AS CHAR) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT CAST(ADDDATE( d, INTERVAL '67327710769444:44' HOUR_SECOND) AS CHAR) FROM t1; +CAST(ADDDATE( d, INTERVAL '67327710769444:44' HOUR_SECOND) AS CHAR) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT CAST(ADDDATE( d, INTERVAL '673277107694:44:44' HOUR_SECOND) AS CHAR) FROM t1; +CAST(ADDDATE( d, INTERVAL '673277107694:44:44' HOUR_SECOND) AS CHAR) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +DROP TABLE t1; +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:23:59:59' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '3652423:23:59:59' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:59:59' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:59:59' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:59' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:59' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:0:315569433599' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:0:0:315569433599' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:0:0:315569433559' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '3652423:0:0:315569433559' DAY_SECOND) +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:0:315569433559' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:0:315569433559' DAY_SECOND) +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:315569433599' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:315569433599' DAY_SECOND) +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +# # End of 10.0 tests # diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index f2c53bd5ece12..7f0f8132eac87 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1705,6 +1705,31 @@ SELECT TIMEDIFF(20140101000000.000 , 20140302010000.000 ) AS dec_dec, TIMEDIFF(20140101000000.000 , '2014-03-02 01:00:00' ) AS dec_str; + +--echo # +--echo # MDEV-10787 Assertion `ltime->neg == 0' failed in void date_to_datetime(MYSQL_TIME*) +--echo # +CREATE TABLE t1 (d DATE); +INSERT INTO t1 VALUES ('2005-07-20'),('2012-12-21'); +SELECT REPLACE( ADDDATE( d, INTERVAL 0.6732771076944444 HOUR_SECOND ), '2', 'x' ) FROM t1; +SELECT REPLACE( ADDDATE( d, INTERVAL '0.6732771076944444' HOUR_SECOND ), '2', 'x' ) FROM t1; +SELECT CAST(ADDDATE( d, INTERVAL 6732771076944444 SECOND) AS CHAR) FROM t1; +SELECT CAST(ADDDATE( d, INTERVAL '67327710769444:44' HOUR_SECOND) AS CHAR) FROM t1; +SELECT CAST(ADDDATE( d, INTERVAL '673277107694:44:44' HOUR_SECOND) AS CHAR) FROM t1; +DROP TABLE t1; + +# Maximum possible DAY_SECOND values in various formats +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:23:59:59' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:59:59' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:59' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:0:315569433599' DAY_SECOND); + +# Out-of-range INTERVAL DAY_SECOND values +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:0:0:315569433559' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:0:315569433559' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:315569433599' DAY_SECOND); + + --echo # --echo # End of 10.0 tests --echo # diff --git a/sql/sql_time.cc b/sql/sql_time.cc index b55b1d76b99f1..1bd68e89ecbc0 100644 --- a/sql/sql_time.cc +++ b/sql/sql_time.cc @@ -930,7 +930,10 @@ bool date_add_interval(MYSQL_TIME *ltime, interval_type int_type, my_bool neg= 0; enum enum_mysql_timestamp_type time_type= ltime->time_type; - if ((ulong) interval.day > MAX_DAY_NUMBER) + if (((ulonglong) interval.day + + (ulonglong) interval.hour / 24 + + (ulonglong) interval.minute / 24 / 60 + + (ulonglong) interval.second / 24 / 60 / 60) > MAX_DAY_NUMBER) goto invalid_date; if (time_type != MYSQL_TIMESTAMP_TIME) From 822fb79799c50b6d630083b1baf5c35b98a0bdfc Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 7 Dec 2016 23:44:52 +0530 Subject: [PATCH 253/295] MDEV-11162 Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint) When JOIN::destroy() is called for a JOIN object that has - join->tmp_join != NULL - also has join->table[0]->sort then the latter was not cleaned up. This could cause a memory leak and/or asserts in the subsequent queries. Fixed by adding a cleanup call. --- mysql-test/r/group_by_innodb.result | 7 +++++++ mysql-test/t/group_by_innodb.test | 10 ++++++++++ sql/sql_select.cc | 1 + 3 files changed, 18 insertions(+) diff --git a/mysql-test/r/group_by_innodb.result b/mysql-test/r/group_by_innodb.result index 381e0d7493c0b..803efb8fd5ce9 100644 --- a/mysql-test/r/group_by_innodb.result +++ b/mysql-test/r/group_by_innodb.result @@ -123,4 +123,11 @@ id xtext optionen 2 number 22,25 1 select Kabel mit Stecker 5-polig,Kabel ohne Stecker DROP TABLE t1, t2; +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ); +( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ) +NULL +SELECT i FROM t1 order by i LIMIT 1; +i +DROP TABLE t1; # End of tests diff --git a/mysql-test/t/group_by_innodb.test b/mysql-test/t/group_by_innodb.test index e072a94fadaf5..074df222294e0 100644 --- a/mysql-test/t/group_by_innodb.test +++ b/mysql-test/t/group_by_innodb.test @@ -125,4 +125,14 @@ ORDER BY id DESC; DROP TABLE t1, t2; +--echo # +--echo # MDEV-11162: Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint) +--echo # + +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ); +SELECT i FROM t1 order by i LIMIT 1; +DROP TABLE t1; + --echo # End of tests + diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 2db9a2b8482c1..f345d3c96876c 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3144,6 +3144,7 @@ JOIN::destroy() */ tmp_table_param.cleanup(); tmp_join->tmp_table_param.copy_field= 0; + cleanup(1); DBUG_RETURN(tmp_join->destroy()); } cond_equal= 0; From 106664f8e86d694a9898c3e564bb72290f221bd6 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Thu, 8 Dec 2016 02:03:34 +0530 Subject: [PATCH 254/295] MDEV-11162 Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint) Updating result for the group_by_innodb.test --- mysql-test/r/group_by_innodb.result | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/r/group_by_innodb.result b/mysql-test/r/group_by_innodb.result index 803efb8fd5ce9..af68640c0b161 100644 --- a/mysql-test/r/group_by_innodb.result +++ b/mysql-test/r/group_by_innodb.result @@ -123,6 +123,9 @@ id xtext optionen 2 number 22,25 1 select Kabel mit Stecker 5-polig,Kabel ohne Stecker DROP TABLE t1, t2; +# +# MDEV-11162: Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint) +# CREATE TABLE t1 (i INT) ENGINE=InnoDB; SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ); ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ) From 8e702bce66a2fb9ed911800f87e75509ae963e8c Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Thu, 8 Dec 2016 17:05:01 +0200 Subject: [PATCH 255/295] MDEV-11504 binlog_encryption.encrypted_master_switch_to_unencrypted fails sporadically in buildbot The reason is a simple race condition. Initially the test was meant to synchronize with master before showing tables, but it turned out that the slave IO thread should fail by this point, and synchronization was removed along with a server bugfix. Now added an intermediate sync instead, to make sure that slave has replicated events before the point of failure --- ...crypted_master_switch_to_unencrypted.result | 2 ++ ...encrypted_master_switch_to_unencrypted.test | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result index ec7bc4c79d6e6..f9a063724caa0 100644 --- a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.result @@ -56,6 +56,8 @@ INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; ##################################################### connection server_2; start slave; +connection server_1; +connection server_2; include/wait_for_slave_io_error.inc [errno=1236] connection server_2; connection server_2; diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test index 3f4289dcb4e64..91231f89307ef 100644 --- a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test @@ -56,6 +56,10 @@ INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; --let SEARCH_PATTERN= table1_no_encryption --source include/search_pattern_in_file.inc +# We are storing the position now, because up to this point the slave +# should be able to synchronize with master +--save_master_pos + --echo ##################################################### --echo # Part 2: restart master, now with binlog encryption --echo ##################################################### @@ -102,14 +106,22 @@ INSERT INTO table3_no_encryption VALUES (NULL,NOW(),'data_no_encryption'); INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; INSERT INTO table3_no_encryption SELECT NULL,NOW(),b FROM table3_no_encryption; ---save_master_pos - --echo ##################################################### --echo # Check: resume replication and check how it goes --echo ##################################################### ---connection server_2 +--connection server_2 start slave; +# The slave should be able to synchronize with master up to +# the previously saved position (when the log was still unencrypted) +--sync_with_master + +--connection server_1 +# Now save the current position and make slave to try to syncrhonize. +# It shouldn't work, the slave IO thread is expected to abort with an error +--save_master_pos + +--connection server_2 --let slave_io_errno=1236 --source include/wait_for_slave_io_error.inc From 870d7589c6c4596b0f1a641add5071469e2c94a7 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Thu, 8 Dec 2016 20:49:54 +0200 Subject: [PATCH 256/295] MDEV-11491 binlog_encryption.rpl_checksum fails sporadically in buildbot The race condition happened if mark_xid_done was considerably delayed, and an extra Binlog_checkpoint event was written into the binary log which was later indicated in an error message. Fixed by ensuring that the event is written before the binary log is rotated to the one which is used in the output. --- mysql-test/extra/rpl_tests/rpl_checksum.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/extra/rpl_tests/rpl_checksum.inc b/mysql-test/extra/rpl_tests/rpl_checksum.inc index 14664fd585a30..8423d2fc1cb6e 100644 --- a/mysql-test/extra/rpl_tests/rpl_checksum.inc +++ b/mysql-test/extra/rpl_tests/rpl_checksum.inc @@ -87,6 +87,7 @@ create table t1 (a int); # testing that binlog rotation preserves opt_binlog_checksum value flush logs; flush logs; +-- source include/wait_for_binlog_checkpoint.inc flush logs; sync_slave_with_master; From ab65db6d3fc0c876130cefb45a63cdaad4018f8c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 8 Dec 2016 21:03:45 +0100 Subject: [PATCH 257/295] Revert "MDEV-10713: signal 11 error on multi-table update - crash in handler::increment_statistics or in make_select or assertion failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS)))" This reverts commit 035a5ac62a0215c2f6e3e363331e3e984d780138. Two minor problems and one regression: 1. caching the value in str_result. Other Item methods may use it, destroying the cache. See, for example, Item::save_in_field, where str_result is moved to use a local buffer (this failed main.grant) 2. Item_func_conv_charset::safe is now set too late, it's initialized only in val_str() but checked before that, this failed many tests in optimized builds. to fix 1 - use tmp_result instead of str_result, to fix 2, use the else branch in the Item_func_conv_charset constructor to set safe purely from charset properties. But this introduces a regression, constant strings can no longer be converted, say, from utf8 to latin1 (because 'safe' will be false). This fails few tests too. There is no way to fix it without reverting the commit and converting constants, as before, in the constructor. --- sql/item_strfunc.cc | 14 +------------- sql/item_strfunc.h | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 93e2edf9975de..94370d45cef6e 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -2951,19 +2951,7 @@ String *Item_func_conv::val_str(String *str) String *Item_func_conv_charset::val_str(String *str) { DBUG_ASSERT(fixed == 1); - if (cached_value == CONST_WILL_BE_CACHED) - { - uint errors= 0; - String tmp, *str= args[0]->val_str(&tmp); - if (!str || str_value.copy(str->ptr(), str->length(), - str->charset(), conv_charset, &errors)) - null_value= 1; - cached_value= CACHED; - str_value.mark_as_const(); - safe= (errors == 0); - is_expensive_cache= 0; - } - if (cached_value == CACHED) + if (use_cached_value) return null_value ? 0 : &str_value; String *arg= args[0]->val_str(str); uint dummy_errors; diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 459dc5af34e94..7606c2815484f 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -817,26 +817,31 @@ class Item_func_quote :public Item_str_func class Item_func_conv_charset :public Item_str_func { + bool use_cached_value; String tmp_value; - enum state_of_cache { NOT_CONST, CONST_WILL_BE_CACHED, CACHED }; - enum state_of_cache cached_value; public: bool safe; CHARSET_INFO *conv_charset; // keep it public - Item_func_conv_charset(Item *a, CHARSET_INFO *cs) :Item_str_func(a), - cached_value(NOT_CONST), safe(0), conv_charset(cs) - {} - Item_func_conv_charset(Item *a, CHARSET_INFO *cs, bool cache_if_const) - :Item_str_func(a), conv_charset(cs) + Item_func_conv_charset(Item *a, CHARSET_INFO *cs) :Item_str_func(a) + { conv_charset= cs; use_cached_value= 0; safe= 0; } + Item_func_conv_charset(Item *a, CHARSET_INFO *cs, bool cache_if_const) + :Item_str_func(a) { - if (cache_if_const && args[0]->const_item()) + conv_charset= cs; + if (cache_if_const && args[0]->const_item() && !args[0]->is_expensive()) { - is_expensive_cache= MY_TEST(args[0]->is_expensive()); - cached_value= CONST_WILL_BE_CACHED; + uint errors= 0; + String tmp, *str= args[0]->val_str(&tmp); + if (!str || str_value.copy(str->ptr(), str->length(), + str->charset(), conv_charset, &errors)) + null_value= 1; + use_cached_value= 1; + str_value.mark_as_const(); + safe= (errors == 0); } else { - cached_value= NOT_CONST; + use_cached_value= 0; /* Conversion from and to "binary" is safe. Conversion to Unicode is safe. @@ -887,7 +892,6 @@ class Item_func_conv_charset :public Item_str_func void fix_length_and_dec(); const char *func_name() const { return "convert"; } virtual void print(String *str, enum_query_type query_type); - virtual bool const_item() const { return cached_value != NOT_CONST; } }; class Item_func_set_collation :public Item_str_func From 03dabfa84d6bc9a8197c8d9fbe80f2a7f6a5b6ac Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 8 Dec 2016 22:54:58 +0100 Subject: [PATCH 258/295] MDEV-10713: signal 11 error on multi-table update - crash in handler::increment_statistics or in make_select or assertion failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS))) Different fix. Don't allow Item_func_sp to be evaluated unless all tables are prelocked. Extend the test case to make sure Item_func_sp::val_str is called (the table must have at least one row for that). --- mysql-test/r/sp.result | 1 + mysql-test/t/sp.test | 1 + sql/item.h | 2 +- sql/item_func.cc | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index d15031989bf56..823c6f78cee39 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8027,6 +8027,7 @@ CREATE TABLE `t1` ( CREATE TABLE `t2` ( `ap_close_to` varchar(8) COLLATE utf8_bin DEFAULT NULL ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +insert t1 values (1); CREATE FUNCTION `f1`(`P_DC_CD` VARBINARY(50), `P_SYS_DATE` DATETIME) RETURNS datetime DETERMINISTIC SQL SECURITY INVOKER diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index b56ab6c3b1178..0e42bf3c831cc 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9316,6 +9316,7 @@ CREATE TABLE `t1` ( CREATE TABLE `t2` ( `ap_close_to` varchar(8) COLLATE utf8_bin DEFAULT NULL ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +insert t1 values (1); --delimiter $$ diff --git a/sql/item.h b/sql/item.h index 41e6fe0e38e7e..89155ac00db1f 100644 --- a/sql/item.h +++ b/sql/item.h @@ -3294,7 +3294,7 @@ class Item_cache_wrapper :public Item_result_field if (result_type() == ROW_RESULT) orig_item->bring_value(); } - virtual bool is_expensive() { return orig_item->is_expensive(); } + bool is_expensive() { return orig_item->is_expensive(); } bool is_expensive_processor(uchar *arg) { return orig_item->is_expensive_processor(arg); } bool check_vcol_func_processor(uchar *arg) diff --git a/sql/item_func.cc b/sql/item_func.cc index 6c80c7d3d864c..89d3cd9e32a85 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -6778,7 +6778,8 @@ Item_func_sp::init_result_field(THD *thd) bool Item_func_sp::is_expensive() { - return !(m_sp->m_chistics->detistic); + return !m_sp->m_chistics->detistic || + current_thd->locked_tables_mode < LTM_LOCK_TABLES; } From 83f7151da5259fa07ab2c26600009d53adef50f3 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Fri, 9 Dec 2016 17:13:43 +0400 Subject: [PATCH 259/295] MDEV-10435 crash with bad stat tables. Functions from sql/statistics.cc don't seem to expect stat tables to fail or to have inadequate structure. Table open errors suppressed and some validity checks added. Invalid tables reported to the server log. --- mysql-test/r/statistics.result | 20 +++++ mysql-test/t/statistics.test | 24 +++++ sql/event_db_repository.cc | 14 +-- sql/rpl_gtid.cc | 14 +-- sql/sql_statistics.cc | 157 ++++++++++++++++++++++++++++++--- sql/sql_statistics.h | 9 +- sql/table.cc | 9 ++ sql/table.h | 10 +++ 8 files changed, 216 insertions(+), 41 deletions(-) diff --git a/mysql-test/r/statistics.result b/mysql-test/r/statistics.result index be2e39665b352..2d0f18f1ed76c 100644 --- a/mysql-test/r/statistics.result +++ b/mysql-test/r/statistics.result @@ -1676,4 +1676,24 @@ analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status Table is already up to date drop table t1; +# +# MDEV-10435 crash with bad stat tables +# +set use_stat_tables='preferably'; +call mtr.add_suppression("Column count of mysql.table_stats is wrong. Expected 3, found 1. The table is probably corrupted"); +rename table mysql.table_stats to test.table_stats; +flush tables; +create table t1 (a int); +rename table t1 to t2, t3 to t4; +ERROR 42S02: Table 'test.t3' doesn't exist +drop table t1; +rename table test.table_stats to mysql.table_stats; +rename table mysql.table_stats to test.table_stats; +create table mysql.table_stats (a int); +flush tables; +create table t1 (a int); +rename table t1 to t2, t3 to t4; +ERROR 42S02: Table 'test.t3' doesn't exist +drop table t1, mysql.table_stats; +rename table test.table_stats to mysql.table_stats; set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/t/statistics.test b/mysql-test/t/statistics.test index 805c169b2a423..3f08e2e133cb1 100644 --- a/mysql-test/t/statistics.test +++ b/mysql-test/t/statistics.test @@ -740,4 +740,28 @@ show variables like 'use_stat_tables'; analyze table t1; drop table t1; +--echo # +--echo # MDEV-10435 crash with bad stat tables +--echo # + +set use_stat_tables='preferably'; +call mtr.add_suppression("Column count of mysql.table_stats is wrong. Expected 3, found 1. The table is probably corrupted"); + +rename table mysql.table_stats to test.table_stats; +flush tables; +create table t1 (a int); +--error ER_NO_SUCH_TABLE +rename table t1 to t2, t3 to t4; +drop table t1; +rename table test.table_stats to mysql.table_stats; + +rename table mysql.table_stats to test.table_stats; +create table mysql.table_stats (a int); +flush tables; +create table t1 (a int); +--error ER_NO_SUCH_TABLE +rename table t1 to t2, t3 to t4; +drop table t1, mysql.table_stats; +rename table test.table_stats to mysql.table_stats; + set use_stat_tables=@save_use_stat_tables; diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc index 666f6e3e24cab..ba782a19f8c84 100644 --- a/sql/event_db_repository.cc +++ b/sql/event_db_repository.cc @@ -166,20 +166,8 @@ const TABLE_FIELD_TYPE event_table_fields[ET_FIELD_COUNT] = static const TABLE_FIELD_DEF event_table_def= {ET_FIELD_COUNT, event_table_fields, 0, (uint*) 0}; -class Event_db_intact : public Table_check_intact -{ -protected: - void report_error(uint, const char *fmt, ...) - { - va_list args; - va_start(args, fmt); - error_log_print(ERROR_LEVEL, fmt, args); - va_end(args); - } -}; - /** In case of an error, a message is printed to the error log. */ -static Event_db_intact table_intact; +static Table_check_intact_log_error table_intact; /** diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index f54ef2b0081a2..dfec97bf021c7 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -448,19 +448,7 @@ static const TABLE_FIELD_DEF mysql_gtid_slave_pos_tabledef= { mysql_rpl_slave_state_pk_parts }; -class Gtid_db_intact : public Table_check_intact -{ -protected: - void report_error(uint, const char *fmt, ...) - { - va_list args; - va_start(args, fmt); - error_log_print(ERROR_LEVEL, fmt, args); - va_end(args); - } -}; - -static Gtid_db_intact gtid_table_intact; +static Table_check_intact_log_error gtid_table_intact; /* Check that the mysql.gtid_slave_pos table has the correct definition. diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 27bc0fb4cd36b..62d56a700b805 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -129,6 +129,128 @@ inline void init_table_list_for_single_stat_table(TABLE_LIST *tbl, } +static Table_check_intact_log_error stat_table_intact; + +static const +TABLE_FIELD_TYPE table_stat_fields[TABLE_STAT_N_FIELDS] = +{ + { + { C_STRING_WITH_LEN("db_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("table_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("cardinality") }, + { C_STRING_WITH_LEN("bigint(21)") }, + { NULL, 0 } + }, +}; +static const uint table_stat_pk_col[]= {0,1}; +static const TABLE_FIELD_DEF +table_stat_def= {TABLE_STAT_N_FIELDS, table_stat_fields, 2, table_stat_pk_col }; + +static const +TABLE_FIELD_TYPE column_stat_fields[COLUMN_STAT_N_FIELDS] = +{ + { + { C_STRING_WITH_LEN("db_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("table_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("column_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("min_value") }, + { C_STRING_WITH_LEN("varbinary(255)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("max_value") }, + { C_STRING_WITH_LEN("varbinary(255)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("nulls_ratio") }, + { C_STRING_WITH_LEN("decimal(12,4)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("avg_length") }, + { C_STRING_WITH_LEN("decimal(12,4)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("avg_frequency") }, + { C_STRING_WITH_LEN("decimal(12,4)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("hist_size") }, + { C_STRING_WITH_LEN("tinyint(3)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("hist_type") }, + { C_STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB')") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("histogram") }, + { C_STRING_WITH_LEN("varbinary(255)") }, + { NULL, 0 } + } +}; +static const uint column_stat_pk_col[]= {0,1,2}; +static const TABLE_FIELD_DEF +column_stat_def= {COLUMN_STAT_N_FIELDS, column_stat_fields, 3, column_stat_pk_col}; + +static const +TABLE_FIELD_TYPE index_stat_fields[INDEX_STAT_N_FIELDS] = +{ + { + { C_STRING_WITH_LEN("db_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("table_name") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("index") }, + { C_STRING_WITH_LEN("varchar(64)") }, + { C_STRING_WITH_LEN("utf8") } + }, + { + { C_STRING_WITH_LEN("prefix_arity") }, + { C_STRING_WITH_LEN("int(11)") }, + { NULL, 0 } + }, + { + { C_STRING_WITH_LEN("avg_frequency") }, + { C_STRING_WITH_LEN("decimal(12,4)") }, + { NULL, 0 } + } +}; +static const uint index_stat_pk_col[]= {0,1,2,3}; +static const TABLE_FIELD_DEF +index_stat_def= {INDEX_STAT_N_FIELDS, index_stat_fields, 4, index_stat_pk_col}; + + /** @brief Open all statistical tables and lock them @@ -139,10 +261,30 @@ inline int open_stat_tables(THD *thd, TABLE_LIST *tables, Open_tables_backup *backup, bool for_write) { + int rc; + + Dummy_error_handler deh; // suppress errors + thd->push_internal_handler(&deh); init_table_list_for_stat_tables(tables, for_write); init_mdl_requests(tables); - return open_system_tables_for_read(thd, tables, backup); -} + rc= open_system_tables_for_read(thd, tables, backup); + thd->pop_internal_handler(); + + + /* If the number of tables changes, we should revise the check below. */ + DBUG_ASSERT(STATISTICS_TABLES == 3); + + if (!rc && + (stat_table_intact.check(tables[TABLE_STAT].table, &table_stat_def) || + stat_table_intact.check(tables[COLUMN_STAT].table, &column_stat_def) || + stat_table_intact.check(tables[INDEX_STAT].table, &index_stat_def))) + { + close_system_tables(thd, backup); + rc= 1; + } + + return rc; +} /** @@ -2725,10 +2867,7 @@ int update_statistics_for_table(THD *thd, TABLE *table) DEBUG_SYNC(thd, "statistics_update_start"); if (open_stat_tables(thd, tables, &open_tables_backup, TRUE)) - { - thd->clear_error(); DBUG_RETURN(rc); - } save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); @@ -3156,10 +3295,7 @@ int delete_statistics_for_table(THD *thd, LEX_STRING *db, LEX_STRING *tab) DBUG_ENTER("delete_statistics_for_table"); if (open_stat_tables(thd, tables, &open_tables_backup, TRUE)) - { - thd->clear_error(); DBUG_RETURN(rc); - } save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); @@ -3398,10 +3534,7 @@ int rename_table_in_stat_tables(THD *thd, LEX_STRING *db, LEX_STRING *tab, DBUG_ENTER("rename_table_in_stat_tables"); if (open_stat_tables(thd, tables, &open_tables_backup, TRUE)) - { - thd->clear_error(); - DBUG_RETURN(rc); - } + DBUG_RETURN(0); // not an error save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 20b2eb664492a..f46583839d1cf 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -52,7 +52,8 @@ enum enum_table_stat_col { TABLE_STAT_DB_NAME, TABLE_STAT_TABLE_NAME, - TABLE_STAT_CARDINALITY + TABLE_STAT_CARDINALITY, + TABLE_STAT_N_FIELDS }; enum enum_column_stat_col @@ -67,7 +68,8 @@ enum enum_column_stat_col COLUMN_STAT_AVG_FREQUENCY, COLUMN_STAT_HIST_SIZE, COLUMN_STAT_HIST_TYPE, - COLUMN_STAT_HISTOGRAM + COLUMN_STAT_HISTOGRAM, + COLUMN_STAT_N_FIELDS }; enum enum_index_stat_col @@ -76,7 +78,8 @@ enum enum_index_stat_col INDEX_STAT_TABLE_NAME, INDEX_STAT_INDEX_NAME, INDEX_STAT_PREFIX_ARITY, - INDEX_STAT_AVG_FREQUENCY + INDEX_STAT_AVG_FREQUENCY, + INDEX_STAT_N_FIELDS }; inline diff --git a/sql/table.cc b/sql/table.cc index 85ffd560992a4..686b3569a9702 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3861,6 +3861,15 @@ Table_check_intact::check(TABLE *table, const TABLE_FIELD_DEF *table_def) } +void Table_check_intact_log_error::report_error(uint, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + error_log_print(ERROR_LEVEL, fmt, args); + va_end(args); +} + + /** Traverse portion of wait-for graph which is reachable through edge represented by this flush ticket in search for deadlocks. diff --git a/sql/table.h b/sql/table.h index eb4076e02e2da..093c357e5750b 100644 --- a/sql/table.h +++ b/sql/table.h @@ -524,6 +524,16 @@ class Table_check_intact }; +/* + If the table isn't valid, report the error to the server log only. +*/ +class Table_check_intact_log_error : public Table_check_intact +{ +protected: + void report_error(uint, const char *fmt, ...); +}; + + /** Class representing the fact that some thread waits for table share to be flushed. Is used to represent information about From eb4f2e063c341d9f3644339c68cb01679e782001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sat, 10 Dec 2016 22:19:09 +0200 Subject: [PATCH 260/295] MDEV-11533: Roles with trailing white spaces are not cleared correctly Role names with trailing whitespaces are truncated in length as of 956e92d90873532fee95581c702f7b76643969ea to fix MDEV-8609. The problem is that the code that creates role mappings expects the string to be null terminated. Add the null terminator to account for that as well. In the future the rest of the code can be cleaned up to never assume c style strings but only LEX_STRINGS. --- .../suite/roles/create_and_drop_role.result | 16 ++++++++++++++++ mysql-test/suite/roles/create_and_drop_role.test | 12 ++++++++++++ sql/sql_yacc.yy | 1 + 3 files changed, 29 insertions(+) diff --git a/mysql-test/suite/roles/create_and_drop_role.result b/mysql-test/suite/roles/create_and_drop_role.result index d565b888c5f82..13631f935bf81 100644 --- a/mysql-test/suite/roles/create_and_drop_role.result +++ b/mysql-test/suite/roles/create_and_drop_role.result @@ -68,3 +68,19 @@ GRANT USAGE ON *.* TO 'r1'@'localhost' DROP USER u1; DROP ROLE r2; DROP USER r1@localhost; +create role 'foo '; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +concat(user, '__') is_role +foo__ Y +select * from mysql.roles_mapping; +Host User Role Admin_option +localhost root foo Y +drop role foo; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +concat(user, '__') is_role +select * from mysql.roles_mapping; +Host User Role Admin_option +show grants; +Grants for root@localhost +GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' WITH GRANT OPTION +GRANT PROXY ON ''@'%' TO 'root'@'localhost' WITH GRANT OPTION diff --git a/mysql-test/suite/roles/create_and_drop_role.test b/mysql-test/suite/roles/create_and_drop_role.test index 71d6de7053fa7..b6e5bd2b29712 100644 --- a/mysql-test/suite/roles/create_and_drop_role.test +++ b/mysql-test/suite/roles/create_and_drop_role.test @@ -95,3 +95,15 @@ SHOW GRANTS FOR r1@localhost; # Related to MDEV-7774, also caused a crash, by DROP USER u1; DROP ROLE r2; DROP USER r1@localhost; + +# +# MDEV-11533: Roles with trailing white spaces are not cleared correctly +# +create role 'foo '; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +select * from mysql.roles_mapping; +drop role foo; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +select * from mysql.roles_mapping; +--sorted_result +show grants; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c1cd2a9413027..c9fd000141af7 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -15178,6 +15178,7 @@ grant_role: CHARSET_INFO *cs= system_charset_info; /* trim end spaces (as they'll be lost in mysql.user anyway) */ $1.length= cs->cset->lengthsp(cs, $1.str, $1.length); + $1.str[$1.length] = '\0'; if ($1.length == 0) { my_error(ER_INVALID_ROLE, MYF(0), ""); From a629b5172e96c96c414fca70fffd64c80f2f7e8f Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sat, 10 Dec 2016 23:04:41 +0200 Subject: [PATCH 261/295] Updated the list of unstable tests --- mysql-test/unstable-tests | 182 ++++++++++++++++---------------------- 1 file changed, 74 insertions(+), 108 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index afd7a3c42c7ce..23c8bedeb00dd 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -26,56 +26,43 @@ main.alter_table : Modified in 10.1.19 main.analyze_stmt_slow_query_log : MDEV-7558 - wrong result main.create_delayed : MDEV-10605 - failed with timeout +main.create_drop_binlog : Uses binlog_start_pos.inc modified in 10.1.20 main.create_or_replace : Modified in 10.1.19 main.ctype_utf16le : MDEV-10675: timeout or extra warnings main.drop : Modified in 10.1.19 main.events_restart : MDEV-11221: assertion failure -main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_math : Modified on 2016-08-10 (merge) -main.func_misc : Modified on 2016-08-10 (merge) -main.grant2 : Modified on 2016-07-18 (MDEV-8569) main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout +main.index_merge_innodb : MDEV-7142 - Wrong execution plan main.information_schema : Modified in 10.1.19 -main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist -main.loaddata : Modified on 2016-08-10 (merge) +main.lowercase_fs_on : Uses search_pattern_in_file.inc modified in 10.1.20 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) main.mysql : Modified in 10.1.19 main.mysql_not_windows : Modified in 10.1.19 -main.mysqlcheck : Modified on 2016-08-10 (merge) +main.mysqlbinlog : Uses binlog_start_pos.inc modified in 10.1.20 +main.mysqldump-max : Uses binlog_start_pos.inc modified in 10.1.20 main.mysqldump-nl : Added in 10.1.19 main.mysqltest : MDEV-9269 - fails on Alpha; also modified in 10.1.19 -main.named_pipe : Modified on 2016-08-02 (MDEV-10383) +main.named_pipe : Uses search_pattern_in_file.inc modified in 10.1.20 main.null : Modified in 10.1.19 -main.openssl_1 : Modified on 2016-07-11 (MDEV-10211) main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.ps_1general : Modified on 2016-07-12 (merge) -main.range : Modified on 2016-08-10 (merge) -main.range_mrr_icp : Modified on 2016-08-10 (merge) main.selectivity : Modified in 10.1.19 main.selectivity_innodb : Modified in 10.1.19 main.show_explain : MDEV-10674 - sporadic failure -main.shutdown : MDEV-10563 - sporadic crashes -main.sp-prelocking : Modified on 2016-08-10 (merge) main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl_ca : Modified on 2016-07-11 (MDEV-10211) -main.ssl_compress : Modified on 2016-07-11 (MDEV-10211) -main.ssl_timeout : Modified on 2016-07-11 (MDEV-10211) main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results +main.statistics : Modified in 10.1.20 main.status : MDEV-8510 - sporadic wrong result main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_sj_mat : Modified on 2016-07-27 (MDEV-10389) -main.type_date : Modified on 2016-08-10 (merge) main.type_datetime_hires : MDEV-10687 - timeout -main.view : Modified on 2016-08-10 (merge) -main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) +main.view : Uses search_pattern_in_file.inc modified in 10.1.20 +main.wait_timeout_not_windows : Uses search_pattern_in_file.inc modified in 10.1.20 #---------------------------------------------------------------- @@ -85,6 +72,12 @@ archive.discover : MDEV-10510 - table is marked as crashed #---------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed +binlog.binlog_incident : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_killed : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_killed_simulate : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_mysqlbinlog2 : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.mix_innodb_myisam_binlog : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- @@ -94,29 +87,35 @@ binlog_encryption.* : Added in 10.1.20 #---------------------------------------------------------------- connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results -connect.jdbc : New test, added on 2016-07-15 -connect.jdbc-new : New test, added on 2016-07-14 -connect.jdbc-oracle : New test, added on 2016-07-13 -connect.jdbc-postgresql : New test, added on 2016-07-13 #---------------------------------------------------------------- encryption.create_or_replace : MDEV-9359 - Assertion failure -encryption.encrypt_and_grep : MDEV-11222 - InnoDB error +encryption.encrypt_and_grep : MDEV-11222 - InnoDB error; also uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_emptyfile : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_encfile_bad : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_encfile_badfile : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_encfile_no : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_nofile : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_syntax : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_tooshort : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_unencfile : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-bad-key-change : uses keys2.txt modified in 10.1.19 encryption.innodb-bad-key-change2 : uses keys2.txt modified in 10.1.19 -encryption.innodb-bad-key-change3 : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-change3 : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-bad-key-change4 : uses keys2.txt modified in 10.1.19 encryption.innodb-bad-key-change5 : uses keys2.txt modified in 10.1.19 encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures, and uses keys2.txt modified in 10.1.19 -encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also modified in 10.1.19 +encryption.innodb-discard-import : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb_encryption_filekeys : MDEV-9962 - timeouts -encryption.innodb_encryption_tables : MDEV-10970 - Crash/assertion failure +encryption.innodb_encryption_row_compressed : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb_first_page : MDEV-10689 - crashes +encryption.innodb-log-encrypt : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.innodb_lotoftables : MDEV-11531 - InnoDB error encryption.innodb-missing-key : Added in 10.1.19 -encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results +encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results; also uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-page_encryption : MDEV-10641 - mutex problem -encryption.innodb-page_encryption_compression : MDEV-10739 - timeouts encryption.innodb_page_encryption_key_change : uses keys2.txt modified in 10.1.19 #---------------------------------------------------------------- @@ -132,80 +131,51 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- -funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge) -funcs_1.processlist_priv_ps : Include file modified on 2016-07-12 (merge) funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result -funcs_2.memory_charset : MDEV-10290 - timeout +funcs_2.memory_charset : MDEV-10290 - Timeout +funcs_2.myisam_charset : MDEV-11535 - Timeout #---------------------------------------------------------------- -galera.GAL-382 : New test, added in 10.1.17 -galera.MW-252 : New test, added in 10.1.17 -galera.MW-258 : New test, added in 10.1.17 -galera.MW-259 : New test, added in 10.1.17 -galera.MW-285 : New test, added in 10.1.17 -galera.MW-286 : New test, added in 10.1.17 -galera.MW-292 : New test, added in 10.1.17 -galera.MW-44 : New test, added in 10.1.17 -galera.galera#414 : New test, added in 10.1.17 -galera.galera_as_slave_autoinc : New test, added in 10.1.17 -galera.galera_as_slave_nonprim : Modified in 10.1.17 -galera.galera_bf_abort_flush_for_export : Modified in 10.1.17 -galera.galera_gcs_fc_limit : Modified in 10.1.17 -galera.galera_ist_recv_bind : New test, added in 10.1.17 -galera.galera_ist_restart_joiner : Modified in 10.1.17 -galera.galera_kill_ddl : Modified in 10.1.17 -galera.galera_parallel_simple : Modified in 10.1.17 -galera.galera_pc_ignore_sb : Modified in 10.1.17 -galera.galera_restart_nochanges : Modified in 10.1.17 -galera.galera_roles : Modified in 10.1.17 -galera.galera_rsu_wsrep_desync : Modified in 10.1.17 -galera.galera_split_brain : Modified in 10.1.17 -galera.galera_ssl_upgrade : Modified in 10.1.17 -galera.galera_suspend_slave : Modified in 10.1.17 -galera.galera_transaction_replay : Modified in 10.1.17 -galera.galera_var_dirty_reads : Modified in 10.1.17 -galera.galera_var_max_ws_rows : New test, added in 10.1.17 -galera.galera_var_max_ws_size : Modified in 10.1.17 -galera.mdev_10518 : New test, added in 10.1.17 -galera.mysql-wsrep#31 : Modified in 10.1.17 - -galera_3nodes.galera_certification_ccc : Modified in 10.1.17 -galera_3nodes.galera_innobackupex_backup : Modified in 10.1.17 -galera_3nodes.galera_ist_gcache_rollover : Modified in 10.1.17 -galera_3nodes.galera_pc_bootstrap : Modified in 10.1.17 -galera_3nodes.galera_pc_weight : Modified in 10.1.17 - galera.* : Added to default suites in 10.1.19 -galera_3nodes.* : Added to default suites in 10.1.19 + +galera.galera_var_dirty_reads : Modified in 10.1.20 +galera.rpl_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 + +galera_3nodes.* : Added to default suites in 10.1.19, MDEV-11490 #---------------------------------------------------------------- -innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.binlog_consistent : MDEV-10618 - Server fails to start; also uses binlog_start_pos.inc modified in 10.1.20 +innodb.group_commit_binlog_pos : Uses binlog_start_pos.inc modified in 10.1.20 +innodb.group_commit_binlog_pos_no_optimize_thread : Uses binlog_start_pos.inc modified in 10.1.20 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) -innodb.innodb_blob_truncate : MDEV-10377 - Assertion failure innodb.innodb-bug-14068765 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb-bug-14084530 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb-fk-warnings : Modified on 2016-07-18 (MDEV-8569) +innodb.innodb-change-buffer-recovery : Uses search_pattern_in_file.inc modified in 10.1.20 +innodb.innodb_defragment_fill_factor : Modified in 10.1.20 +innodb.innodb-lock-schedule-algorithm : Modified in 10.1.20 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result innodb.innodb-wl5522 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb-wl5522-1 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb-wl5522-debug : modified in 10.1.19 -innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption, also modified in 10.1.19 innodb.innodb-wl5522-zip : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.trigger : Modified in 10.1.20 innodb.xa_recovery : Modified in 10.1.19 #---------------------------------------------------------------- -innodb_zip.innodb_prefix_index_liftedlimit : MDEV-10686 - timeout +innodb_fts.create : Added in 10.1.20 +innodb_fts.innodb_fts_misc : MDEV-11233 - Crash on CREATE FULLTEXT INDEX #---------------------------------------------------------------- +maria.collations : Added in 10.1.20 maria.encrypt-wrong-key : uses keys2.txt modified in 10.1.19 +maria.maria-connect : Uses binlog_start_pos.inc modified in 10.1.20 #---------------------------------------------------------------- @@ -222,9 +192,9 @@ mroonga/storage.repair_table_no_index_file : MDEV-9364 - multi_source.gtid : MDEV-10620, MDEV-10417 - Timeout in wait condition, fails on Mips multi_source.info_logs : MDEV-10042 - wrong result -multi_source.multisource : MDEV-10417 - Fails on Mips -multi_source.reset_slave : MDEV-10690 - wrong result -multi_source.simple : MDEV-4633 - Wrong slave status output +multi_source.multisource : MDEV-10417 - Fails on Mips; also uses binlog_start_pos.inc modified in 10.1.20 +multi_source.reset_slave : MDEV-10690 - wrong result; also uses binlog_start_pos.inc modified in 10.1.20 +multi_source.simple : MDEV-4633 - Wrong slave status output; also uses binlog_start_pos.inc modified in 10.1.20 multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_heartbeats #---------------------------------------------------------------- @@ -243,11 +213,14 @@ perfschema.threads_mysql : MDEV-10677 - sporadic wrong resul #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url -plugins.pam : Modified on 2016-08-03 (MDEV-7329) -plugins.pam_cleartext : Modified on 2016-08-03 plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc + +#---------------------------------------------------------------- + +roles.create_and_grant_role : MDEV-11533 - Extra grant in output + #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log @@ -255,6 +228,7 @@ rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering on master +rpl.rpl_checksum : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips @@ -262,6 +236,7 @@ rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result rpl.rpl_drop_db : Modified in 10.1.19 rpl.rpl_gtid_basic : MDEV-10681 - server startup problem rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master +rpl.rpl_gtid_errorlog : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_mdev9033 : MDEV-10680 - warnings rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown @@ -271,6 +246,7 @@ rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog +rpl.rpl_mdev10863 : Added in 10.1.20 rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts @@ -278,18 +254,22 @@ rpl.rpl_parallel_optimistic : MDEV-10511 - timeout rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips +rpl.rpl_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 +rpl.rpl_password_boundaries : MDEV-11534 - Slave IO warnings rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout +rpl.rpl_row_flsh_tbls : Uses binlog_start_pos.inc modified in 10.1.20 rpl.rpl_row_log_innodb : MDEV-10688 - Wrong result rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x -rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status +rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem +rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock -rpl.rpl_stop_slave_error : Modified in 10.1.19 +rpl.rpl_stm_flsh_tbls : Uses binlog_start_pos.inc modified in 10.1.20 +rpl.rpl_stop_slave_error : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master -rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha #---------------------------------------------------------------- @@ -317,40 +297,24 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x -sys_vars.general_log_file_basic : Modified on 2016-08-09 (MDEV-10465) sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout -sys_vars.slow_query_log_file_basic : Modified on 2016-08-09 (MDEV-10465) sys_vars.sysvars_innodb : MDEV-6958 - error-prone rdiffs sys_vars.sysvars_server_embedded : MDEV-6958 - error-prone rdiffs sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists sys_vars.innodb_fatal_semaphore_wait_threshold : MDEV-10513 - crashes -sys_vars.sysvars_wsrep : Modified in 10.1.17 -sys_vars.wsrep_max_ws_size_basic : Modified in 10.1.17 +sys_vars.table_open_cache_instances_basic : Modified in 10.1.20 #---------------------------------------------------------------- tokudb.cluster_filter : MDEV-10678 - Wrong execution plan tokudb.cluster_filter_hidden : MDEV-10678 - Wrong execution plan tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan -tokudb.i_s_tokudb_lock_waits_released : Modified in 10.1.17 -tokudb.i_s_tokudb_locks_released : Modified in 10.1.17 -tokudb.* : MDEV-9891 - massive crashes on shutdown - -tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown +tokudb.dir_per_db : MDEV-11537 - Wrong result tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output -tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown - -tokudb_parts.* : MDEV-9891 - massive crashes on shutdown - -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown -tokudb_backup.* : MDEV-9891 - massive crashes on shutdown -tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown -tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown -tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown +tokudb_rpl.rpl_parallel_optimistic : Added in 10.1.20 #---------------------------------------------------------------- @@ -363,8 +327,10 @@ vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout #---------------------------------------------------------------- -wsrep.mdev_10186 : Modified in 10.1.19 +wsrep.binlog_format : MDEV-11532 - WSREP has not yet prepared node +wsrep.mdev_10186 : Modified in 10.1.19 #---------------------------------------------------------------- -wsrep_info.* : suite.pm modified in 10.1.19 +wsrep_info.* : suite.pm modified in 10.1.19 +wsrep_info.plugin : MDEV-11530 - Warnings; also modified in 10.1.20 From 5d9ca52219e4a4c92fc4a80ffc09b5075b1e5616 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 12 Dec 2016 00:59:40 +0200 Subject: [PATCH 262/295] Updated the list of unstable tests after the merge --- mysql-test/unstable-tests | 223 ++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 115 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index afd7a3c42c7ce..7b5c2756f11c1 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -25,57 +25,59 @@ main.alter_table : Modified in 10.1.19 main.analyze_stmt_slow_query_log : MDEV-7558 - wrong result +main.create : Modified in 10.1.20 main.create_delayed : MDEV-10605 - failed with timeout +main.create_drop_binlog : Uses binlog_start_pos.inc modified in 10.1.20 main.create_or_replace : Modified in 10.1.19 main.ctype_utf16le : MDEV-10675: timeout or extra warnings +main.ctype_utf8 : Modified in 10.1.20 +main.ctype_utf8mb4 : Modified in 10.1.20 +main.default : Modified in 10.1.20 +main.derived : Modified in 10.1.20 +main.derived_view : Modified in 10.1.20 main.drop : Modified in 10.1.19 main.events_restart : MDEV-11221: assertion failure -main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_math : Modified on 2016-08-10 (merge) -main.func_misc : Modified on 2016-08-10 (merge) -main.grant2 : Modified on 2016-07-18 (MDEV-8569) +main.fulltext_charsets : Added in 10.1.20 +main.func_time : Modified in 10.1.20 +main.group_by : Modified in 10.1.20 +main.group_by_innodb : Modified in 10.1.20 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout +main.index_merge_innodb : MDEV-7142 - Wrong execution plan main.information_schema : Modified in 10.1.19 -main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist -main.loaddata : Modified on 2016-08-10 (merge) +main.loaddata : Modified in 10.1.20 +main.lowercase_fs_on : Uses search_pattern_in_file.inc modified in 10.1.20 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) main.mysql : Modified in 10.1.19 main.mysql_not_windows : Modified in 10.1.19 -main.mysqlcheck : Modified on 2016-08-10 (merge) +main.mysqlbinlog : Uses binlog_start_pos.inc modified in 10.1.20 +main.mysqldump-max : Uses binlog_start_pos.inc modified in 10.1.20 main.mysqldump-nl : Added in 10.1.19 main.mysqltest : MDEV-9269 - fails on Alpha; also modified in 10.1.19 -main.named_pipe : Modified on 2016-08-02 (MDEV-10383) +main.named_pipe : Uses search_pattern_in_file.inc modified in 10.1.20 main.null : Modified in 10.1.19 -main.openssl_1 : Modified on 2016-07-11 (MDEV-10211) main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan +main.parser : Modified in 10.1.20 main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.ps_1general : Modified on 2016-07-12 (merge) -main.range : Modified on 2016-08-10 (merge) -main.range_mrr_icp : Modified on 2016-08-10 (merge) -main.selectivity : Modified in 10.1.19 +main.selectivity : Modified in 10.1.20 main.selectivity_innodb : Modified in 10.1.19 main.show_explain : MDEV-10674 - sporadic failure -main.shutdown : MDEV-10563 - sporadic crashes -main.sp-prelocking : Modified on 2016-08-10 (merge) +main.sp : Modified in 10.1.20 main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl_ca : Modified on 2016-07-11 (MDEV-10211) -main.ssl_compress : Modified on 2016-07-11 (MDEV-10211) -main.ssl_timeout : Modified on 2016-07-11 (MDEV-10211) main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results +main.statistics : Modified in 10.1.20 main.status : MDEV-8510 - sporadic wrong result +main.subselect : Modified in 10.1.20 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_sj_mat : Modified on 2016-07-27 (MDEV-10389) -main.type_date : Modified on 2016-08-10 (merge) main.type_datetime_hires : MDEV-10687 - timeout -main.view : Modified on 2016-08-10 (merge) -main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) +main.type_decimal : Modified in 10.1.20 +main.view : Uses search_pattern_in_file.inc modified in 10.1.20 +main.wait_timeout_not_windows : Uses search_pattern_in_file.inc modified in 10.1.20 #---------------------------------------------------------------- @@ -85,6 +87,12 @@ archive.discover : MDEV-10510 - table is marked as crashed #---------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed +binlog.binlog_incident : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_killed : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_killed_simulate : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_mysqlbinlog2 : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.mix_innodb_myisam_binlog : Uses binlog_start_pos.inc modified in 10.1.20 +binlog.binlog_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- @@ -94,29 +102,35 @@ binlog_encryption.* : Added in 10.1.20 #---------------------------------------------------------------- connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results -connect.jdbc : New test, added on 2016-07-15 -connect.jdbc-new : New test, added on 2016-07-14 -connect.jdbc-oracle : New test, added on 2016-07-13 -connect.jdbc-postgresql : New test, added on 2016-07-13 #---------------------------------------------------------------- encryption.create_or_replace : MDEV-9359 - Assertion failure -encryption.encrypt_and_grep : MDEV-11222 - InnoDB error +encryption.encrypt_and_grep : MDEV-11222 - InnoDB error; also uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_emptyfile : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_encfile_bad : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_encfile_badfile : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_encfile_no : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_nofile : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_syntax : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_tooshort : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.filekeys_unencfile : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-bad-key-change : uses keys2.txt modified in 10.1.19 encryption.innodb-bad-key-change2 : uses keys2.txt modified in 10.1.19 -encryption.innodb-bad-key-change3 : uses keys2.txt modified in 10.1.19 +encryption.innodb-bad-key-change3 : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-bad-key-change4 : uses keys2.txt modified in 10.1.19 encryption.innodb-bad-key-change5 : uses keys2.txt modified in 10.1.19 encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures, and uses keys2.txt modified in 10.1.19 -encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also modified in 10.1.19 +encryption.innodb-discard-import : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb_encryption_filekeys : MDEV-9962 - timeouts -encryption.innodb_encryption_tables : MDEV-10970 - Crash/assertion failure +encryption.innodb_encryption_row_compressed : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb_first_page : MDEV-10689 - crashes +encryption.innodb-log-encrypt : Uses search_pattern_in_file.inc modified in 10.1.20 +encryption.innodb_lotoftables : MDEV-11531 - InnoDB error encryption.innodb-missing-key : Added in 10.1.19 -encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results +encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results; also uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-page_encryption : MDEV-10641 - mutex problem -encryption.innodb-page_encryption_compression : MDEV-10739 - timeouts encryption.innodb_page_encryption_key_change : uses keys2.txt modified in 10.1.19 #---------------------------------------------------------------- @@ -132,80 +146,53 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- -funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge) -funcs_1.processlist_priv_ps : Include file modified on 2016-07-12 (merge) funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result -funcs_2.memory_charset : MDEV-10290 - timeout +funcs_2.memory_charset : MDEV-10290 - Timeout +funcs_2.myisam_charset : MDEV-11535 - Timeout #---------------------------------------------------------------- -galera.GAL-382 : New test, added in 10.1.17 -galera.MW-252 : New test, added in 10.1.17 -galera.MW-258 : New test, added in 10.1.17 -galera.MW-259 : New test, added in 10.1.17 -galera.MW-285 : New test, added in 10.1.17 -galera.MW-286 : New test, added in 10.1.17 -galera.MW-292 : New test, added in 10.1.17 -galera.MW-44 : New test, added in 10.1.17 -galera.galera#414 : New test, added in 10.1.17 -galera.galera_as_slave_autoinc : New test, added in 10.1.17 -galera.galera_as_slave_nonprim : Modified in 10.1.17 -galera.galera_bf_abort_flush_for_export : Modified in 10.1.17 -galera.galera_gcs_fc_limit : Modified in 10.1.17 -galera.galera_ist_recv_bind : New test, added in 10.1.17 -galera.galera_ist_restart_joiner : Modified in 10.1.17 -galera.galera_kill_ddl : Modified in 10.1.17 -galera.galera_parallel_simple : Modified in 10.1.17 -galera.galera_pc_ignore_sb : Modified in 10.1.17 -galera.galera_restart_nochanges : Modified in 10.1.17 -galera.galera_roles : Modified in 10.1.17 -galera.galera_rsu_wsrep_desync : Modified in 10.1.17 -galera.galera_split_brain : Modified in 10.1.17 -galera.galera_ssl_upgrade : Modified in 10.1.17 -galera.galera_suspend_slave : Modified in 10.1.17 -galera.galera_transaction_replay : Modified in 10.1.17 -galera.galera_var_dirty_reads : Modified in 10.1.17 -galera.galera_var_max_ws_rows : New test, added in 10.1.17 -galera.galera_var_max_ws_size : Modified in 10.1.17 -galera.mdev_10518 : New test, added in 10.1.17 -galera.mysql-wsrep#31 : Modified in 10.1.17 - -galera_3nodes.galera_certification_ccc : Modified in 10.1.17 -galera_3nodes.galera_innobackupex_backup : Modified in 10.1.17 -galera_3nodes.galera_ist_gcache_rollover : Modified in 10.1.17 -galera_3nodes.galera_pc_bootstrap : Modified in 10.1.17 -galera_3nodes.galera_pc_weight : Modified in 10.1.17 - galera.* : Added to default suites in 10.1.19 -galera_3nodes.* : Added to default suites in 10.1.19 + +galera.galera_var_dirty_reads : Modified in 10.1.20 +galera.rpl_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 + +galera_3nodes.* : Added to default suites in 10.1.19, MDEV-11490 #---------------------------------------------------------------- -innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.binlog_consistent : MDEV-10618 - Server fails to start; also uses binlog_start_pos.inc modified in 10.1.20 +innodb.group_commit_binlog_pos : Uses binlog_start_pos.inc modified in 10.1.20 +innodb.group_commit_binlog_pos_no_optimize_thread : Uses binlog_start_pos.inc modified in 10.1.20 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) -innodb.innodb_blob_truncate : MDEV-10377 - Assertion failure innodb.innodb-bug-14068765 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb-bug-14084530 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb-fk-warnings : Modified on 2016-07-18 (MDEV-8569) +innodb.innodb-change-buffer-recovery : Uses search_pattern_in_file.inc modified in 10.1.20 +innodb.innodb_defragment_fill_factor : Modified in 10.1.20 +innodb.innodb-lock-schedule-algorithm : Modified in 10.1.20 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result innodb.innodb-wl5522 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 innodb.innodb-wl5522-1 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 -innodb.innodb-wl5522-debug : modified in 10.1.19 -innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption, also modified in 10.1.19 +innodb.innodb-wl5522-debug : Modified in 10.1.19 +innodb.innodb-wl5522-debug-zip : Modified in 10.1.20 innodb.innodb-wl5522-zip : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.table_index_statistics : Added in 10.1.20 +innodb.trigger : Modified in 10.1.20 innodb.xa_recovery : Modified in 10.1.19 #---------------------------------------------------------------- -innodb_zip.innodb_prefix_index_liftedlimit : MDEV-10686 - timeout +innodb_fts.create : Added in 10.1.20 +innodb_fts.innodb_fts_misc : MDEV-11233 - Crash on CREATE FULLTEXT INDEX #---------------------------------------------------------------- +maria.collations : Added in 10.1.20 maria.encrypt-wrong-key : uses keys2.txt modified in 10.1.19 +maria.maria-connect : Uses binlog_start_pos.inc modified in 10.1.20 #---------------------------------------------------------------- @@ -222,9 +209,9 @@ mroonga/storage.repair_table_no_index_file : MDEV-9364 - multi_source.gtid : MDEV-10620, MDEV-10417 - Timeout in wait condition, fails on Mips multi_source.info_logs : MDEV-10042 - wrong result -multi_source.multisource : MDEV-10417 - Fails on Mips -multi_source.reset_slave : MDEV-10690 - wrong result -multi_source.simple : MDEV-4633 - Wrong slave status output +multi_source.multisource : MDEV-10417 - Fails on Mips; also uses binlog_start_pos.inc modified in 10.1.20 +multi_source.reset_slave : MDEV-10690 - wrong result; also uses binlog_start_pos.inc modified in 10.1.20 +multi_source.simple : MDEV-4633 - Wrong slave status output; also uses binlog_start_pos.inc modified in 10.1.20 multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_heartbeats #---------------------------------------------------------------- @@ -243,11 +230,16 @@ perfschema.threads_mysql : MDEV-10677 - sporadic wrong resul #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url -plugins.pam : Modified on 2016-08-03 (MDEV-7329) -plugins.pam_cleartext : Modified on 2016-08-03 plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc + +#---------------------------------------------------------------- + +roles.role_case_sensitive-10744 : Added in 10.1.20 +roles.create_and_drop_role : Modified in 10.1.20 +roles.create_and_grant_role : MDEV-11533 - Extra grant in output + #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log @@ -255,6 +247,7 @@ rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering on master +rpl.rpl_checksum : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips @@ -262,6 +255,7 @@ rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result rpl.rpl_drop_db : Modified in 10.1.19 rpl.rpl_gtid_basic : MDEV-10681 - server startup problem rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master +rpl.rpl_gtid_errorlog : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_mdev9033 : MDEV-10680 - warnings rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown @@ -271,6 +265,7 @@ rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog +rpl.rpl_mdev10863 : Added in 10.1.20 rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts @@ -278,18 +273,22 @@ rpl.rpl_parallel_optimistic : MDEV-10511 - timeout rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips +rpl.rpl_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 +rpl.rpl_password_boundaries : MDEV-11534 - Slave IO warnings rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout +rpl.rpl_row_flsh_tbls : Uses binlog_start_pos.inc modified in 10.1.20 rpl.rpl_row_log_innodb : MDEV-10688 - Wrong result rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x -rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status +rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem +rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock -rpl.rpl_stop_slave_error : Modified in 10.1.19 +rpl.rpl_stm_flsh_tbls : Uses binlog_start_pos.inc modified in 10.1.20 +rpl.rpl_stop_slave_error : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master -rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha #---------------------------------------------------------------- @@ -316,41 +315,33 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- -sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x -sys_vars.general_log_file_basic : Modified on 2016-08-09 (MDEV-10465) -sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout -sys_vars.slow_query_log_file_basic : Modified on 2016-08-09 (MDEV-10465) -sys_vars.sysvars_innodb : MDEV-6958 - error-prone rdiffs -sys_vars.sysvars_server_embedded : MDEV-6958 - error-prone rdiffs -sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists +sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x +sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout +sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists sys_vars.innodb_fatal_semaphore_wait_threshold : MDEV-10513 - crashes -sys_vars.sysvars_wsrep : Modified in 10.1.17 -sys_vars.wsrep_max_ws_size_basic : Modified in 10.1.17 +sys_vars.replicate_do_db_basic : Modified in 10.1.20 +sys_vars.replicate_do_table_basic : Modified in 10.1.20 +sys_vars.replicate_ignore_db_basic : Modified in 10.1.20 +sys_vars.replicate_ignore_table_basic : Modified in 10.1.20 +sys_vars.replicate_wild_do_table_basic : Modified in 10.1.20 +sys_vars.replicate_wild_ignore_table_basic : Modified in 10.1.20 +sys_vars.sysvars_innodb : MDEV-6958 - error-prone rdiffs +sys_vars.sysvars_server_embedded : MDEV-6958 - error-prone rdiffs +sys_vars.table_open_cache_instances_basic : Modified in 10.1.20 #---------------------------------------------------------------- tokudb.cluster_filter : MDEV-10678 - Wrong execution plan tokudb.cluster_filter_hidden : MDEV-10678 - Wrong execution plan tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan -tokudb.i_s_tokudb_lock_waits_released : Modified in 10.1.17 -tokudb.i_s_tokudb_locks_released : Modified in 10.1.17 -tokudb.* : MDEV-9891 - massive crashes on shutdown - -tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown +tokudb.dir_per_db : MDEV-11537 - Wrong result +tokudb.table_index_statistics : Added in 10.1.20 tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output -tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown - -tokudb_parts.* : MDEV-9891 - massive crashes on shutdown - -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown -tokudb_backup.* : MDEV-9891 - massive crashes on shutdown -tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown -tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown -tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown +tokudb_rpl.rpl_parallel_optimistic : Added in 10.1.20 +tokudb_rpl.rpl_tokudb_rfr_partition_table : Added in 10.1.20 #---------------------------------------------------------------- @@ -363,8 +354,10 @@ vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout #---------------------------------------------------------------- -wsrep.mdev_10186 : Modified in 10.1.19 +wsrep.binlog_format : MDEV-11532 - WSREP has not yet prepared node +wsrep.mdev_10186 : Modified in 10.1.19 #---------------------------------------------------------------- -wsrep_info.* : suite.pm modified in 10.1.19 +wsrep_info.* : suite.pm modified in 10.1.19 +wsrep_info.plugin : MDEV-11530 - Warnings; also modified in 10.1.20 From dbb06d2eaba63df0eae921da8a971c664ca18010 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Nov 2016 19:44:48 -0500 Subject: [PATCH 263/295] MDEV-10954: MariaDB Galera: wsrep_sst_common: line 120: which: command not found Add 'which' to REQUIRES list. --- cmake/cpack_rpm.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 703e7424159ce..f1aab2c353c53 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -160,7 +160,7 @@ SETA(CPACK_RPM_server_PACKAGE_REQUIRES IF(WITH_WSREP) SETA(CPACK_RPM_server_PACKAGE_REQUIRES "galera" "rsync" "lsof" "grep" "gawk" "iproute" - "coreutils" "findutils" "tar") + "coreutils" "findutils" "tar" "which") ENDIF() SET(CPACK_RPM_server_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-prein.sh) From 9c88a54c1064ebdcf461e6c524a36ce5cc9370bd Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 5 Dec 2016 20:07:30 -0500 Subject: [PATCH 264/295] MDEV-11179: WSREP transaction excceded size limit in Galera cluster ... causes MariaDB to crash On error, the wsrep replication buffer (binlog) is dumped to a file to aid investigations. In order to also include the binlog header, FDLE object is also needed. This object is only available for wsrep- threads. Fix: Instantiate an FDLE object for non-wsrep threads. --- sql/wsrep_applier.cc | 3 +++ sql/wsrep_binlog.cc | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/sql/wsrep_applier.cc b/sql/wsrep_applier.cc index 426789b4d38fa..b9c259f7b5e42 100644 --- a/sql/wsrep_applier.cc +++ b/sql/wsrep_applier.cc @@ -73,6 +73,9 @@ Format_description_log_event* wsrep_get_apply_format(THD* thd) { return (Format_description_log_event*) thd->wsrep_apply_format; } + + DBUG_ASSERT(thd->wsrep_rgi); + return thd->wsrep_rgi->rli->relay_log.description_event_for_exec; } diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc index 36917674128d1..d3f59cee5f2d6 100644 --- a/sql/wsrep_binlog.cc +++ b/sql/wsrep_binlog.cc @@ -442,11 +442,13 @@ void thd_binlog_flush_pending_rows_event(THD *thd, bool stmt_end) void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf, size_t buf_len) { + DBUG_ENTER("wsrep_dump_rbr_buf_with_header"); + char filename[PATH_MAX]= {0}; File file; IO_CACHE cache; Log_event_writer writer(&cache); - Format_description_log_event *ev= wsrep_get_apply_format(thd); + Format_description_log_event *ev; int len= my_snprintf(filename, PATH_MAX, "%s/GRA_%ld_%lld_v2.log", wsrep_data_home_dir, thd->thread_id, @@ -455,7 +457,7 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf, if (len >= PATH_MAX) { WSREP_ERROR("RBR dump path too long: %d, skipping dump.", len); - return; + DBUG_VOID_RETURN; } if ((file= mysql_file_open(key_file_wsrep_gra_log, filename, @@ -477,6 +479,13 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf, goto cleanup2; } + /* + Instantiate an FDLE object for non-wsrep threads (to be written + to the dump file). + */ + ev= (thd->wsrep_applier) ? wsrep_get_apply_format(thd) : + (new Format_description_log_event(4)); + if (writer.write(ev) || my_b_write(&cache, (uchar*)rbr_buf, buf_len) || flush_io_cache(&cache)) { @@ -489,5 +498,9 @@ void wsrep_dump_rbr_buf_with_header(THD *thd, const void *rbr_buf, cleanup1: mysql_file_close(file, MYF(MY_WME)); + + if (!thd->wsrep_applier) delete ev; + + DBUG_VOID_RETURN; } From 67b570af506a42f14a8ae30603d52ca9de68cce5 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 5 Dec 2016 20:58:49 -0500 Subject: [PATCH 265/295] MDEV-10545: Server crashed in my_copy_fix_mb on querying I_S and P_S tables After applying/replaying the transaction, the memory that stored the query string was also wrongly freed. --- sql/wsrep_applier.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/wsrep_applier.cc b/sql/wsrep_applier.cc index b9c259f7b5e42..e10c19287c4d2 100644 --- a/sql/wsrep_applier.cc +++ b/sql/wsrep_applier.cc @@ -365,8 +365,10 @@ wsrep_cb_status_t wsrep_commit_cb(void* const ctx, else rcode = wsrep_rollback(thd); + /* Cleanup */ wsrep_set_apply_format(thd, NULL); thd->mdl_context.release_transactional_locks(); + thd->reset_query(); /* Mutex protected */ free_root(thd->mem_root,MYF(MY_KEEP_PREALLOC)); thd->tx_isolation= (enum_tx_isolation) thd->variables.tx_isolation; From 72cc73cea2e7071277b6a88bae0236cabf11788a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 13 Dec 2016 11:51:33 +0200 Subject: [PATCH 266/295] MDEV-10368: get_latest_version() called too often Reduce the number of calls to encryption_get_key_get_latest_version when doing key rotation with two different methods: (1) We need to fetch key information when tablespace not yet have a encryption information, invalid keys are handled now differently (see below). There was extra call to detect if key_id is not found on key rotation. (2) If key_id is not found from encryption plugin, do not try fetching new key_version for it as it will fail anyway. We store return value from encryption_get_key_get_latest_version call and if it returns ENCRYPTION_KEY_VERSION_INVALID there is no need to call it again. --- storage/innobase/buf/buf0buf.cc | 4 +- storage/innobase/fil/fil0crypt.cc | 175 ++++++++++++++------------ storage/innobase/fil/fil0fil.cc | 11 +- storage/innobase/handler/ha_innodb.cc | 15 +-- storage/innobase/include/fil0crypt.h | 114 ++++++++++++++++- storage/innobase/include/srv0srv.h | 4 + storage/innobase/row/row0ftsort.cc | 5 +- storage/innobase/row/row0merge.cc | 5 +- storage/innobase/srv/srv0srv.cc | 2 + storage/xtradb/buf/buf0buf.cc | 4 +- storage/xtradb/fil/fil0crypt.cc | 175 ++++++++++++++------------ storage/xtradb/fil/fil0fil.cc | 11 +- storage/xtradb/handler/ha_innodb.cc | 15 +-- storage/xtradb/include/fil0crypt.h | 115 ++++++++++++++++- storage/xtradb/include/srv0srv.h | 4 + storage/xtradb/row/row0ftsort.cc | 5 +- storage/xtradb/row/row0merge.cc | 5 +- storage/xtradb/srv/srv0srv.cc | 2 + 18 files changed, 464 insertions(+), 207 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index a05f3715cb991..c918522233ffe 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -6145,12 +6145,12 @@ buf_page_encrypt_before_write( return src_frame; } - if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data != NULL && crypt_data->not_encrypted()) { /* Encryption is disabled */ encrypted = false; } - if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->is_default_encryption())) { /* Encryption is disabled */ encrypted = false; } diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 72e3454cbb54e..3d044e82849f7 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -142,6 +142,23 @@ fil_space_crypt_cleanup() os_event_free(fil_crypt_throttle_sleep_event); } +/** +Get latest key version from encryption plugin. +@return key version or ENCRYPTION_KEY_VERSION_INVALID */ +uint +fil_space_crypt_struct::key_get_latest_version(void) +{ + uint key_version = key_found; + + if (is_key_found()) { + key_version = encryption_key_get_latest_version(key_id); + srv_stats.n_key_requests.inc(); + key_found = key_version; + } + + return key_version; +} + /****************************************************************** Get the latest(key-version), waking the encrypt thread, if needed */ static inline @@ -150,20 +167,25 @@ fil_crypt_get_latest_key_version( /*=============================*/ fil_space_crypt_t* crypt_data) /*!< in: crypt data */ { - uint rc = encryption_key_get_latest_version(crypt_data->key_id); + ut_ad(crypt_data != NULL); - if (fil_crypt_needs_rotation(crypt_data->encryption, - crypt_data->min_key_version, - rc, srv_fil_crypt_rotate_key_age)) { - os_event_set(fil_crypt_threads_event); + uint key_version = crypt_data->key_get_latest_version(); + + if (crypt_data->is_key_found()) { + + if (fil_crypt_needs_rotation(crypt_data->encryption, + crypt_data->min_key_version, + key_version, + srv_fil_crypt_rotate_key_age)) { + os_event_set(fil_crypt_threads_event); + } } - return rc; + return key_version; } /****************************************************************** Mutex helper for crypt_data->scheme */ -static void crypt_data_scheme_locker( /*=====================*/ @@ -183,37 +205,46 @@ crypt_data_scheme_locker( /****************************************************************** Create a fil_space_crypt_t object @return crypt object */ -UNIV_INTERN +static fil_space_crypt_t* fil_space_create_crypt_data( /*========================*/ - fil_encryption_t encrypt_mode, /*!< in: encryption mode */ - uint key_id) /*!< in: encryption key id */ + uint type, + fil_encryption_t encrypt_mode, + uint min_key_version, + uint key_id, + ulint offset) { const uint sz = sizeof(fil_space_crypt_t); - fil_space_crypt_t* crypt_data = - static_cast(malloc(sz)); - - memset(crypt_data, 0, sz); + void* buf = mem_zalloc(sz); + fil_space_crypt_t* crypt_data = NULL; - if (encrypt_mode == FIL_SPACE_ENCRYPTION_OFF || - (!srv_encrypt_tables && encrypt_mode == FIL_SPACE_ENCRYPTION_DEFAULT)) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; - } else { - crypt_data->type = CRYPT_SCHEME_1; - crypt_data->min_key_version = encryption_key_get_latest_version(key_id); + if (buf) { + crypt_data = new(buf) + fil_space_crypt_struct( + type, + min_key_version, + key_id, + offset, + encrypt_mode); } - mutex_create(fil_crypt_data_mutex_key, - &crypt_data->mutex, SYNC_NO_ORDER_CHECK); - crypt_data->locker = crypt_data_scheme_locker; - my_random_bytes(crypt_data->iv, sizeof(crypt_data->iv)); - crypt_data->encryption = encrypt_mode; - crypt_data->inited = true; - crypt_data->key_id = key_id; return crypt_data; } +/****************************************************************** +Create a fil_space_crypt_t object +@return crypt object */ +UNIV_INTERN +fil_space_crypt_t* +fil_space_create_crypt_data( +/*========================*/ + fil_encryption_t encrypt_mode, /*!< in: encryption mode */ + uint key_id) /*!< in: encryption key id */ +{ + return (fil_space_create_crypt_data(0, encrypt_mode, 0, key_id, 0)); +} + /****************************************************************** Merge fil_space_crypt_t object */ UNIV_INTERN @@ -236,7 +267,7 @@ fil_space_merge_crypt_data( dst->type = src->type; dst->min_key_version = src->min_key_version; dst->keyserver_requests += src->keyserver_requests; - dst->inited = src->inited; + dst->closing = src->closing; mutex_exit(&dst->mutex); } @@ -308,19 +339,12 @@ fil_space_read_crypt_data( fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1( page + offset + MAGIC_SZ + 2 + iv_length + 8); - const uint sz = sizeof(fil_space_crypt_t) + iv_length; - crypt_data = static_cast(malloc(sz)); - memset(crypt_data, 0, sz); - + crypt_data = fil_space_create_crypt_data(encryption, key_id); + /* We need to overwrite these as above function will initialize + members */ crypt_data->type = type; crypt_data->min_key_version = min_key_version; - crypt_data->key_id = key_id; crypt_data->page0_offset = offset; - crypt_data->encryption = encryption; - mutex_create(fil_crypt_data_mutex_key, - &crypt_data->mutex, SYNC_NO_ORDER_CHECK); - crypt_data->locker = crypt_data_scheme_locker; - crypt_data->inited = true; memcpy(crypt_data->iv, page + offset + MAGIC_SZ + 2, iv_length); return crypt_data; @@ -335,15 +359,9 @@ fil_space_destroy_crypt_data( fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { - /* Make sure that this thread owns the crypt_data - and make it unawailable, this does not fully - avoid the race between drop table and crypt thread */ - mutex_enter(&(*crypt_data)->mutex); - (*crypt_data)->inited = false; - mutex_exit(&(*crypt_data)->mutex); - mutex_free(& (*crypt_data)->mutex); - memset(*crypt_data, 0, sizeof(fil_space_crypt_t)); - free(*crypt_data); + fil_space_crypt_t* c = *crypt_data; + c->~fil_space_crypt_struct(); + mem_free(c); (*crypt_data) = NULL; } } @@ -491,6 +509,7 @@ fil_parse_write_crypt_data( } fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(encryption, key_id); + /* Need to overwrite these as above will initialize fields. */ crypt_data->page0_offset = offset; crypt_data->min_key_version = min_key_version; crypt_data->encryption = encryption; @@ -648,7 +667,7 @@ fil_space_encrypt( return src_frame; } - ut_a(crypt_data != NULL && crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF); + ut_a(crypt_data != NULL && crypt_data->is_encrypted()); byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame); @@ -729,7 +748,7 @@ fil_space_check_encryption_read( return false; } - if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->not_encrypted()) { return false; } @@ -781,7 +800,7 @@ fil_space_decrypt( return false; } - ut_a(crypt_data != NULL && crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF); + ut_a(crypt_data != NULL && crypt_data->is_encrypted()); /* read space & lsn */ ulint header_len = FIL_PAGE_DATA; @@ -1019,20 +1038,13 @@ Copy global key state */ static void fil_crypt_get_key_state( /*====================*/ - key_state_t *new_state) /*!< out: key state */ + key_state_t* new_state, /*!< out: key state */ + fil_space_crypt_t* crypt_data) /*!< in, out: crypt_data */ { if (srv_encrypt_tables) { - new_state->key_version = - encryption_key_get_latest_version(new_state->key_id); + new_state->key_version = crypt_data->key_get_latest_version(); new_state->rotate_key_age = srv_fil_crypt_rotate_key_age; - if (new_state->key_version == ENCRYPTION_KEY_VERSION_INVALID) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Used key_id %u can't be found from key file.", - new_state->key_id); - } - - ut_a(new_state->key_version != ENCRYPTION_KEY_VERSION_INVALID); ut_a(new_state->key_version != ENCRYPTION_KEY_NOT_ENCRYPTED); } else { new_state->key_version = 0; @@ -1092,9 +1104,7 @@ fil_crypt_is_closing( fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); if (crypt_data) { - mutex_enter(&crypt_data->mutex); - closing = crypt_data->closing; - mutex_exit(&crypt_data->mutex); + closing = crypt_data->is_closing(false); } return closing; @@ -1349,11 +1359,17 @@ fil_crypt_space_needs_rotation( } return false; } + + crypt_data->key_get_latest_version(); + + if (!crypt_data->is_key_found()) { + return false; + } } /* If used key_id is not found from encryption plugin we can't continue to rotate the tablespace */ - if (fil_crypt_get_latest_key_version(crypt_data) == ENCRYPTION_KEY_VERSION_INVALID) { + if (!crypt_data->is_key_found()) { return false; } @@ -1368,7 +1384,7 @@ fil_crypt_space_needs_rotation( } /* prevent threads from starting to rotate space */ - if (crypt_data->closing) { + if (crypt_data->is_closing(true)) { break; } @@ -1377,13 +1393,13 @@ fil_crypt_space_needs_rotation( } /* No need to rotate space if encryption is disabled */ - if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->not_encrypted()) { break; } if (crypt_data->key_id != key_state->key_id) { key_state->key_id= crypt_data->key_id; - fil_crypt_get_key_state(key_state); + fil_crypt_get_key_state(key_state, crypt_data); } bool need_key_rotation = fil_crypt_needs_rotation( @@ -1396,12 +1412,14 @@ fil_crypt_space_needs_rotation( time_t diff = time(0) - crypt_data->rotate_state.scrubbing. last_scrub_completed; + bool need_scrubbing = crypt_data->rotate_state.scrubbing.is_active && diff >= srv_background_scrub_data_interval; - if (need_key_rotation == false && need_scrubbing == false) + if (need_key_rotation == false && need_scrubbing == false) { break; + } mutex_exit(&crypt_data->mutex); /* NOTE! fil_decr_pending_ops is performed outside */ @@ -1617,8 +1635,9 @@ fil_crypt_find_space_to_rotate( os_event_wait_time(fil_crypt_threads_event, 1000000); } - if (state->should_shutdown()) + if (state->should_shutdown()) { return false; + } if (state->first) { state->first = false; @@ -1680,7 +1699,7 @@ fil_crypt_start_rotate_space( crypt_data->rotate_state.start_time = time(0); if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && - crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF && + crypt_data->is_encrypted() && key_state->key_version != 0) { /* this is rotation unencrypted => encrypted */ crypt_data->type = CRYPT_SCHEME_1; @@ -1717,7 +1736,7 @@ fil_crypt_find_page_to_rotate( mutex_enter(&crypt_data->mutex); ut_ad(key_state->key_id == crypt_data->key_id); - if (crypt_data->closing == false && + if (!crypt_data->is_closing(true) && crypt_data->rotate_state.next_offset < crypt_data->rotate_state.max_offset) { @@ -1974,7 +1993,7 @@ fil_crypt_rotate_page( /* statistics */ state->crypt_stat.pages_modified++; } else { - if (crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->is_encrypted()) { ut_a(kv >= crypt_data->min_key_version || (kv == 0 && key_state->key_version == 0)); @@ -2177,7 +2196,7 @@ fil_crypt_complete_rotate_space( fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); /* Space might already be dropped */ - if (crypt_data != NULL && crypt_data->inited) { + if (crypt_data != NULL && !crypt_data->is_closing(false)) { mutex_enter(&crypt_data->mutex); /** @@ -2513,7 +2532,7 @@ fil_space_crypt_close_tablespace( fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - if (crypt_data == NULL || !crypt_data->inited) { + if (crypt_data == NULL || crypt_data->is_closing(false)) { mutex_exit(&fil_crypt_threads_mutex); return; } @@ -2566,6 +2585,8 @@ fil_space_crypt_get_status( { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); + memset(status, 0, sizeof(*status)); + if (crypt_data != NULL) { status->space = id; status->scheme = crypt_data->type; @@ -2586,6 +2607,7 @@ fil_space_crypt_get_status( } else { status->rotating = false; } + mutex_exit(&crypt_data->mutex); if (srv_encrypt_tables || crypt_data->min_key_version) { @@ -2595,7 +2617,6 @@ fil_space_crypt_get_status( status->current_key_version = 0; } } else { - memset(status, 0, sizeof(*status)); if (srv_encrypt_tables) { os_event_set(fil_crypt_threads_event); } @@ -2628,6 +2649,7 @@ fil_space_get_scrub_status( struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); + memset(status, 0, sizeof(*status)); if (crypt_data != NULL) { @@ -2650,9 +2672,8 @@ fil_space_get_scrub_status( } else { status->scrubbing = false; } + mutex_exit(&crypt_data->mutex); - } else { - memset(status, 0, sizeof(*status)); } return crypt_data == NULL ? 1 : 0; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7a50ccd22711c..1aa2555f1b36b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2073,9 +2073,7 @@ fil_read_first_page( /* If file space is encrypted we need to have at least some encryption service available where to get keys */ - if ((cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (cdata && cdata->should_encrypt()) { if (!encryption_key_id_exists(cdata->key_id)) { ib_logf(IB_LOG_LEVEL_ERROR, @@ -6569,10 +6567,7 @@ fil_iterate( bool encrypted = false; /* Use additional crypt io buffer if tablespace is encrypted */ - if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (iter.crypt_data != NULL && iter.crypt_data->should_encrypt()) { encrypted = true; readptr = iter.crypt_io_buffer; writeptr = iter.crypt_io_buffer; @@ -7293,7 +7288,7 @@ fil_space_get_crypt_data( if (!space->page_0_crypt_read) { ib_logf(IB_LOG_LEVEL_WARN, - "Space %lu name %s contains encryption %d information for key_id %d but page0 is not read.", + "Space %lu name %s contains encryption %d information for key_id %u but page0 is not read.", space->id, space->name, space->crypt_data ? space->crypt_data->encryption : 0, diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 983ee0217cf13..35d7ef7ee6fbc 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1106,6 +1106,8 @@ static SHOW_VAR innodb_status_variables[]= { {"scrub_background_page_split_failures_unknown", (char*) &export_vars.innodb_scrub_page_split_failures_unknown, SHOW_LONG}, + {"encryption_num_key_requests", + (char*) &export_vars.innodb_encryption_key_requests, SHOW_LONGLONG}, {NullS, NullS, SHOW_LONG} }; @@ -5935,9 +5937,8 @@ ha_innobase::open( or used key_id is not available. */ if (ib_table) { fil_space_crypt_t* crypt_data = ib_table->crypt_data; - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + + if (crypt_data && crypt_data->should_encrypt()) { if (!encryption_key_id_exists(crypt_data->key_id)) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, @@ -20735,12 +20736,12 @@ ib_push_warning( const char *format,/*!< in: warning message */ ...) { - va_list args; - THD *thd = (THD *)trx->mysql_thd; - char *buf; + if (trx && trx->mysql_thd) { + THD *thd = (THD *)trx->mysql_thd; + va_list args; + char *buf; #define MAX_BUF_SIZE 4*1024 - if (thd) { va_start(args, format); buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); vsprintf(buf,format, args); diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 8ffa4e2073fcd..8bb0ce65a6bee 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -75,6 +75,21 @@ struct key_struct (that is L in CRYPT_SCHEME_1) */ }; +/** is encryption enabled */ +extern ulong srv_encrypt_tables; + +#ifdef UNIV_PFS_MUTEX +extern mysql_pfs_key_t fil_crypt_data_mutex_key; +#endif + +/** Mutex helper for crypt_data->scheme +@param[in, out] schme encryption scheme +@param[in] exit should we exit or enter mutex ? */ +void +crypt_data_scheme_locker( + st_encryption_scheme* scheme, + int exit); + struct fil_space_rotate_state_t { time_t start_time; /*!< time when rotation started */ @@ -96,13 +111,110 @@ struct fil_space_rotate_state_t struct fil_space_crypt_struct : st_encryption_scheme { + public: + /** Constructor. Does not initialize the members! + The object is expected to be placed in a buffer that + has been zero-initialized. */ + fil_space_crypt_struct( + ulint new_type, + uint new_min_key_version, + uint new_key_id, + ulint offset, + fil_encryption_t new_encryption) + : st_encryption_scheme(), + min_key_version(new_min_key_version), + page0_offset(offset), + encryption(new_encryption), + closing(false), + key_found(), + rotate_state() + { + key_found = new_min_key_version; + key_id = new_key_id; + my_random_bytes(iv, sizeof(iv)); + mutex_create(fil_crypt_data_mutex_key, + &mutex, SYNC_NO_ORDER_CHECK); + locker = crypt_data_scheme_locker; + type = new_type; + + if (new_encryption == FIL_SPACE_ENCRYPTION_OFF || + (!srv_encrypt_tables && + new_encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + type = CRYPT_SCHEME_UNENCRYPTED; + } else { + type = CRYPT_SCHEME_1; + min_key_version = key_get_latest_version(); + } + } + + /** Destructor */ + ~fil_space_crypt_struct() + { + closing = true; + mutex_free(&mutex); + } + + /** Get latest key version from encryption plugin + @retval key_version or + @retval ENCRYPTION_KEY_VERSION_INVALID if used key_id + is not found from encryption plugin. */ + uint key_get_latest_version(void); + + /** Returns true if key was found from encryption plugin + and false if not. */ + bool is_key_found() const { + return key_found != ENCRYPTION_KEY_VERSION_INVALID; + } + + /** Returns true if tablespace should be encrypted */ + bool should_encrypt() const { + return ((encryption == FIL_SPACE_ENCRYPTION_ON) || + (srv_encrypt_tables && + encryption == FIL_SPACE_ENCRYPTION_DEFAULT)); + } + + /** Return true if tablespace is encrypted. */ + bool is_encrypted() const { + return (encryption != FIL_SPACE_ENCRYPTION_OFF); + } + + /** Return true if default tablespace encryption is used, */ + bool is_default_encryption() const { + return (encryption == FIL_SPACE_ENCRYPTION_DEFAULT); + } + + /** Return true if tablespace is not encrypted. */ + bool not_encrypted() const { + return (encryption == FIL_SPACE_ENCRYPTION_OFF); + } + + /** Is this tablespace closing. */ + bool is_closing(bool is_fixed) { + bool closed; + if (!is_fixed) { + mutex_enter(&mutex); + } + closed = closing; + if (!is_fixed) { + mutex_exit(&mutex); + } + return closed; + } + uint min_key_version; // min key version for this space ulint page0_offset; // byte offset on page 0 for crypt data fil_encryption_t encryption; // Encryption setup ib_mutex_t mutex; // mutex protecting following variables bool closing; // is tablespace being closed - bool inited; + + /** Return code from encryption_key_get_latest_version. + If ENCRYPTION_KEY_VERSION_INVALID encryption plugin + could not find the key and there is no need to call + get_latest_key_version again as keys are read only + at startup. */ + uint key_found; + fil_space_rotate_state_t rotate_state; }; diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index d75f60944cc8d..a5713afbd4934 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -182,6 +182,9 @@ struct srv_stats_t { /** Number of times page 0 is read from tablespace */ ulint_ctr_64_t page0_read; + + /** Number of encryption_get_latest_key_version calls */ + ulint_ctr_64_t n_key_requests; }; extern const char* srv_main_thread_op_info; @@ -1041,6 +1044,7 @@ struct export_var_t{ ulint innodb_encryption_rotation_pages_modified; ulint innodb_encryption_rotation_pages_flushed; ulint innodb_encryption_rotation_estimated_iops; + ib_int64_t innodb_encryption_key_requests; ulint innodb_scrub_page_reorganizations; ulint innodb_scrub_page_splits; diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index b941105e502c4..cc676af4ef3ad 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -223,10 +223,7 @@ row_fts_psort_info_init( common_info->opt_doc_id_size = opt_doc_id_size; crypt_data = fil_space_get_crypt_data(new_table->space); - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (crypt_data && crypt_data->should_encrypt()) { common_info->crypt_data = crypt_data; encrypted = true; } else { diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 12b3cf1f57287..4a2a8116ec7e5 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3998,10 +3998,7 @@ row_merge_build_indexes( /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (crypt_data && crypt_data->should_encrypt()) { crypt_block = static_cast( os_mem_alloc_large(&block_size)); diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 076bc36613281..1e323625b133d 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1663,6 +1663,8 @@ srv_export_innodb_status(void) crypt_stat.pages_flushed; export_vars.innodb_encryption_rotation_estimated_iops = crypt_stat.estimated_iops; + export_vars.innodb_encryption_key_requests = + srv_stats.n_key_requests; export_vars.innodb_scrub_page_reorganizations = scrub_stat.page_reorganizations; diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index ebfc813f13daa..31a3f4978b844 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -6327,12 +6327,12 @@ buf_page_encrypt_before_write( return src_frame; } - if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data != NULL && crypt_data->not_encrypted()) { /* Encryption is disabled */ encrypted = false; } - if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->is_default_encryption())) { /* Encryption is disabled */ encrypted = false; } diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 72e3454cbb54e..3d044e82849f7 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -142,6 +142,23 @@ fil_space_crypt_cleanup() os_event_free(fil_crypt_throttle_sleep_event); } +/** +Get latest key version from encryption plugin. +@return key version or ENCRYPTION_KEY_VERSION_INVALID */ +uint +fil_space_crypt_struct::key_get_latest_version(void) +{ + uint key_version = key_found; + + if (is_key_found()) { + key_version = encryption_key_get_latest_version(key_id); + srv_stats.n_key_requests.inc(); + key_found = key_version; + } + + return key_version; +} + /****************************************************************** Get the latest(key-version), waking the encrypt thread, if needed */ static inline @@ -150,20 +167,25 @@ fil_crypt_get_latest_key_version( /*=============================*/ fil_space_crypt_t* crypt_data) /*!< in: crypt data */ { - uint rc = encryption_key_get_latest_version(crypt_data->key_id); + ut_ad(crypt_data != NULL); - if (fil_crypt_needs_rotation(crypt_data->encryption, - crypt_data->min_key_version, - rc, srv_fil_crypt_rotate_key_age)) { - os_event_set(fil_crypt_threads_event); + uint key_version = crypt_data->key_get_latest_version(); + + if (crypt_data->is_key_found()) { + + if (fil_crypt_needs_rotation(crypt_data->encryption, + crypt_data->min_key_version, + key_version, + srv_fil_crypt_rotate_key_age)) { + os_event_set(fil_crypt_threads_event); + } } - return rc; + return key_version; } /****************************************************************** Mutex helper for crypt_data->scheme */ -static void crypt_data_scheme_locker( /*=====================*/ @@ -183,37 +205,46 @@ crypt_data_scheme_locker( /****************************************************************** Create a fil_space_crypt_t object @return crypt object */ -UNIV_INTERN +static fil_space_crypt_t* fil_space_create_crypt_data( /*========================*/ - fil_encryption_t encrypt_mode, /*!< in: encryption mode */ - uint key_id) /*!< in: encryption key id */ + uint type, + fil_encryption_t encrypt_mode, + uint min_key_version, + uint key_id, + ulint offset) { const uint sz = sizeof(fil_space_crypt_t); - fil_space_crypt_t* crypt_data = - static_cast(malloc(sz)); - - memset(crypt_data, 0, sz); + void* buf = mem_zalloc(sz); + fil_space_crypt_t* crypt_data = NULL; - if (encrypt_mode == FIL_SPACE_ENCRYPTION_OFF || - (!srv_encrypt_tables && encrypt_mode == FIL_SPACE_ENCRYPTION_DEFAULT)) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; - } else { - crypt_data->type = CRYPT_SCHEME_1; - crypt_data->min_key_version = encryption_key_get_latest_version(key_id); + if (buf) { + crypt_data = new(buf) + fil_space_crypt_struct( + type, + min_key_version, + key_id, + offset, + encrypt_mode); } - mutex_create(fil_crypt_data_mutex_key, - &crypt_data->mutex, SYNC_NO_ORDER_CHECK); - crypt_data->locker = crypt_data_scheme_locker; - my_random_bytes(crypt_data->iv, sizeof(crypt_data->iv)); - crypt_data->encryption = encrypt_mode; - crypt_data->inited = true; - crypt_data->key_id = key_id; return crypt_data; } +/****************************************************************** +Create a fil_space_crypt_t object +@return crypt object */ +UNIV_INTERN +fil_space_crypt_t* +fil_space_create_crypt_data( +/*========================*/ + fil_encryption_t encrypt_mode, /*!< in: encryption mode */ + uint key_id) /*!< in: encryption key id */ +{ + return (fil_space_create_crypt_data(0, encrypt_mode, 0, key_id, 0)); +} + /****************************************************************** Merge fil_space_crypt_t object */ UNIV_INTERN @@ -236,7 +267,7 @@ fil_space_merge_crypt_data( dst->type = src->type; dst->min_key_version = src->min_key_version; dst->keyserver_requests += src->keyserver_requests; - dst->inited = src->inited; + dst->closing = src->closing; mutex_exit(&dst->mutex); } @@ -308,19 +339,12 @@ fil_space_read_crypt_data( fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1( page + offset + MAGIC_SZ + 2 + iv_length + 8); - const uint sz = sizeof(fil_space_crypt_t) + iv_length; - crypt_data = static_cast(malloc(sz)); - memset(crypt_data, 0, sz); - + crypt_data = fil_space_create_crypt_data(encryption, key_id); + /* We need to overwrite these as above function will initialize + members */ crypt_data->type = type; crypt_data->min_key_version = min_key_version; - crypt_data->key_id = key_id; crypt_data->page0_offset = offset; - crypt_data->encryption = encryption; - mutex_create(fil_crypt_data_mutex_key, - &crypt_data->mutex, SYNC_NO_ORDER_CHECK); - crypt_data->locker = crypt_data_scheme_locker; - crypt_data->inited = true; memcpy(crypt_data->iv, page + offset + MAGIC_SZ + 2, iv_length); return crypt_data; @@ -335,15 +359,9 @@ fil_space_destroy_crypt_data( fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { - /* Make sure that this thread owns the crypt_data - and make it unawailable, this does not fully - avoid the race between drop table and crypt thread */ - mutex_enter(&(*crypt_data)->mutex); - (*crypt_data)->inited = false; - mutex_exit(&(*crypt_data)->mutex); - mutex_free(& (*crypt_data)->mutex); - memset(*crypt_data, 0, sizeof(fil_space_crypt_t)); - free(*crypt_data); + fil_space_crypt_t* c = *crypt_data; + c->~fil_space_crypt_struct(); + mem_free(c); (*crypt_data) = NULL; } } @@ -491,6 +509,7 @@ fil_parse_write_crypt_data( } fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(encryption, key_id); + /* Need to overwrite these as above will initialize fields. */ crypt_data->page0_offset = offset; crypt_data->min_key_version = min_key_version; crypt_data->encryption = encryption; @@ -648,7 +667,7 @@ fil_space_encrypt( return src_frame; } - ut_a(crypt_data != NULL && crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF); + ut_a(crypt_data != NULL && crypt_data->is_encrypted()); byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame); @@ -729,7 +748,7 @@ fil_space_check_encryption_read( return false; } - if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->not_encrypted()) { return false; } @@ -781,7 +800,7 @@ fil_space_decrypt( return false; } - ut_a(crypt_data != NULL && crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF); + ut_a(crypt_data != NULL && crypt_data->is_encrypted()); /* read space & lsn */ ulint header_len = FIL_PAGE_DATA; @@ -1019,20 +1038,13 @@ Copy global key state */ static void fil_crypt_get_key_state( /*====================*/ - key_state_t *new_state) /*!< out: key state */ + key_state_t* new_state, /*!< out: key state */ + fil_space_crypt_t* crypt_data) /*!< in, out: crypt_data */ { if (srv_encrypt_tables) { - new_state->key_version = - encryption_key_get_latest_version(new_state->key_id); + new_state->key_version = crypt_data->key_get_latest_version(); new_state->rotate_key_age = srv_fil_crypt_rotate_key_age; - if (new_state->key_version == ENCRYPTION_KEY_VERSION_INVALID) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Used key_id %u can't be found from key file.", - new_state->key_id); - } - - ut_a(new_state->key_version != ENCRYPTION_KEY_VERSION_INVALID); ut_a(new_state->key_version != ENCRYPTION_KEY_NOT_ENCRYPTED); } else { new_state->key_version = 0; @@ -1092,9 +1104,7 @@ fil_crypt_is_closing( fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); if (crypt_data) { - mutex_enter(&crypt_data->mutex); - closing = crypt_data->closing; - mutex_exit(&crypt_data->mutex); + closing = crypt_data->is_closing(false); } return closing; @@ -1349,11 +1359,17 @@ fil_crypt_space_needs_rotation( } return false; } + + crypt_data->key_get_latest_version(); + + if (!crypt_data->is_key_found()) { + return false; + } } /* If used key_id is not found from encryption plugin we can't continue to rotate the tablespace */ - if (fil_crypt_get_latest_key_version(crypt_data) == ENCRYPTION_KEY_VERSION_INVALID) { + if (!crypt_data->is_key_found()) { return false; } @@ -1368,7 +1384,7 @@ fil_crypt_space_needs_rotation( } /* prevent threads from starting to rotate space */ - if (crypt_data->closing) { + if (crypt_data->is_closing(true)) { break; } @@ -1377,13 +1393,13 @@ fil_crypt_space_needs_rotation( } /* No need to rotate space if encryption is disabled */ - if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->not_encrypted()) { break; } if (crypt_data->key_id != key_state->key_id) { key_state->key_id= crypt_data->key_id; - fil_crypt_get_key_state(key_state); + fil_crypt_get_key_state(key_state, crypt_data); } bool need_key_rotation = fil_crypt_needs_rotation( @@ -1396,12 +1412,14 @@ fil_crypt_space_needs_rotation( time_t diff = time(0) - crypt_data->rotate_state.scrubbing. last_scrub_completed; + bool need_scrubbing = crypt_data->rotate_state.scrubbing.is_active && diff >= srv_background_scrub_data_interval; - if (need_key_rotation == false && need_scrubbing == false) + if (need_key_rotation == false && need_scrubbing == false) { break; + } mutex_exit(&crypt_data->mutex); /* NOTE! fil_decr_pending_ops is performed outside */ @@ -1617,8 +1635,9 @@ fil_crypt_find_space_to_rotate( os_event_wait_time(fil_crypt_threads_event, 1000000); } - if (state->should_shutdown()) + if (state->should_shutdown()) { return false; + } if (state->first) { state->first = false; @@ -1680,7 +1699,7 @@ fil_crypt_start_rotate_space( crypt_data->rotate_state.start_time = time(0); if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && - crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF && + crypt_data->is_encrypted() && key_state->key_version != 0) { /* this is rotation unencrypted => encrypted */ crypt_data->type = CRYPT_SCHEME_1; @@ -1717,7 +1736,7 @@ fil_crypt_find_page_to_rotate( mutex_enter(&crypt_data->mutex); ut_ad(key_state->key_id == crypt_data->key_id); - if (crypt_data->closing == false && + if (!crypt_data->is_closing(true) && crypt_data->rotate_state.next_offset < crypt_data->rotate_state.max_offset) { @@ -1974,7 +1993,7 @@ fil_crypt_rotate_page( /* statistics */ state->crypt_stat.pages_modified++; } else { - if (crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->is_encrypted()) { ut_a(kv >= crypt_data->min_key_version || (kv == 0 && key_state->key_version == 0)); @@ -2177,7 +2196,7 @@ fil_crypt_complete_rotate_space( fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); /* Space might already be dropped */ - if (crypt_data != NULL && crypt_data->inited) { + if (crypt_data != NULL && !crypt_data->is_closing(false)) { mutex_enter(&crypt_data->mutex); /** @@ -2513,7 +2532,7 @@ fil_space_crypt_close_tablespace( fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - if (crypt_data == NULL || !crypt_data->inited) { + if (crypt_data == NULL || crypt_data->is_closing(false)) { mutex_exit(&fil_crypt_threads_mutex); return; } @@ -2566,6 +2585,8 @@ fil_space_crypt_get_status( { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); + memset(status, 0, sizeof(*status)); + if (crypt_data != NULL) { status->space = id; status->scheme = crypt_data->type; @@ -2586,6 +2607,7 @@ fil_space_crypt_get_status( } else { status->rotating = false; } + mutex_exit(&crypt_data->mutex); if (srv_encrypt_tables || crypt_data->min_key_version) { @@ -2595,7 +2617,6 @@ fil_space_crypt_get_status( status->current_key_version = 0; } } else { - memset(status, 0, sizeof(*status)); if (srv_encrypt_tables) { os_event_set(fil_crypt_threads_event); } @@ -2628,6 +2649,7 @@ fil_space_get_scrub_status( struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); + memset(status, 0, sizeof(*status)); if (crypt_data != NULL) { @@ -2650,9 +2672,8 @@ fil_space_get_scrub_status( } else { status->scrubbing = false; } + mutex_exit(&crypt_data->mutex); - } else { - memset(status, 0, sizeof(*status)); } return crypt_data == NULL ? 1 : 0; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 0604e1ee55f55..28f262b50c711 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -2118,9 +2118,7 @@ fil_read_first_page( /* If file space is encrypted we need to have at least some encryption service available where to get keys */ - if ((cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (cdata && cdata->should_encrypt()) { if (!encryption_key_id_exists(cdata->key_id)) { ib_logf(IB_LOG_LEVEL_ERROR, @@ -6627,10 +6625,7 @@ fil_iterate( bool encrypted = false; /* Use additional crypt io buffer if tablespace is encrypted */ - if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (iter.crypt_data != NULL && iter.crypt_data->should_encrypt()) { encrypted = true; readptr = iter.crypt_io_buffer; writeptr = iter.crypt_io_buffer; @@ -7399,7 +7394,7 @@ fil_space_get_crypt_data( if (!space->page_0_crypt_read) { ib_logf(IB_LOG_LEVEL_WARN, - "Space %lu name %s contains encryption %d information for key_id %d but page0 is not read.", + "Space %lu name %s contains encryption %d information for key_id %u but page0 is not read.", space->id, space->name, space->crypt_data ? space->crypt_data->encryption : 0, diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 85a35dabde2b1..984d508bd0424 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1320,6 +1320,8 @@ static SHOW_VAR innodb_status_variables[]= { {"scrub_background_page_split_failures_unknown", (char*) &export_vars.innodb_scrub_page_split_failures_unknown, SHOW_LONG}, + {"encryption_num_key_requests", + (char*) &export_vars.innodb_encryption_key_requests, SHOW_LONGLONG}, {NullS, NullS, SHOW_LONG} }; @@ -6714,9 +6716,8 @@ ha_innobase::open( or used key_id is not available. */ if (ib_table) { fil_space_crypt_t* crypt_data = ib_table->crypt_data; - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + + if (crypt_data && crypt_data->should_encrypt()) { if (!encryption_key_id_exists(crypt_data->key_id)) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, @@ -22712,12 +22713,12 @@ ib_push_warning( const char *format,/*!< in: warning message */ ...) { - va_list args; - THD *thd = (THD *)trx->mysql_thd; - char *buf; + if (trx && trx->mysql_thd) { + THD *thd = (THD *)trx->mysql_thd; + va_list args; + char *buf; #define MAX_BUF_SIZE 4*1024 - if (thd) { va_start(args, format); buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); vsprintf(buf,format, args); diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index b656cd3985d76..8bb0ce65a6bee 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -75,6 +75,21 @@ struct key_struct (that is L in CRYPT_SCHEME_1) */ }; +/** is encryption enabled */ +extern ulong srv_encrypt_tables; + +#ifdef UNIV_PFS_MUTEX +extern mysql_pfs_key_t fil_crypt_data_mutex_key; +#endif + +/** Mutex helper for crypt_data->scheme +@param[in, out] schme encryption scheme +@param[in] exit should we exit or enter mutex ? */ +void +crypt_data_scheme_locker( + st_encryption_scheme* scheme, + int exit); + struct fil_space_rotate_state_t { time_t start_time; /*!< time when rotation started */ @@ -96,13 +111,110 @@ struct fil_space_rotate_state_t struct fil_space_crypt_struct : st_encryption_scheme { + public: + /** Constructor. Does not initialize the members! + The object is expected to be placed in a buffer that + has been zero-initialized. */ + fil_space_crypt_struct( + ulint new_type, + uint new_min_key_version, + uint new_key_id, + ulint offset, + fil_encryption_t new_encryption) + : st_encryption_scheme(), + min_key_version(new_min_key_version), + page0_offset(offset), + encryption(new_encryption), + closing(false), + key_found(), + rotate_state() + { + key_found = new_min_key_version; + key_id = new_key_id; + my_random_bytes(iv, sizeof(iv)); + mutex_create(fil_crypt_data_mutex_key, + &mutex, SYNC_NO_ORDER_CHECK); + locker = crypt_data_scheme_locker; + type = new_type; + + if (new_encryption == FIL_SPACE_ENCRYPTION_OFF || + (!srv_encrypt_tables && + new_encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + type = CRYPT_SCHEME_UNENCRYPTED; + } else { + type = CRYPT_SCHEME_1; + min_key_version = key_get_latest_version(); + } + } + + /** Destructor */ + ~fil_space_crypt_struct() + { + closing = true; + mutex_free(&mutex); + } + + /** Get latest key version from encryption plugin + @retval key_version or + @retval ENCRYPTION_KEY_VERSION_INVALID if used key_id + is not found from encryption plugin. */ + uint key_get_latest_version(void); + + /** Returns true if key was found from encryption plugin + and false if not. */ + bool is_key_found() const { + return key_found != ENCRYPTION_KEY_VERSION_INVALID; + } + + /** Returns true if tablespace should be encrypted */ + bool should_encrypt() const { + return ((encryption == FIL_SPACE_ENCRYPTION_ON) || + (srv_encrypt_tables && + encryption == FIL_SPACE_ENCRYPTION_DEFAULT)); + } + + /** Return true if tablespace is encrypted. */ + bool is_encrypted() const { + return (encryption != FIL_SPACE_ENCRYPTION_OFF); + } + + /** Return true if default tablespace encryption is used, */ + bool is_default_encryption() const { + return (encryption == FIL_SPACE_ENCRYPTION_DEFAULT); + } + + /** Return true if tablespace is not encrypted. */ + bool not_encrypted() const { + return (encryption == FIL_SPACE_ENCRYPTION_OFF); + } + + /** Is this tablespace closing. */ + bool is_closing(bool is_fixed) { + bool closed; + if (!is_fixed) { + mutex_enter(&mutex); + } + closed = closing; + if (!is_fixed) { + mutex_exit(&mutex); + } + return closed; + } + uint min_key_version; // min key version for this space ulint page0_offset; // byte offset on page 0 for crypt data fil_encryption_t encryption; // Encryption setup ib_mutex_t mutex; // mutex protecting following variables bool closing; // is tablespace being closed - bool inited; + + /** Return code from encryption_key_get_latest_version. + If ENCRYPTION_KEY_VERSION_INVALID encryption plugin + could not find the key and there is no need to call + get_latest_key_version again as keys are read only + at startup. */ + uint key_found; + fil_space_rotate_state_t rotate_state; }; @@ -424,7 +536,6 @@ fil_crypt_calculate_checksum( ulint zip_size, /*!< in: zip_size or 0 */ byte* dst_frame); /*!< in: page where to calculate */ - #ifndef UNIV_NONINL #include "fil0crypt.ic" #endif diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 454d9294a1ce8..aa7e0452792a5 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -189,6 +189,9 @@ struct srv_stats_t { /** Number of times page 0 is read from tablespace */ ulint_ctr_64_t page0_read; + + /** Number of encryption_get_latest_key_version calls */ + ulint_ctr_64_t n_key_requests; }; extern const char* srv_main_thread_op_info; @@ -1275,6 +1278,7 @@ struct export_var_t{ ulint innodb_encryption_rotation_pages_modified; ulint innodb_encryption_rotation_pages_flushed; ulint innodb_encryption_rotation_estimated_iops; + ib_int64_t innodb_encryption_key_requests; ulint innodb_scrub_page_reorganizations; ulint innodb_scrub_page_splits; diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 8534b30d85483..9f182fc5e7095 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -226,10 +226,7 @@ row_fts_psort_info_init( common_info->opt_doc_id_size = opt_doc_id_size; crypt_data = fil_space_get_crypt_data(new_table->space); - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (crypt_data && crypt_data->should_encrypt()) { common_info->crypt_data = crypt_data; encrypted = true; } else { diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 22ae1eac4527b..fac6feed62f5e 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -4002,10 +4002,7 @@ row_merge_build_indexes( /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (crypt_data && crypt_data->should_encrypt()) { crypt_block = static_cast( os_mem_alloc_large(&block_size)); diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index f9c75ffe5760b..eff680a4ef0d6 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -2076,6 +2076,8 @@ srv_export_innodb_status(void) crypt_stat.pages_flushed; export_vars.innodb_encryption_rotation_estimated_iops = crypt_stat.estimated_iops; + export_vars.innodb_encryption_key_requests = + srv_stats.n_key_requests; export_vars.innodb_scrub_page_reorganizations = scrub_stat.page_reorganizations; From 25a9a3da346b6d763cf77ce82b82c65f4b1769cb Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 08:39:36 +0530 Subject: [PATCH 267/295] Revert "MDEV-11016 wsrep_node_is_ready() check is too strict" This reverts commit 7ed5563bbee301bf8217080dc78ea6a3e78e23a8. --- .../suite/galera/r/galera_var_dirty_reads.result | 13 ------------- .../suite/galera/t/galera_var_dirty_reads.test | 7 ------- sql/sql_parse.cc | 5 +---- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6a2aa1eb5e71e..6d703c8cf95a7 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,19 +18,6 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 -SET @@session.wsrep_dirty_reads=OFF; -SELECT 2; -2 -2 -SELECT @@max_allowed_packet; -@@max_allowed_packet -4194304 -SELECT 2+2 from DUAL; -2+2 -4 -SELECT sysdate() from DUAL; -sysdate() -2016-10-28 23:13:06 SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 0c81779ca6555..dfd8d5ecf2905 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -36,13 +36,6 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; -#Select query which does not access table should be allowed MDEV-11016 -SET @@session.wsrep_dirty_reads=OFF; -SELECT 2; -SELECT @@max_allowed_packet; -SELECT 2+2 from DUAL; -SELECT sysdate() from DUAL; - --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 7d5c59f77e330..5169b7c557356 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2650,15 +2650,12 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. We however, allow SET, - SHOW and SELECT queries (only if wsrep_dirty_reads is set or when it - does not access any table). + SHOW and SELECT queries (only if wsrep_dirty_reads is set). */ if (lex->sql_command != SQLCOM_SET_OPTION && !wsrep_is_show_query(lex->sql_command) && !(thd->variables.wsrep_dirty_reads && lex->sql_command == SQLCOM_SELECT) && - !(lex->sql_command == SQLCOM_SELECT && - !all_tables) && !wsrep_node_is_ready(thd)) goto error; } From 0c79de2419194f850e92de6ffa48d2e92055ee50 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 09:30:43 +0530 Subject: [PATCH 268/295] MDEV-11479 Improved wsrep_dirty_reads Tasks:- Changes in wsrep_dirty_reads variable 1.) Global + Session scope (Current: session-only) 2.) Can be set using command line. 3.) Allow all commands that do not change data (besides SELECT) 4.) Allow prepared Statements that do not change data 5.) Works with wsrep_sync_wait enabled --- .../galera/r/galera_var_dirty_reads.result | 70 +++++++++++++++++++ .../galera/t/galera_var_dirty_reads.test | 68 ++++++++++++++++++ .../sys_vars/r/wsrep_dirty_reads_basic.result | 23 +++++- .../sys_vars/t/wsrep_dirty_reads_basic.test | 17 ++++- sql/sql_parse.cc | 9 ++- sql/sys_vars.cc | 5 +- sql/wsrep_mysqld.cc | 4 ++ sql/wsrep_mysqld.h | 1 + 8 files changed, 188 insertions(+), 9 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6d703c8cf95a7..8a3175912c70b 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -3,6 +3,10 @@ INSERT INTO t1 VALUES(1); SELECT * FROM t1; i 1 +create user user1; +grant all privileges on *.* to user1; +create user user2; +grant all privileges on *.* to user2; SET @@global.wsrep_cluster_address = ''; SET @@session.wsrep_dirty_reads=OFF; SET SESSION wsrep_sync_wait=0; @@ -18,8 +22,74 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +connect con1, localhost, user1,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2; +SET SESSION wsrep_sync_wait=0; +set session wsrep_dirty_reads=1; +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +set session wsrep_dirty_reads=0; +execute stmt_show; +ERROR 08S01: WSREP has not yet prepared node for application use +execute stmt_select; +ERROR 08S01: WSREP has not yet prepared node for application use +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET wsrep_dirty_reads=ON; +select @@session.wsrep_dirty_reads; +@@session.wsrep_dirty_reads +1 +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET @@global.wsrep_dirty_reads=ON; +connect con2, localhost, user2,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2; +select @@session.wsrep_dirty_reads; +@@session.wsrep_dirty_reads +1 +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET SESSION wsrep_sync_wait=1; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET SESSION wsrep_sync_wait=7; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +connection node_2; +SET @@global.wsrep_dirty_reads=OFF; +connection node_1; SELECT * FROM t1; i 1 DROP TABLE t1; +drop user user1; +drop user user2; +disconnect node_2; +disconnect node_1; # End of test diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index dfd8d5ecf2905..bcdb1574a3dd5 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -17,6 +17,11 @@ CREATE TABLE t1(i INT) ENGINE=INNODB; INSERT INTO t1 VALUES(1); SELECT * FROM t1; +create user user1; +grant all privileges on *.* to user1; +create user user2; +grant all privileges on *.* to user2; + SET @@global.wsrep_cluster_address = ''; SET @@session.wsrep_dirty_reads=OFF; @@ -36,6 +41,67 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; +--enable_connect_log +--connect (con1, localhost, user1,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2) +#Just test the session behavior +SET SESSION wsrep_sync_wait=0; + +set session wsrep_dirty_reads=1; +#Prepared statement creation should be allowed MDEV-11479 +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +set session wsrep_dirty_reads=0; + +#No Preapare stmt/proceure will be allowed +--error ER_UNKNOWN_COM_ERROR +execute stmt_show; +--error ER_UNKNOWN_COM_ERROR +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +SET wsrep_dirty_reads=ON; +select @@session.wsrep_dirty_reads; +#Only prepare statement which does not change data should be allowed +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; +SET @@global.wsrep_dirty_reads=ON; + +--connect (con2, localhost, user2,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2) +#Just test the session behavior +select @@session.wsrep_dirty_reads; + +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; + +#Only prepare statement which does not change data should be allowed +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +#wsrep_dirty_read should work when wsrep_sync_wait is 1 or non zero +#because we already are disconnected , So It does not make any sense +#to wait for other nodes +SET SESSION wsrep_sync_wait=1; +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +SET SESSION wsrep_sync_wait=7; +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +--connection node_2 +SET @@global.wsrep_dirty_reads=OFF; + --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log @@ -45,6 +111,8 @@ SELECT * FROM t1; SELECT * FROM t1; # Cleanup DROP TABLE t1; +drop user user1; +drop user user2; # Restore original auto_increment_offset values. --source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result b/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result index d2a62d6136f7f..1968103873aaf 100644 --- a/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result +++ b/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result @@ -5,12 +5,13 @@ SET @wsrep_dirty_reads_session_saved = @@session.wsrep_dirty_reads; # default SELECT @@global.wsrep_dirty_reads; -ERROR HY000: Variable 'wsrep_dirty_reads' is a SESSION variable +@@global.wsrep_dirty_reads +0 SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads 0 -# scope and valid values +# valid values for session SET @@session.wsrep_dirty_reads=OFF; SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads @@ -24,11 +25,29 @@ SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads 0 +# valid values for global +SET @@global.wsrep_dirty_reads=OFF; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +0 +SET @@global.wsrep_dirty_reads=ON; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +1 +SET @@global.wsrep_dirty_reads=default; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +0 + # invalid values SET @@session.wsrep_dirty_reads=NULL; ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'NULL' SET @@session.wsrep_dirty_reads='junk'; ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'junk' +SET @@global.wsrep_dirty_reads=NULL; +ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'NULL' +SET @@global.wsrep_dirty_reads='junk'; +ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'junk' # restore the initial values SET @@session.wsrep_dirty_reads = @wsrep_dirty_reads_session_saved; diff --git a/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test b/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test index a47524fcfe3af..ffe767a051be0 100644 --- a/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test +++ b/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test @@ -8,12 +8,12 @@ SET @wsrep_dirty_reads_session_saved = @@session.wsrep_dirty_reads; --echo # default ---error ER_INCORRECT_GLOBAL_LOCAL_VAR + SELECT @@global.wsrep_dirty_reads; SELECT @@session.wsrep_dirty_reads; --echo ---echo # scope and valid values +--echo # valid values for session SET @@session.wsrep_dirty_reads=OFF; SELECT @@session.wsrep_dirty_reads; SET @@session.wsrep_dirty_reads=ON; @@ -21,12 +21,25 @@ SELECT @@session.wsrep_dirty_reads; SET @@session.wsrep_dirty_reads=default; SELECT @@session.wsrep_dirty_reads; +--echo +--echo # valid values for global +SET @@global.wsrep_dirty_reads=OFF; +SELECT @@global.wsrep_dirty_reads; +SET @@global.wsrep_dirty_reads=ON; +SELECT @@global.wsrep_dirty_reads; +SET @@global.wsrep_dirty_reads=default; +SELECT @@global.wsrep_dirty_reads; + --echo --echo # invalid values --error ER_WRONG_VALUE_FOR_VAR SET @@session.wsrep_dirty_reads=NULL; --error ER_WRONG_VALUE_FOR_VAR SET @@session.wsrep_dirty_reads='junk'; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.wsrep_dirty_reads=NULL; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.wsrep_dirty_reads='junk'; --echo --echo # restore the initial values diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 5169b7c557356..b214a1759be12 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2649,13 +2649,16 @@ mysql_execute_command(THD *thd) } /* - Bail out if DB snapshot has not been installed. We however, allow SET, - SHOW and SELECT queries (only if wsrep_dirty_reads is set). + Bail out if DB snapshot has not been installed. SET and SHOW commands, + however, are always allowed. + + We additionally allow all other commands that do not change data in + case wsrep_dirty_reads is enabled. */ if (lex->sql_command != SQLCOM_SET_OPTION && !wsrep_is_show_query(lex->sql_command) && !(thd->variables.wsrep_dirty_reads && - lex->sql_command == SQLCOM_SELECT) && + !is_update_query(lex->sql_command)) && !wsrep_node_is_ready(thd)) goto error; } diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 0518e434cbefe..bb349e78724c2 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -4976,8 +4976,9 @@ static Sys_var_mybool Sys_wsrep_restart_slave( GLOBAL_VAR(wsrep_restart_slave), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); static Sys_var_mybool Sys_wsrep_dirty_reads( - "wsrep_dirty_reads", "Do not reject SELECT queries even when the node " - "is not ready.", SESSION_ONLY(wsrep_dirty_reads), NO_CMD_LINE, + "wsrep_dirty_reads", + "Allow reads even when the node is not in the primary component.", + SESSION_VAR(wsrep_dirty_reads), CMD_LINE(OPT_ARG), DEFAULT(FALSE), NO_MUTEX_GUARD, NOT_IN_BINLOG); static Sys_var_uint Sys_wsrep_gtid_domain_id( diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 0deb19dfc7717..62d7b7ecff113 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -96,6 +96,8 @@ bool wsrep_new_cluster = false; // Bootstrap the cluster ? bool wsrep_gtid_mode = 0; // gtid_domain_id for galera transactions. uint32 wsrep_gtid_domain_id = 0; +// Allow reads even if the node is not in the primary component. +bool wsrep_dirty_reads = false; /* * End configuration options @@ -958,6 +960,8 @@ bool wsrep_must_sync_wait (THD* thd, uint mask) { return (thd->variables.wsrep_sync_wait & mask) && thd->variables.wsrep_on && + !(thd->variables.wsrep_dirty_reads && + !is_update_query(thd->lex->sql_command)) && !thd->in_active_multi_stmt_transaction() && thd->wsrep_conflict_state != REPLAYING && thd->wsrep_sync_wait_gtid.seqno == WSREP_SEQNO_UNDEFINED; diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 04ccc1a7e45d9..cf549ffe544c9 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -89,6 +89,7 @@ extern ulong wsrep_running_threads; extern bool wsrep_new_cluster; extern bool wsrep_gtid_mode; extern uint32 wsrep_gtid_domain_id; +extern bool wsrep_dirty_reads; enum enum_wsrep_OSU_method { WSREP_OSU_TOI, From f41bd7e54512d4e283ba909b4c3a700382cf1147 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 13 Dec 2016 05:07:02 +0530 Subject: [PATCH 269/295] MDEV-11060 sql/protocol.cc:532: void Protocol::end_statement(): Assertion `0' failed In file sql/opt_range.cc,when calculate_cond_selectivity_for_table() is called with optimizer_use_condition_selectivity=4 then - thd->no_errors is set to 1 - the original value of thd->no_error is not restored to its original value - this is causing the assertion to fail in the subsequent queries Fixed by restoring the original value of thd->no_errors --- mysql-test/r/selectivity_innodb.result | 20 ++++++++++++++++ mysql-test/t/selectivity_innodb.test | 33 ++++++++++++++++++++++++++ sql/opt_range.cc | 1 + 3 files changed, 54 insertions(+) diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 4355812d42f73..070cc49ca7f8d 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1670,6 +1670,26 @@ DROP TABLE t1,t2; # # End of 10.0 tests # +# +# Start of 10.1 tests +# +# +# MDEV-11060: sql/protocol.cc:532: void Protocol::end_statement(): Assertion `0' failed +# +set optimizer_use_condition_selectivity=4; +drop view if exists v1; +create table t1 (a int not null, b int, c int) engine=InnoDB; +create trigger trgi before insert on t1 for each row set new.a=if(new.a is null,new.b,new.c); +create table t2 (d int, e int) engine=InnoDB; +update t1, t2 set a=NULL, b=2, c=NULL where b=d and e=200; +create view v1 as select * from t1, t2 where d=2; +insert v1 (a,c) values (NULL, 20); +ERROR 23000: Column 'a' cannot be null +drop table t1,t2; +drop view v1; +# +# End of 10.1 tests +# set use_stat_tables= @tmp_ust; set optimizer_use_condition_selectivity= @tmp_oucs; SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/mysql-test/t/selectivity_innodb.test b/mysql-test/t/selectivity_innodb.test index 25aa0abbc3b45..e2dba03436387 100644 --- a/mysql-test/t/selectivity_innodb.test +++ b/mysql-test/t/selectivity_innodb.test @@ -138,6 +138,39 @@ DROP TABLE t1,t2; --echo # End of 10.0 tests --echo # + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-11060: sql/protocol.cc:532: void Protocol::end_statement(): Assertion `0' failed +--echo # + + +set optimizer_use_condition_selectivity=4; + +--disable_warnings +drop view if exists v1; +--enable_warnings + +create table t1 (a int not null, b int, c int) engine=InnoDB; +create trigger trgi before insert on t1 for each row set new.a=if(new.a is null,new.b,new.c); + +create table t2 (d int, e int) engine=InnoDB; +update t1, t2 set a=NULL, b=2, c=NULL where b=d and e=200; + +create view v1 as select * from t1, t2 where d=2; +--error ER_BAD_NULL_ERROR +insert v1 (a,c) values (NULL, 20); + +drop table t1,t2; +drop view v1; + +--echo # +--echo # End of 10.1 tests +--echo # + set use_stat_tables= @tmp_ust; set optimizer_use_condition_selectivity= @tmp_oucs; SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index bed57f3273f22..0577ef01bb2af 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -3112,6 +3112,7 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond) } free_alloc: + thd->no_errors= 0; thd->mem_root= param.old_root; free_root(&alloc, MYF(0)); From d93bbcad369c723c4ddf81a528576c9872d26d44 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 20:13:36 +0530 Subject: [PATCH 270/295] MDEV-11479 Improved wsrep_dirty_reads Updated sysvars_wsrep.result file. --- mysql-test/suite/sys_vars/r/sysvars_wsrep.result | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result index 36d04afb80d09..53838366a16f0 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result +++ b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result @@ -143,18 +143,18 @@ READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME WSREP_DIRTY_READS SESSION_VALUE OFF -GLOBAL_VALUE NULL +GLOBAL_VALUE OFF GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE OFF -VARIABLE_SCOPE SESSION ONLY +VARIABLE_SCOPE SESSION VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Do not reject SELECT queries even when the node is not ready. +VARIABLE_COMMENT Allow reads even when the node is not in the primary component. NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO -COMMAND_LINE_ARGUMENT NULL +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME WSREP_DRUPAL_282555_WORKAROUND SESSION_VALUE NULL GLOBAL_VALUE OFF From c13b5011629b5ff7b969d648265002e4d1ba94c2 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 14 Dec 2016 19:20:17 +0000 Subject: [PATCH 271/295] Fix broken cmake -DBUILD_CONFIG=mysql_release on Windows. mysql_release.cmake set WITH_JEMALLOC=static, which makes windows builds fail since there is no jemalloc either static or shared there --- cmake/build_configurations/mysql_release.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index ef9713e05568c..363cbac584d2f 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -83,7 +83,8 @@ IF(FEATURE_SET) ENDIF() OPTION(ENABLED_LOCAL_INFILE "" ON) -IF(RPM) +IF(WIN32) +ELSEIF(RPM) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") ELSEIF(DEB) From 8e198336c20a1bbce561dcdd8080dafcfe130233 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Thu, 15 Dec 2016 10:34:41 -0500 Subject: [PATCH 272/295] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7f6861d96b725..ad620531c9a3f 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=20 +MYSQL_VERSION_PATCH=21 From 8777458a6eb73ac1d7d864ebac390ea7039e21c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 14 Dec 2016 19:56:39 +0200 Subject: [PATCH 273/295] MDEV-6076 Persistent AUTO_INCREMENT for InnoDB This should be functionally equivalent to WL#6204 in MySQL 8.0.0, with the notable difference that the file format changes are limited to repurposing a previously unused data field in B-tree pages. For persistent InnoDB tables, write the last used AUTO_INCREMENT value to the root page of the clustered index, in the previously unused (0) PAGE_MAX_TRX_ID field, now aliased as PAGE_ROOT_AUTO_INC. Unlike some other previously unused InnoDB data fields, this one was actually always zero-initialized, at least since MySQL 3.23.49. The writes to PAGE_ROOT_AUTO_INC are protected by SX or X latch on the root page. The SX latch will allow concurrent read access to the root page. (The field PAGE_ROOT_AUTO_INC will only be read on the first-time call to ha_innobase::open() from the SQL layer. The PAGE_ROOT_AUTO_INC can only be updated when executing SQL, so read/write races are not possible.) During INSERT, the PAGE_ROOT_AUTO_INC is updated by the low-level function btr_cur_search_to_nth_level(), adding no extra page access. [Adaptive hash index lookup will be disabled during INSERT.] If some rare UPDATE modifies an AUTO_INCREMENT column, the PAGE_ROOT_AUTO_INC will be adjusted in a separate mini-transaction in ha_innobase::update_row(). When a page is reorganized, we have to preserve the PAGE_ROOT_AUTO_INC field. During ALTER TABLE, the initial AUTO_INCREMENT value will be copied from the table. ALGORITHM=COPY and online log apply in LOCK=NONE will update PAGE_ROOT_AUTO_INC in real time. innodb_col_no(): Determine the dict_table_t::cols[] element index corresponding to a Field of a non-virtual column. (The MySQL 5.7 implementation of virtual columns breaks the 1:1 relationship between Field::field_index and dict_table_t::cols[]. Virtual columns are omitted from dict_table_t::cols[]. Therefore, we must translate the field_index of AUTO_INCREMENT columns into an index of dict_table_t::cols[].) Upgrade from old data files: By default, the AUTO_INCREMENT sequence in old data files would appear to be reset, because PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC would contain the value 0 in each clustered index page. In new data files, PAGE_ROOT_AUTO_INC can only be 0 if the table is empty or does not contain any AUTO_INCREMENT column. For backward compatibility, we use the old method of SELECT MAX(auto_increment_column) for initializing the sequence. btr_read_autoinc(): Read the AUTO_INCREMENT sequence from a new-format data file. btr_read_autoinc_with_fallback(): A variant of btr_read_autoinc() that will resort to reading MAX(auto_increment_column) for data files that did not use AUTO_INCREMENT yet. It was manually tested that during the execution of innodb.autoinc_persist the compatibility logic is not activated (for new files, PAGE_ROOT_AUTO_INC is never 0 in nonempty clustered index root pages). initialize_auto_increment(): Replaces ha_innobase::innobase_initialize_autoinc(). This initializes the AUTO_INCREMENT metadata. Only called from ha_innobase::open(). ha_innobase::info_low(): Do not try to lazily initialize dict_table_t::autoinc. It must already have been initialized by ha_innobase::open() or ha_innobase::create(). Note: The adjustments to class ha_innopart were not tested, because the source code (native InnoDB partitioning) is not being compiled. --- .../include/kill_and_restart_mysqld.inc | 19 + .../innodb/include/autoinc_persist_alter.inc | 61 + .../suite/innodb/r/autoinc_persist.result | 1131 +++++++++++++++++ .../innodb/r/innodb-autoinc-44030.result | 11 +- mysql-test/suite/innodb/r/innodb.result | 8 +- .../suite/innodb/t/autoinc_persist.test | 533 ++++++++ .../innodb/t/innodb-autoinc-44030-master.opt | 3 - .../suite/innodb/t/innodb-autoinc-44030.test | 19 +- mysql-test/suite/innodb/t/innodb.test | 9 +- .../r/partition_auto_increment_innodb.result | 12 +- storage/innobase/btr/btr0btr.cc | 149 ++- storage/innobase/btr/btr0cur.cc | 50 +- storage/innobase/btr/btr0pcur.cc | 3 +- storage/innobase/dict/dict0dict.cc | 93 -- storage/innobase/dict/dict0mem.cc | 6 - storage/innobase/handler/ha_innodb.cc | 332 ++--- storage/innobase/handler/ha_innodb.h | 8 +- storage/innobase/handler/ha_innopart.cc | 99 +- storage/innobase/handler/ha_innopart.h | 13 +- storage/innobase/handler/handler0alter.cc | 174 ++- storage/innobase/include/btr0btr.h | 30 +- storage/innobase/include/btr0cur.h | 5 +- storage/innobase/include/btr0pcur.h | 4 +- storage/innobase/include/btr0pcur.ic | 4 +- storage/innobase/include/dict0dict.h | 97 +- storage/innobase/include/dict0dict.ic | 36 - storage/innobase/include/dict0mem.h | 5 + storage/innobase/include/page0page.h | 37 +- storage/innobase/include/page0page.ic | 12 + storage/innobase/include/row0row.h | 17 + storage/innobase/include/row0row.ic | 49 + storage/innobase/include/row0sel.h | 15 +- storage/innobase/page/page0page.cc | 34 + storage/innobase/page/page0zip.cc | 26 +- storage/innobase/row/row0import.cc | 19 +- storage/innobase/row/row0ins.cc | 39 +- storage/innobase/row/row0mysql.cc | 12 +- storage/innobase/row/row0sel.cc | 73 +- 38 files changed, 2531 insertions(+), 716 deletions(-) create mode 100644 mysql-test/include/kill_and_restart_mysqld.inc create mode 100644 mysql-test/suite/innodb/include/autoinc_persist_alter.inc create mode 100644 mysql-test/suite/innodb/r/autoinc_persist.result create mode 100644 mysql-test/suite/innodb/t/autoinc_persist.test delete mode 100644 mysql-test/suite/innodb/t/innodb-autoinc-44030-master.opt diff --git a/mysql-test/include/kill_and_restart_mysqld.inc b/mysql-test/include/kill_and_restart_mysqld.inc new file mode 100644 index 0000000000000..f2ac9b504d226 --- /dev/null +++ b/mysql-test/include/kill_and_restart_mysqld.inc @@ -0,0 +1,19 @@ +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect + +if ($restart_parameters) +{ + --echo # Kill and restart: $restart_parameters + --exec echo "restart: $restart_parameters" > $_expect_file_name +} +if (!$restart_parameters) +{ + --echo # Kill and restart + --exec echo "restart" > $_expect_file_name +} + +--shutdown_server 0 +--source include/wait_until_disconnected.inc +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect diff --git a/mysql-test/suite/innodb/include/autoinc_persist_alter.inc b/mysql-test/suite/innodb/include/autoinc_persist_alter.inc new file mode 100644 index 0000000000000..ddff573b0565f --- /dev/null +++ b/mysql-test/suite/innodb/include/autoinc_persist_alter.inc @@ -0,0 +1,61 @@ + +eval CREATE TABLE $table LIKE $template; + +eval INSERT INTO $table SELECT * FROM $template; + +eval SELECT * FROM $table; + +eval SHOW CREATE TABLE $table; + +--echo # This will keep the autoinc counter +eval ALTER TABLE $table AUTO_INCREMENT = 250, ALGORITHM = $algorithm; +--echo # We expect the counter to be 250 +eval SHOW CREATE TABLE $table; + +--echo # This should keep the autoinc counter as well +eval ALTER TABLE $table ADD COLUMN b INT, ALGORITHM = $algorithm; +--echo # We expect the counter to be 250 +eval SHOW CREATE TABLE $table; + +eval DELETE FROM $table WHERE a > 150; + +eval SELECT * FROM $table; + +--echo # This should reset the autoinc counter to the one specified +--echo # Since it's smaller than current one but bigger than existing +--echo # biggest counter in the table +eval ALTER TABLE $table AUTO_INCREMENT = 180, ALGORITHM = $algorithm; +--echo # We expect the counter to be 180 +eval SHOW CREATE TABLE $table; + +--echo # This should reset the autoinc counter to the next value of +--echo # current max counter in the table, since the specified value +--echo # is smaller than the existing biggest value(50 < 123) +eval ALTER TABLE $table DROP COLUMN b, AUTO_INCREMENT = 50, ALGORITHM = $algorithm; +--echo # We expect the counter to be 123 +eval SHOW CREATE TABLE $table; + +eval INSERT INTO $table VALUES(0), (0); + +eval SELECT MAX(a) AS `Expect 124` FROM $table; + +eval OPTIMIZE TABLE $table; + +eval SHOW CREATE TABLE $table; + +--source include/restart_mysqld.inc + +--echo # We expect the counter to still be 125 +eval SHOW CREATE TABLE $table; + +eval DELETE FROM $table WHERE a >= 123; + +eval CREATE UNIQUE INDEX idx_aa ON $table(a); + +--source include/restart_mysqld.inc + +eval INSERT INTO $table VALUES(0), (0); + +eval SELECT MAX(a) AS `Expect 126` FROM $table; + +eval DROP TABLE $table; diff --git a/mysql-test/suite/innodb/r/autoinc_persist.result b/mysql-test/suite/innodb/r/autoinc_persist.result new file mode 100644 index 0000000000000..b9eefe30ddc58 --- /dev/null +++ b/mysql-test/suite/innodb/r/autoinc_persist.result @@ -0,0 +1,1131 @@ +# +# MDEV-6076 Persistent AUTO_INCREMENT for InnoDB +# +# WL#6204 InnoDB persistent max value for autoinc columns +# +# Most of this test case is copied from the test innodb.autoinc_persist +# that was introduced in MySQL 8.0.0. The observable behaviour +# of MDEV-6076 is equivalent to WL#6204, with the exception that +# there is less buffering taking place and redo log checkpoints +# are not being treated specially. +# Due to less buffering, there is no debug instrumentation testing +# for MDEV-6076. +# +# Pre-create several tables +SET SQL_MODE='STRICT_ALL_TABLES'; +CREATE TABLE t1(a TINYINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t1 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31); +SELECT * FROM t1; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +CREATE TABLE t2(a TINYINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t2 VALUES(-5); +ERROR 22003: Out of range value for column 'a' at row 1 +INSERT INTO t2 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31); +SELECT * FROM t2; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +CREATE TABLE t3(a SMALLINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t3 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (1024), (4096); +SELECT * FROM t3; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +1024 +4096 +CREATE TABLE t4(a SMALLINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t4 VALUES(-5); +ERROR 22003: Out of range value for column 'a' at row 1 +INSERT INTO t4 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (1024), (4096); +SELECT * FROM t4; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +1024 +4096 +CREATE TABLE t5(a MEDIUMINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t5 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (1000000), (1000005); +SELECT * FROM t5; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +1000000 +1000005 +CREATE TABLE t6(a MEDIUMINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t6 VALUES(-5); +ERROR 22003: Out of range value for column 'a' at row 1 +INSERT INTO t6 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (1000000), (1000005); +SELECT * FROM t6; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +1000000 +1000005 +CREATE TABLE t7(a INT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t7 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (100000000), (100000008); +SELECT * FROM t7; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +100000000 +100000008 +CREATE TABLE t8(a INT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t8 VALUES(-5); +ERROR 22003: Out of range value for column 'a' at row 1 +INSERT INTO t8 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (100000000), (100000008); +SELECT * FROM t8; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +100000000 +100000008 +CREATE TABLE t9(a BIGINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t9 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (100000000000), (100000000006); +SELECT * FROM t9; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +100000000000 +100000000006 +CREATE TABLE t10(a BIGINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t10 VALUES(-5); +ERROR 22003: Out of range value for column 'a' at row 1 +INSERT INTO t10 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (100000000000), (100000000006); +SELECT * FROM t10; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +100000000000 +100000000006 +CREATE TABLE t11(a FLOAT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31); +SELECT * FROM t11; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +set global innodb_flush_log_at_trx_commit=1; +CREATE TABLE t12(a DOUBLE AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31); +SELECT * FROM t12; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +CREATE TABLE t13(a INT AUTO_INCREMENT PRIMARY KEY) ENGINE = InnoDB, +AUTO_INCREMENT = 1234; +# Scenario 1: Normal restart, to test if the counters are persisted +# We expect these results should be equal to above SELECTs +SELECT * FROM t1; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +SELECT * FROM t2; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +SELECT * FROM t3; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +1024 +4096 +SELECT * FROM t4; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +1024 +4096 +SELECT * FROM t5; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +1000000 +1000005 +SELECT * FROM t6; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +1000000 +1000005 +SELECT * FROM t7; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +100000000 +100000008 +SELECT * FROM t8; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +100000000 +100000008 +SELECT * FROM t9; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +100000000000 +100000000006 +SELECT * FROM t10; +a +1 +2 +3 +4 +8 +10 +11 +20 +30 +31 +100000000000 +100000000006 +SELECT * FROM t11; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +SELECT * FROM t12; +a +-10 +-1 +1 +2 +3 +4 +5 +20 +30 +31 +SELECT * FROM t13; +a +SHOW CREATE TABLE t13; +Table Create Table +t13 CREATE TABLE `t13` ( + `a` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=1234 DEFAULT CHARSET=latin1 +INSERT INTO t13 VALUES(0); +SELECT a AS `Expect 1234` FROM t13; +Expect 1234 +1234 +# Scenario 2: Delete some values, to test the counters should not be the +# one which is the largest in current table +set global innodb_flush_log_at_trx_commit=1; +DELETE FROM t1 WHERE a > 30; +SELECT MAX(a) AS `Expect 30` FROM t1; +Expect 30 +30 +DELETE FROM t3 WHERE a > 2000; +SELECT MAX(a) AS `Expect 2000` FROM t3; +Expect 2000 +1024 +DELETE FROM t5 WHERE a > 1000000; +SELECT MAX(a) AS `Expect 1000000` FROM t5; +Expect 1000000 +1000000 +DELETE FROM t7 WHERE a > 100000000; +SELECT MAX(a) AS `Expect 100000000` FROM t7; +Expect 100000000 +100000000 +DELETE FROM t9 WHERE a > 100000000000; +SELECT MAX(a) AS `Expect 100000000000` FROM t9; +Expect 100000000000 +100000000000 +INSERT INTO t1 VALUES(0), (0); +SELECT MAX(a) AS `Expect 33` FROM t1; +Expect 33 +33 +INSERT INTO t3 VALUES(0), (0); +SELECT MAX(a) AS `Expect 4098` FROM t3; +Expect 4098 +4098 +INSERT INTO t5 VALUES(0), (0); +SELECT MAX(a) AS `Expect 1000007` FROM t5; +Expect 1000007 +1000007 +INSERT INTO t7 VALUES(0), (0); +SELECT MAX(a) AS `Expect 100000010` FROM t7; +Expect 100000010 +100000010 +INSERT INTO t9 VALUES(0), (0); +SELECT MAX(a) AS `Expect 100000000008` FROM t9; +Expect 100000000008 +100000000008 +# Scenario 3: Insert some bigger counters, the next counter should start +# from there +INSERT INTO t1 VALUES(40), (0); +INSERT INTO t1 VALUES(42), (0); +SELECT a AS `Expect 43, 42` FROM t1 ORDER BY a DESC LIMIT 4; +Expect 43, 42 +43 +42 +41 +40 +INSERT INTO t3 VALUES(5000), (0); +INSERT INTO t3 VALUES(5010), (0); +SELECT a AS `Expect 5011, 5010` FROM t3 ORDER BY a DESC LIMIT 4; +Expect 5011, 5010 +5011 +5010 +5001 +5000 +INSERT INTO t5 VALUES(1000010), (0); +INSERT INTO t5 VALUES(1000020), (0); +SELECT a AS `Expect 1000021, 1000020` FROM t5 ORDER BY a DESC LIMIT 4; +Expect 1000021, 1000020 +1000021 +1000020 +1000011 +1000010 +INSERT INTO t7 VALUES(100000020), (0); +INSERT INTO t7 VALUES(100000030), (0); +SELECT a AS `Expect 100000031, 100000030` FROM t7 ORDER BY a DESC LIMIT 4; +Expect 100000031, 100000030 +100000031 +100000030 +100000021 +100000020 +INSERT INTO t9 VALUES(100000000010), (0); +INSERT INTO t9 VALUES(100000000020), (0); +SELECT a AS `Expect 100000000021, 100000000020` FROM t9 ORDER BY a DESC LIMIT 4; +Expect 100000000021, 100000000020 +100000000021 +100000000020 +100000000011 +100000000010 +# Scenario 4: Update some values, to test the counters should be updated +# to the bigger value, but not smaller value. +INSERT INTO t1 VALUES(50), (55); +UPDATE t1 SET a = 105 WHERE a = 5; +UPDATE t1 SET a = 100 WHERE a = 55; +# This should insert 102, 106, 107, and make next counter 109. +INSERT INTO t1 VALUES(102), (0), (0); +SELECT a AS `Expect 107, 106` FROM t1 ORDER BY a DESC LIMIT 2; +Expect 107, 106 +107 +106 +DELETE FROM t1 WHERE a > 105; +INSERT INTO t1 VALUES(0); +SELECT MAX(a) AS `Expect 109` FROM t1; +Expect 109 +109 +# Test the same things on t3, t5, t7, t9, to test if DDTableBuffer would +# be updated accordingly +INSERT INTO t3 VALUES(60), (65); +UPDATE t3 SET a = 6005 WHERE a = 5; +UPDATE t3 SET a = 6000 WHERE a = 60; +# This should insert 6002, 6006, 6007, and make next counter 6009. +INSERT INTO t3 VALUES(6002), (0), (0); +SELECT a AS `Expect 6007, 6006` FROM t3 ORDER BY a DESC LIMIT 2; +Expect 6007, 6006 +6007 +6006 +DELETE FROM t3 WHERE a > 6005; +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 6009` FROM t3; +Expect 6009 +6009 +INSERT INTO t5 VALUES(100), (200); +UPDATE t5 SET a = 1000105 WHERE a = 5; +UPDATE t5 SET a = 1000100 WHERE a = 100; +# This should insert 1000102, 1000106, 1000107, and make next counter +# 1000109. +INSERT INTO t5 VALUES(1000102), (0), (0); +SELECT a AS `Expect 1000107, 1000106` FROM t5 ORDER BY a DESC LIMIT 2; +Expect 1000107, 1000106 +1000107 +1000106 +DELETE FROM t5 WHERE a > 1000105; +INSERT INTO t5 VALUES(0); +SELECT MAX(a) AS `Expect 1000109` FROM t5; +Expect 1000109 +1000109 +INSERT INTO t7 VALUES(100), (200); +UPDATE t7 SET a = 100000105 WHERE a = 5; +UPDATE t7 SET a = 100000100 WHERE a = 100; +# This should insert 100000102, 1100000106, 100000107, and make next +# counter 100000109. +INSERT INTO t7 VALUES(100000102), (0), (0); +SELECT a AS `Expect 100000107, 100000106` FROM t7 ORDER BY a DESC LIMIT 2; +Expect 100000107, 100000106 +100000107 +100000106 +DELETE FROM t7 WHERE a > 100000105; +INSERT INTO t7 VALUES(0); +SELECT MAX(a) AS `Expect 100000109` FROM t7; +Expect 100000109 +100000109 +set global innodb_flush_log_at_trx_commit=1; +INSERT INTO t9 VALUES(100), (200); +UPDATE t9 SET a = 100000000105 WHERE a = 5; +UPDATE t9 SET a = 100000000100 WHERE a = 100; +# This should insert 100000000102, 100000000106, 100000000107, and make +# next counter 100000000109. +INSERT INTO t9 VALUES(100000000102), (0), (0); +SELECT a AS `Expect 100000000107, 100000000106` FROM t9 ORDER BY a DESC LIMIT 2; +Expect 100000000107, 100000000106 +100000000107 +100000000106 +DELETE FROM t9 WHERE a > 100000000105; +INSERT INTO t9 VALUES(0); +SELECT MAX(a) AS `Expect 100000000109` FROM t9; +Expect 100000000109 +100000000109 +INSERT INTO t1 VALUES(0), (0); +SELECT a AS `Expect 110, 111` FROM t1 ORDER BY a DESC LIMIT 2; +Expect 110, 111 +111 +110 +INSERT INTO t3 VALUES(0), (0); +SELECT a AS `Expect 6010, 6011` FROM t3 ORDER BY a DESC LIMIT 2; +Expect 6010, 6011 +6011 +6010 +INSERT INTO t5 VALUES(0), (0); +SELECT a AS `Expect 1100111, 1100110` FROM t5 ORDER BY a DESC LIMIT 2; +Expect 1100111, 1100110 +1000111 +1000110 +INSERT INTO t7 VALUES(0), (0); +SELECT a AS `Expect 100000111, 100000110` FROM t7 ORDER BY a DESC LIMIT 2; +Expect 100000111, 100000110 +100000111 +100000110 +INSERT INTO t9 VALUES(0), (0); +SELECT a AS `Expect 100000000111, 100000000110` FROM t9 ORDER BY a DESC LIMIT 2; +Expect 100000000111, 100000000110 +100000000111 +100000000110 +# Scenario 5: Test kill the server +INSERT INTO t1 VALUES(125); +DELETE FROM t1 WHERE a = 125; +INSERT INTO t3 VALUES(6100); +DELETE FROM t3 WHERE a = 6100; +INSERT INTO t5 VALUES(1100200); +DELETE FROM t5 WHERE a = 1100200; +INSERT INTO t7 VALUES(100000200); +DELETE FROM t7 WHERE a = 100000200; +set global innodb_flush_log_at_trx_commit=1; +INSERT INTO t9 VALUES(100000000200); +DELETE FROM t9 WHERE a = 100000000200; +# Kill and restart +INSERT INTO t1 VALUES(0); +SELECT a AS `Expect 126` FROM t1 ORDER BY a DESC LIMIT 1; +Expect 126 +126 +INSERT INTO t3 VALUES(0); +SELECT a AS `Expect 6101` FROM t3 ORDER BY a DESC LIMIT 1; +Expect 6101 +6101 +INSERT INTO t5 VALUES(0); +SELECT a AS `Expect 1100201` FROM t5 ORDER BY a DESC LIMIT 1; +Expect 1100201 +1100201 +INSERT INTO t7 VALUES(0); +SELECT a AS `Expect 100000201` FROM t7 ORDER BY a DESC LIMIT 1; +Expect 100000201 +100000201 +INSERT INTO t9 VALUES(0); +SELECT a AS `Expect 100000000201` FROM t9 ORDER BY a DESC LIMIT 1; +Expect 100000000201 +100000000201 +# Scenario 6: Test truncate will reset the counters to 0 +TRUNCATE TABLE t1; +TRUNCATE TABLE t3; +TRUNCATE TABLE t5; +TRUNCATE TABLE t7; +TRUNCATE TABLE t9; +INSERT INTO t1 VALUES(0), (0); +SELECT * FROM t1; +a +1 +2 +INSERT INTO t3 VALUES(0), (0); +SELECT * FROM t3; +a +1 +2 +INSERT INTO t5 VALUES(0), (0); +SELECT * FROM t5; +a +1 +2 +INSERT INTO t7 VALUES(0), (0); +SELECT * FROM t7; +a +1 +2 +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; +a +1 +2 +set global innodb_flush_log_at_trx_commit=1; +TRUNCATE TABLE t1; +TRUNCATE TABLE t3; +TRUNCATE TABLE t5; +TRUNCATE TABLE t7; +TRUNCATE TABLE t9; +# Kill and restart +INSERT INTO t1 VALUES(0), (0); +SELECT * FROM t1; +a +1 +2 +INSERT INTO t3 VALUES(0), (0); +SELECT * FROM t3; +a +1 +2 +INSERT INTO t5 VALUES(0), (0); +SELECT * FROM t5; +a +1 +2 +INSERT INTO t7 VALUES(0), (0); +SELECT * FROM t7; +a +1 +2 +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; +a +1 +2 +# Scenario 7: Test explicit rename table won't change the counter +set global innodb_flush_log_at_trx_commit=1; +RENAME TABLE t9 to t19; +INSERT INTO t19 VALUES(0), (0); +SELECT * FROM t19; +a +1 +2 +3 +4 +DELETE FROM t19 WHERE a = 4; +# Kill and restart +RENAME TABLE t19 to t9; +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; +a +1 +2 +3 +5 +6 +TRUNCATE TABLE t9; +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; +a +1 +2 +# Scenario 8: Test ALTER TABLE operations +INSERT INTO t3 VALUES(0), (0), (100), (200), (1000); +SELECT * FROM t3; +a +1 +2 +3 +4 +100 +200 +1000 +DELETE FROM t3 WHERE a > 300; +SELECT MAX(a) AS `Expect 200` FROM t3; +Expect 200 +200 +# This will not change the counter to 150, but to 201, which is the next +# of current max counter in the table +ALTER TABLE t3 AUTO_INCREMENT = 150; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=201 DEFAULT CHARSET=latin1 +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 201` FROM t3; +Expect 201 +201 +# This will change the counter to 500, which is bigger than any counter +# in the table +ALTER TABLE t3 AUTO_INCREMENT = 500; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=500 DEFAULT CHARSET=latin1 +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 500` FROM t3; +Expect 500 +500 +TRUNCATE TABLE t3; +ALTER TABLE t3 AUTO_INCREMENT = 100; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=100 DEFAULT CHARSET=latin1 +INSERT INTO t3 VALUES(0), (0); +SELECT * FROM t3; +a +100 +101 +INSERT INTO t3 VALUES(150), (180); +UPDATE t3 SET a = 200 WHERE a = 150; +INSERT INTO t3 VALUES(220); +# This still fails to set to 120, but just 221 +ALTER TABLE t3 AUTO_INCREMENT = 120; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=221 DEFAULT CHARSET=latin1 +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 221` FROM t3; +Expect 221 +221 +DELETE FROM t3 WHERE a > 120; +ALTER TABLE t3 AUTO_INCREMENT = 120; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=120 DEFAULT CHARSET=latin1 +# MDEV-6076: Test adding an AUTO_INCREMENT COLUMN +CREATE TABLE mdev6076a (b INT) ENGINE=InnoDB; +INSERT INTO mdev6076a VALUES(2),(1); +CREATE TABLE mdev6076b (b INT) ENGINE=InnoDB; +INSERT INTO mdev6076b VALUES(2),(1); +ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, LOCK=NONE; +ERROR 0A000: LOCK=NONE is not supported. Reason: Adding an auto-increment column requires a lock. Try LOCK=SHARED +ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, ALGORITHM=INPLACE; +ALTER TABLE mdev6076b ADD COLUMN a SERIAL FIRST, AUTO_INCREMENT=100, +ALGORITHM=INPLACE; +# Kill and restart +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 120` FROM t3; +Expect 120 +120 +INSERT INTO mdev6076a SET b=0; +SELECT * FROM mdev6076a; +a b +1 2 +2 1 +3 0 +INSERT INTO mdev6076b SET b=0; +SELECT * FROM mdev6076b; +a b +100 2 +101 1 +102 0 +DROP TABLE mdev6076a, mdev6076b; +set global innodb_flush_log_at_trx_commit=1; +INSERT INTO t3 VALUES(0), (0), (200), (210); +# Test the different algorithms in ALTER TABLE +CREATE TABLE t_inplace LIKE t3; +INSERT INTO t_inplace SELECT * FROM t3; +SELECT * FROM t_inplace; +a +100 +101 +120 +121 +122 +200 +210 +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=211 DEFAULT CHARSET=latin1 +# This will keep the autoinc counter +ALTER TABLE t_inplace AUTO_INCREMENT = 250, ALGORITHM = INPLACE; +# We expect the counter to be 250 +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=250 DEFAULT CHARSET=latin1 +# This should keep the autoinc counter as well +ALTER TABLE t_inplace ADD COLUMN b INT, ALGORITHM = INPLACE; +# We expect the counter to be 250 +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + `b` int(11) DEFAULT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=250 DEFAULT CHARSET=latin1 +DELETE FROM t_inplace WHERE a > 150; +SELECT * FROM t_inplace; +a b +100 NULL +101 NULL +120 NULL +121 NULL +122 NULL +# This should reset the autoinc counter to the one specified +# Since it's smaller than current one but bigger than existing +# biggest counter in the table +ALTER TABLE t_inplace AUTO_INCREMENT = 180, ALGORITHM = INPLACE; +# We expect the counter to be 180 +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + `b` int(11) DEFAULT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=180 DEFAULT CHARSET=latin1 +# This should reset the autoinc counter to the next value of +# current max counter in the table, since the specified value +# is smaller than the existing biggest value(50 < 123) +ALTER TABLE t_inplace DROP COLUMN b, AUTO_INCREMENT = 50, ALGORITHM = INPLACE; +# We expect the counter to be 123 +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=123 DEFAULT CHARSET=latin1 +INSERT INTO t_inplace VALUES(0), (0); +SELECT MAX(a) AS `Expect 124` FROM t_inplace; +Expect 124 +124 +OPTIMIZE TABLE t_inplace; +Table Op Msg_type Msg_text +test.t_inplace optimize note Table does not support optimize, doing recreate + analyze instead +test.t_inplace optimize status OK +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 +# We expect the counter to still be 125 +SHOW CREATE TABLE t_inplace; +Table Create Table +t_inplace CREATE TABLE `t_inplace` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 +DELETE FROM t_inplace WHERE a >= 123; +CREATE UNIQUE INDEX idx_aa ON t_inplace(a); +INSERT INTO t_inplace VALUES(0), (0); +SELECT MAX(a) AS `Expect 126` FROM t_inplace; +Expect 126 +126 +DROP TABLE t_inplace; +CREATE TABLE t_copy LIKE t3; +INSERT INTO t_copy SELECT * FROM t3; +SELECT * FROM t_copy; +a +100 +101 +120 +121 +122 +200 +210 +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=211 DEFAULT CHARSET=latin1 +# This will keep the autoinc counter +ALTER TABLE t_copy AUTO_INCREMENT = 250, ALGORITHM = COPY; +# We expect the counter to be 250 +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=250 DEFAULT CHARSET=latin1 +# This should keep the autoinc counter as well +ALTER TABLE t_copy ADD COLUMN b INT, ALGORITHM = COPY; +# We expect the counter to be 250 +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + `b` int(11) DEFAULT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=250 DEFAULT CHARSET=latin1 +DELETE FROM t_copy WHERE a > 150; +SELECT * FROM t_copy; +a b +100 NULL +101 NULL +120 NULL +121 NULL +122 NULL +# This should reset the autoinc counter to the one specified +# Since it's smaller than current one but bigger than existing +# biggest counter in the table +ALTER TABLE t_copy AUTO_INCREMENT = 180, ALGORITHM = COPY; +# We expect the counter to be 180 +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + `b` int(11) DEFAULT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=180 DEFAULT CHARSET=latin1 +# This should reset the autoinc counter to the next value of +# current max counter in the table, since the specified value +# is smaller than the existing biggest value(50 < 123) +ALTER TABLE t_copy DROP COLUMN b, AUTO_INCREMENT = 50, ALGORITHM = COPY; +# We expect the counter to be 123 +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=123 DEFAULT CHARSET=latin1 +INSERT INTO t_copy VALUES(0), (0); +SELECT MAX(a) AS `Expect 124` FROM t_copy; +Expect 124 +124 +OPTIMIZE TABLE t_copy; +Table Op Msg_type Msg_text +test.t_copy optimize note Table does not support optimize, doing recreate + analyze instead +test.t_copy optimize status OK +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 +# We expect the counter to still be 125 +SHOW CREATE TABLE t_copy; +Table Create Table +t_copy CREATE TABLE `t_copy` ( + `a` smallint(6) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 +DELETE FROM t_copy WHERE a >= 123; +CREATE UNIQUE INDEX idx_aa ON t_copy(a); +INSERT INTO t_copy VALUES(0), (0); +SELECT MAX(a) AS `Expect 126` FROM t_copy; +Expect 126 +126 +DROP TABLE t_copy; +# Scenario 9: Test the sql_mode = NO_AUTO_VALUE_ON_ZERO +CREATE TABLE t30 (a INT NOT NULL AUTO_INCREMENT PRIMARY KEY, b INT, key(b)) ENGINE = InnoDB; +set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; +INSERT INTO t30 VALUES(NULL, 1), (200, 2), (0, 3); +INSERT INTO t30(b) VALUES(4), (5), (6), (7); +SELECT * FROM t30 ORDER BY b; +a b +1 1 +200 2 +0 3 +201 4 +202 5 +203 6 +204 7 +ALTER TABLE t30 MODIFY b MEDIUMINT; +SELECT * FROM t30 ORDER BY b; +a b +1 1 +200 2 +0 3 +201 4 +202 5 +203 6 +204 7 +set global innodb_flush_log_at_trx_commit=1; +CREATE TABLE t31 (a INT) ENGINE = InnoDB; +INSERT INTO t31 VALUES(1), (2); +ALTER TABLE t31 ADD b INT AUTO_INCREMENT PRIMARY KEY; +INSERT INTO t31 VALUES(3, 0), (4, NULL), (5, NULL); +INSERT INTO t31 VALUES(6, 0); +ERROR 23000: Duplicate entry '0' for key 'PRIMARY' +SELECT * FROM t31; +a b +3 0 +1 1 +2 2 +4 3 +5 4 +# Kill and restart +# This will not insert 0 +INSERT INTO t31(a) VALUES(6), (0); +SELECT * FROM t31; +a b +3 0 +1 1 +2 2 +4 3 +5 4 +6 5 +0 6 +DROP TABLE t31; +set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; +DELETE FROM t30 WHERE a = 0; +UPDATE t30 set a = 0 where b = 5; +SELECT * FROM t30 ORDER BY b; +a b +1 1 +200 2 +201 4 +0 5 +203 6 +204 7 +DELETE FROM t30 WHERE a = 0; +UPDATE t30 SET a = NULL WHERE b = 6; +Warnings: +Warning 1048 Column 'a' cannot be null +UPDATE t30 SET a = 300 WHERE b = 7; +SELECT * FROM t30 ORDER BY b; +a b +1 1 +200 2 +201 4 +0 6 +300 7 +SET SQL_MODE = 0; +# Scenario 10: Rollback would not rollback the counter +CREATE TABLE t32 ( +a BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; +INSERT INTO t32 VALUES(0), (0); +set global innodb_flush_log_at_trx_commit=1; +START TRANSACTION; +INSERT INTO t32 VALUES(0), (0); +SELECT MAX(a) AS `Expect 4` FROM t32; +Expect 4 +4 +DELETE FROM t32 WHERE a >= 2; +ROLLBACK; +# Kill and restart +SELECT MAX(a) AS `Expect 2` FROM t32; +Expect 2 +2 +INSERT INTO t32 VALUES(0), (0); +SELECT MAX(a) AS `Expect 6` FROM t32; +Expect 6 +6 +# Scenario 11: Test duplicate primary key/secondary key will not stop +# increasing the counter +CREATE TABLE t33 ( +a BIGINT NOT NULL PRIMARY KEY, +b BIGINT NOT NULL AUTO_INCREMENT, +KEY(b)) ENGINE = InnoDB; +INSERT INTO t33 VALUES(1, NULL); +INSERT INTO t33 VALUES(2, NULL); +INSERT INTO t33 VALUES(2, NULL); +ERROR 23000: Duplicate entry '2' for key 'PRIMARY' +INSERT INTO t33 VALUES(3, NULL); +SELECT MAX(b) AS `Expect 4` FROM t33; +Expect 4 +4 +TRUNCATE TABLE t33; +INSERT INTO t33 VALUES(1, NULL); +INSERT INTO t33 VALUES(2, NULL); +set global innodb_flush_log_at_trx_commit=1; +START TRANSACTION; +UPDATE t33 SET a = 10 WHERE a = 1; +INSERT INTO t33 VALUES(2, NULL); +ERROR 23000: Duplicate entry '2' for key 'PRIMARY' +COMMIT; +# Kill and restart +INSERT INTO t33 VALUES(3, NULL); +SELECT MAX(b) AS `Expect 4` FROM t33; +Expect 4 +4 +DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t30, t32, t33; diff --git a/mysql-test/suite/innodb/r/innodb-autoinc-44030.result b/mysql-test/suite/innodb/r/innodb-autoinc-44030.result index cf3ca93db27c4..5ec1bd38d808f 100644 --- a/mysql-test/suite/innodb/r/innodb-autoinc-44030.result +++ b/mysql-test/suite/innodb/r/innodb-autoinc-44030.result @@ -3,15 +3,24 @@ CREATE TABLE t1 (c1 INT PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; INSERT INTO t1 VALUES (null); INSERT INTO t1 VALUES (null); ALTER TABLE t1 CHANGE c1 d1 INT NOT NULL AUTO_INCREMENT; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 SELECT * FROM t1; d1 1 2 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `d1` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`d1`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1 +INSERT INTO t1 VALUES(null); SELECT * FROM t1; d1 1 2 -INSERT INTO t1 VALUES(null); +3 ALTER TABLE t1 AUTO_INCREMENT = 3; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 0 diff --git a/mysql-test/suite/innodb/r/innodb.result b/mysql-test/suite/innodb/r/innodb.result index 213fb8a783199..029753bc3ee0e 100644 --- a/mysql-test/suite/innodb/r/innodb.result +++ b/mysql-test/suite/innodb/r/innodb.result @@ -2429,11 +2429,17 @@ drop table t1; create table t1 (a int not null auto_increment primary key, val int) engine=InnoDB; insert into t1 (val) values (1); update t1 set a=2 where a=1; -insert into t1 (val) values (1); +insert into t1 (val) values (3); +select * from t1; +a val +2 1 +3 3 +insert into t1 values (2, 2); ERROR 23000: Duplicate entry '2' for key 'PRIMARY' select * from t1; a val 2 1 +3 3 drop table t1; CREATE TABLE t1 (GRADE DECIMAL(4) NOT NULL, PRIMARY KEY (GRADE)) ENGINE=INNODB; INSERT INTO t1 (GRADE) VALUES (151),(252),(343); diff --git a/mysql-test/suite/innodb/t/autoinc_persist.test b/mysql-test/suite/innodb/t/autoinc_persist.test new file mode 100644 index 0000000000000..e4ea2db6bd2c7 --- /dev/null +++ b/mysql-test/suite/innodb/t/autoinc_persist.test @@ -0,0 +1,533 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +--echo # +--echo # MDEV-6076 Persistent AUTO_INCREMENT for InnoDB +--echo # +--echo # WL#6204 InnoDB persistent max value for autoinc columns +--echo # +--echo # Most of this test case is copied from the test innodb.autoinc_persist +--echo # that was introduced in MySQL 8.0.0. The observable behaviour +--echo # of MDEV-6076 is equivalent to WL#6204, with the exception that +--echo # there is less buffering taking place and redo log checkpoints +--echo # are not being treated specially. +--echo # Due to less buffering, there is no debug instrumentation testing +--echo # for MDEV-6076. +--echo # + +--echo # Pre-create several tables + +SET SQL_MODE='STRICT_ALL_TABLES'; + +CREATE TABLE t1(a TINYINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t1 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31); +SELECT * FROM t1; + +CREATE TABLE t2(a TINYINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t2 VALUES(-5); +INSERT INTO t2 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31); +SELECT * FROM t2; + +CREATE TABLE t3(a SMALLINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t3 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (1024), (4096); +SELECT * FROM t3; + +CREATE TABLE t4(a SMALLINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t4 VALUES(-5); +INSERT INTO t4 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (1024), (4096); +SELECT * FROM t4; + +CREATE TABLE t5(a MEDIUMINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t5 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (1000000), (1000005); +SELECT * FROM t5; + +CREATE TABLE t6(a MEDIUMINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t6 VALUES(-5); +INSERT INTO t6 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (1000000), (1000005); +SELECT * FROM t6; + +CREATE TABLE t7(a INT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t7 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (100000000), (100000008); +SELECT * FROM t7; + +CREATE TABLE t8(a INT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t8 VALUES(-5); +INSERT INTO t8 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (100000000), (100000008); +SELECT * FROM t8; + +CREATE TABLE t9(a BIGINT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t9 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31), (100000000000), (100000000006); +SELECT * FROM t9; + +CREATE TABLE t10(a BIGINT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t10 VALUES(-5); +INSERT INTO t10 VALUES(0), (0), (0), (0), (8), (10), (0), +(20), (30), (31), (100000000000), (100000000006); +SELECT * FROM t10; + +CREATE TABLE t11(a FLOAT AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31); +SELECT * FROM t11; + +# Since autoinc counter is persisted by redo logs, we don't want to +# lose them on kill and restart, so to make the result after restart stable. +set global innodb_flush_log_at_trx_commit=1; + +CREATE TABLE t12(a DOUBLE AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), +(20), (30), (31); +SELECT * FROM t12; + +CREATE TABLE t13(a INT AUTO_INCREMENT PRIMARY KEY) ENGINE = InnoDB, +AUTO_INCREMENT = 1234; + +--echo # Scenario 1: Normal restart, to test if the counters are persisted +--source include/restart_mysqld.inc + +--echo # We expect these results should be equal to above SELECTs +SELECT * FROM t1; +SELECT * FROM t2; +SELECT * FROM t3; +SELECT * FROM t4; +SELECT * FROM t5; +SELECT * FROM t6; +SELECT * FROM t7; +SELECT * FROM t8; +SELECT * FROM t9; +SELECT * FROM t10; +SELECT * FROM t11; +SELECT * FROM t12; + +SELECT * FROM t13; +SHOW CREATE TABLE t13; +INSERT INTO t13 VALUES(0); +SELECT a AS `Expect 1234` FROM t13; + +--echo # Scenario 2: Delete some values, to test the counters should not be the +--echo # one which is the largest in current table + +set global innodb_flush_log_at_trx_commit=1; + +DELETE FROM t1 WHERE a > 30; +SELECT MAX(a) AS `Expect 30` FROM t1; +DELETE FROM t3 WHERE a > 2000; +SELECT MAX(a) AS `Expect 2000` FROM t3; +DELETE FROM t5 WHERE a > 1000000; +SELECT MAX(a) AS `Expect 1000000` FROM t5; +DELETE FROM t7 WHERE a > 100000000; +SELECT MAX(a) AS `Expect 100000000` FROM t7; +DELETE FROM t9 WHERE a > 100000000000; +SELECT MAX(a) AS `Expect 100000000000` FROM t9; + +--source include/restart_mysqld.inc + +INSERT INTO t1 VALUES(0), (0); +SELECT MAX(a) AS `Expect 33` FROM t1; +INSERT INTO t3 VALUES(0), (0); +SELECT MAX(a) AS `Expect 4098` FROM t3; +INSERT INTO t5 VALUES(0), (0); +SELECT MAX(a) AS `Expect 1000007` FROM t5; +INSERT INTO t7 VALUES(0), (0); +SELECT MAX(a) AS `Expect 100000010` FROM t7; +INSERT INTO t9 VALUES(0), (0); +SELECT MAX(a) AS `Expect 100000000008` FROM t9; + +--echo # Scenario 3: Insert some bigger counters, the next counter should start +--echo # from there + +INSERT INTO t1 VALUES(40), (0); +INSERT INTO t1 VALUES(42), (0); +SELECT a AS `Expect 43, 42` FROM t1 ORDER BY a DESC LIMIT 4; +INSERT INTO t3 VALUES(5000), (0); +INSERT INTO t3 VALUES(5010), (0); +SELECT a AS `Expect 5011, 5010` FROM t3 ORDER BY a DESC LIMIT 4; +INSERT INTO t5 VALUES(1000010), (0); +INSERT INTO t5 VALUES(1000020), (0); +SELECT a AS `Expect 1000021, 1000020` FROM t5 ORDER BY a DESC LIMIT 4; +INSERT INTO t7 VALUES(100000020), (0); +INSERT INTO t7 VALUES(100000030), (0); +SELECT a AS `Expect 100000031, 100000030` FROM t7 ORDER BY a DESC LIMIT 4; +INSERT INTO t9 VALUES(100000000010), (0); +INSERT INTO t9 VALUES(100000000020), (0); +SELECT a AS `Expect 100000000021, 100000000020` FROM t9 ORDER BY a DESC LIMIT 4; + +--echo # Scenario 4: Update some values, to test the counters should be updated +--echo # to the bigger value, but not smaller value. + +INSERT INTO t1 VALUES(50), (55); +# Updating to bigger value will update the auto-increment counter +UPDATE t1 SET a = 105 WHERE a = 5; +# Updating to smaller value will not update the counter +UPDATE t1 SET a = 100 WHERE a = 55; +--echo # This should insert 102, 106, 107, and make next counter 109. +INSERT INTO t1 VALUES(102), (0), (0); +SELECT a AS `Expect 107, 106` FROM t1 ORDER BY a DESC LIMIT 2; +DELETE FROM t1 WHERE a > 105; +INSERT INTO t1 VALUES(0); +SELECT MAX(a) AS `Expect 109` FROM t1; + +--echo # Test the same things on t3, t5, t7, t9, to test if DDTableBuffer would +--echo # be updated accordingly + +INSERT INTO t3 VALUES(60), (65); +# Updating to bigger value will update the auto-increment counter +UPDATE t3 SET a = 6005 WHERE a = 5; +# Updating to smaller value will not update the counter +UPDATE t3 SET a = 6000 WHERE a = 60; +--echo # This should insert 6002, 6006, 6007, and make next counter 6009. +INSERT INTO t3 VALUES(6002), (0), (0); +SELECT a AS `Expect 6007, 6006` FROM t3 ORDER BY a DESC LIMIT 2; +DELETE FROM t3 WHERE a > 6005; +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 6009` FROM t3; + +INSERT INTO t5 VALUES(100), (200); +# Updating to bigger value will update the auto-increment counter +UPDATE t5 SET a = 1000105 WHERE a = 5; +# Updating to smaller value will not update the counter +UPDATE t5 SET a = 1000100 WHERE a = 100; +--echo # This should insert 1000102, 1000106, 1000107, and make next counter +--echo # 1000109. +INSERT INTO t5 VALUES(1000102), (0), (0); +SELECT a AS `Expect 1000107, 1000106` FROM t5 ORDER BY a DESC LIMIT 2; +DELETE FROM t5 WHERE a > 1000105; +INSERT INTO t5 VALUES(0); +SELECT MAX(a) AS `Expect 1000109` FROM t5; + +INSERT INTO t7 VALUES(100), (200); +# Updating to bigger value will update the auto-increment counter +UPDATE t7 SET a = 100000105 WHERE a = 5; +# Updating to smaller value will not update the counter +UPDATE t7 SET a = 100000100 WHERE a = 100; +--echo # This should insert 100000102, 1100000106, 100000107, and make next +--echo # counter 100000109. +INSERT INTO t7 VALUES(100000102), (0), (0); +SELECT a AS `Expect 100000107, 100000106` FROM t7 ORDER BY a DESC LIMIT 2; +DELETE FROM t7 WHERE a > 100000105; +INSERT INTO t7 VALUES(0); +SELECT MAX(a) AS `Expect 100000109` FROM t7; + +set global innodb_flush_log_at_trx_commit=1; + +INSERT INTO t9 VALUES(100), (200); +# Updating to bigger value will update the auto-increment counter +UPDATE t9 SET a = 100000000105 WHERE a = 5; +# Updating to smaller value will not update the counter +UPDATE t9 SET a = 100000000100 WHERE a = 100; +--echo # This should insert 100000000102, 100000000106, 100000000107, and make +--echo # next counter 100000000109. +INSERT INTO t9 VALUES(100000000102), (0), (0); +SELECT a AS `Expect 100000000107, 100000000106` FROM t9 ORDER BY a DESC LIMIT 2; +DELETE FROM t9 WHERE a > 100000000105; +INSERT INTO t9 VALUES(0); +SELECT MAX(a) AS `Expect 100000000109` FROM t9; + +--source include/restart_mysqld.inc + +INSERT INTO t1 VALUES(0), (0); +SELECT a AS `Expect 110, 111` FROM t1 ORDER BY a DESC LIMIT 2; + +INSERT INTO t3 VALUES(0), (0); +SELECT a AS `Expect 6010, 6011` FROM t3 ORDER BY a DESC LIMIT 2; + +INSERT INTO t5 VALUES(0), (0); +SELECT a AS `Expect 1100111, 1100110` FROM t5 ORDER BY a DESC LIMIT 2; + +INSERT INTO t7 VALUES(0), (0); +SELECT a AS `Expect 100000111, 100000110` FROM t7 ORDER BY a DESC LIMIT 2; + +INSERT INTO t9 VALUES(0), (0); +SELECT a AS `Expect 100000000111, 100000000110` FROM t9 ORDER BY a DESC LIMIT 2; + +--echo # Scenario 5: Test kill the server + +INSERT INTO t1 VALUES(125); +DELETE FROM t1 WHERE a = 125; + +INSERT INTO t3 VALUES(6100); +DELETE FROM t3 WHERE a = 6100; + +INSERT INTO t5 VALUES(1100200); +DELETE FROM t5 WHERE a = 1100200; + +INSERT INTO t7 VALUES(100000200); +DELETE FROM t7 WHERE a = 100000200; + +set global innodb_flush_log_at_trx_commit=1; + +INSERT INTO t9 VALUES(100000000200); +DELETE FROM t9 WHERE a = 100000000200; + +--source include/kill_and_restart_mysqld.inc + +INSERT INTO t1 VALUES(0); +SELECT a AS `Expect 126` FROM t1 ORDER BY a DESC LIMIT 1; + +INSERT INTO t3 VALUES(0); +SELECT a AS `Expect 6101` FROM t3 ORDER BY a DESC LIMIT 1; + +INSERT INTO t5 VALUES(0); +SELECT a AS `Expect 1100201` FROM t5 ORDER BY a DESC LIMIT 1; + +INSERT INTO t7 VALUES(0); +SELECT a AS `Expect 100000201` FROM t7 ORDER BY a DESC LIMIT 1; + +INSERT INTO t9 VALUES(0); +SELECT a AS `Expect 100000000201` FROM t9 ORDER BY a DESC LIMIT 1; + +--echo # Scenario 6: Test truncate will reset the counters to 0 + +TRUNCATE TABLE t1; +TRUNCATE TABLE t3; +TRUNCATE TABLE t5; +TRUNCATE TABLE t7; +TRUNCATE TABLE t9; + +INSERT INTO t1 VALUES(0), (0); +SELECT * FROM t1; + +INSERT INTO t3 VALUES(0), (0); +SELECT * FROM t3; + +INSERT INTO t5 VALUES(0), (0); +SELECT * FROM t5; + +INSERT INTO t7 VALUES(0), (0); +SELECT * FROM t7; + +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; + +set global innodb_flush_log_at_trx_commit=1; + +TRUNCATE TABLE t1; +TRUNCATE TABLE t3; +TRUNCATE TABLE t5; +TRUNCATE TABLE t7; +TRUNCATE TABLE t9; + +--source include/kill_and_restart_mysqld.inc + +INSERT INTO t1 VALUES(0), (0); +SELECT * FROM t1; + +INSERT INTO t3 VALUES(0), (0); +SELECT * FROM t3; + +INSERT INTO t5 VALUES(0), (0); +SELECT * FROM t5; + +INSERT INTO t7 VALUES(0), (0); +SELECT * FROM t7; + +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; + +--echo # Scenario 7: Test explicit rename table won't change the counter + +set global innodb_flush_log_at_trx_commit=1; + +RENAME TABLE t9 to t19; +INSERT INTO t19 VALUES(0), (0); +SELECT * FROM t19; +DELETE FROM t19 WHERE a = 4; + +--source include/kill_and_restart_mysqld.inc + +RENAME TABLE t19 to t9; +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; + +TRUNCATE TABLE t9; + +INSERT INTO t9 VALUES(0), (0); +SELECT * FROM t9; + +--echo # Scenario 8: Test ALTER TABLE operations + +INSERT INTO t3 VALUES(0), (0), (100), (200), (1000); +SELECT * FROM t3; +DELETE FROM t3 WHERE a > 300; +SELECT MAX(a) AS `Expect 200` FROM t3; +--echo # This will not change the counter to 150, but to 201, which is the next +--echo # of current max counter in the table +ALTER TABLE t3 AUTO_INCREMENT = 150; +SHOW CREATE TABLE t3; +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 201` FROM t3; +--echo # This will change the counter to 500, which is bigger than any counter +--echo # in the table +ALTER TABLE t3 AUTO_INCREMENT = 500; +SHOW CREATE TABLE t3; +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 500` FROM t3; + +TRUNCATE TABLE t3; +ALTER TABLE t3 AUTO_INCREMENT = 100; +SHOW CREATE TABLE t3; +INSERT INTO t3 VALUES(0), (0); +SELECT * FROM t3; + +INSERT INTO t3 VALUES(150), (180); +UPDATE t3 SET a = 200 WHERE a = 150; +INSERT INTO t3 VALUES(220); +--echo # This still fails to set to 120, but just 221 +ALTER TABLE t3 AUTO_INCREMENT = 120; +SHOW CREATE TABLE t3; +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 221` FROM t3; + +DELETE FROM t3 WHERE a > 120; + +ALTER TABLE t3 AUTO_INCREMENT = 120; +SHOW CREATE TABLE t3; + +--echo # MDEV-6076: Test adding an AUTO_INCREMENT COLUMN +CREATE TABLE mdev6076a (b INT) ENGINE=InnoDB; +INSERT INTO mdev6076a VALUES(2),(1); +CREATE TABLE mdev6076b (b INT) ENGINE=InnoDB; +INSERT INTO mdev6076b VALUES(2),(1); +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, LOCK=NONE; +ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, ALGORITHM=INPLACE; +ALTER TABLE mdev6076b ADD COLUMN a SERIAL FIRST, AUTO_INCREMENT=100, +ALGORITHM=INPLACE; + +--source include/kill_and_restart_mysqld.inc + +INSERT INTO t3 VALUES(0); +SELECT MAX(a) AS `Expect 120` FROM t3; + +INSERT INTO mdev6076a SET b=0; +SELECT * FROM mdev6076a; +INSERT INTO mdev6076b SET b=0; +SELECT * FROM mdev6076b; +DROP TABLE mdev6076a, mdev6076b; + +set global innodb_flush_log_at_trx_commit=1; + +INSERT INTO t3 VALUES(0), (0), (200), (210); + +--echo # Test the different algorithms in ALTER TABLE + +--let $template = t3 +--let $algorithm = INPLACE +--let $table = t_inplace +--source suite/innodb/include/autoinc_persist_alter.inc +--let $algorithm = COPY +--let $table = t_copy +--source suite/innodb/include/autoinc_persist_alter.inc + +--echo # Scenario 9: Test the sql_mode = NO_AUTO_VALUE_ON_ZERO + +CREATE TABLE t30 (a INT NOT NULL AUTO_INCREMENT PRIMARY KEY, b INT, key(b)) ENGINE = InnoDB; + +set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; + +INSERT INTO t30 VALUES(NULL, 1), (200, 2), (0, 3); +INSERT INTO t30(b) VALUES(4), (5), (6), (7); +SELECT * FROM t30 ORDER BY b; +ALTER TABLE t30 MODIFY b MEDIUMINT; +SELECT * FROM t30 ORDER BY b; + +set global innodb_flush_log_at_trx_commit=1; + +CREATE TABLE t31 (a INT) ENGINE = InnoDB; +INSERT INTO t31 VALUES(1), (2); +ALTER TABLE t31 ADD b INT AUTO_INCREMENT PRIMARY KEY; +INSERT INTO t31 VALUES(3, 0), (4, NULL), (5, NULL); +--error ER_DUP_ENTRY +INSERT INTO t31 VALUES(6, 0); +SELECT * FROM t31; + +--source include/kill_and_restart_mysqld.inc + +--echo # This will not insert 0 +INSERT INTO t31(a) VALUES(6), (0); +SELECT * FROM t31; +DROP TABLE t31; + +set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; + +DELETE FROM t30 WHERE a = 0; +UPDATE t30 set a = 0 where b = 5; +SELECT * FROM t30 ORDER BY b; +DELETE FROM t30 WHERE a = 0; + +UPDATE t30 SET a = NULL WHERE b = 6; +UPDATE t30 SET a = 300 WHERE b = 7; + +SELECT * FROM t30 ORDER BY b; + +SET SQL_MODE = 0; + +--echo # Scenario 10: Rollback would not rollback the counter +CREATE TABLE t32 ( +a BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; + +INSERT INTO t32 VALUES(0), (0); + +set global innodb_flush_log_at_trx_commit=1; + +START TRANSACTION; +INSERT INTO t32 VALUES(0), (0); +SELECT MAX(a) AS `Expect 4` FROM t32; +DELETE FROM t32 WHERE a >= 2; +ROLLBACK; + +--source include/kill_and_restart_mysqld.inc + +SELECT MAX(a) AS `Expect 2` FROM t32; +INSERT INTO t32 VALUES(0), (0); +SELECT MAX(a) AS `Expect 6` FROM t32; + +--echo # Scenario 11: Test duplicate primary key/secondary key will not stop +--echo # increasing the counter + +CREATE TABLE t33 ( +a BIGINT NOT NULL PRIMARY KEY, +b BIGINT NOT NULL AUTO_INCREMENT, +KEY(b)) ENGINE = InnoDB; + +INSERT INTO t33 VALUES(1, NULL); +INSERT INTO t33 VALUES(2, NULL); +--error ER_DUP_ENTRY +INSERT INTO t33 VALUES(2, NULL); + +INSERT INTO t33 VALUES(3, NULL); +SELECT MAX(b) AS `Expect 4` FROM t33; + +TRUNCATE TABLE t33; + +INSERT INTO t33 VALUES(1, NULL); +INSERT INTO t33 VALUES(2, NULL); + +set global innodb_flush_log_at_trx_commit=1; + +START TRANSACTION; +UPDATE t33 SET a = 10 WHERE a = 1; +--error ER_DUP_ENTRY +INSERT INTO t33 VALUES(2, NULL); +COMMIT; + +--source include/kill_and_restart_mysqld.inc + +INSERT INTO t33 VALUES(3, NULL); +SELECT MAX(b) AS `Expect 4` FROM t33; + +DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t30, t32, t33; diff --git a/mysql-test/suite/innodb/t/innodb-autoinc-44030-master.opt b/mysql-test/suite/innodb/t/innodb-autoinc-44030-master.opt deleted file mode 100644 index 303ec1be1d08d..0000000000000 --- a/mysql-test/suite/innodb/t/innodb-autoinc-44030-master.opt +++ /dev/null @@ -1,3 +0,0 @@ ---default-storage-engine=MyISAM ---innodb-strict-mode=0 ---innodb-file-per-table=0 diff --git a/mysql-test/suite/innodb/t/innodb-autoinc-44030.test b/mysql-test/suite/innodb/t/innodb-autoinc-44030.test index 256e7d838ea07..61c561273516a 100644 --- a/mysql-test/suite/innodb/t/innodb-autoinc-44030.test +++ b/mysql-test/suite/innodb/t/innodb-autoinc-44030.test @@ -2,27 +2,24 @@ # embedded server does not support restarting -- source include/not_embedded.inc -# -# 44030: Error: (1500) Couldn't read the MAX(ID) autoinc value from +# Before MDEV-6076 Persistent AUTO_INCREMENT for InnoDB +# this was a test for +# Bug #44030: Error: (1500) Couldn't read the MAX(ID) autoinc value from # the index (PRIMARY) # This test requires a restart of the server SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; CREATE TABLE t1 (c1 INT PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; INSERT INTO t1 VALUES (null); INSERT INTO t1 VALUES (null); +--enable_info ALTER TABLE t1 CHANGE c1 d1 INT NOT NULL AUTO_INCREMENT; +--disable_info SELECT * FROM t1; -# Restart the server -- source include/restart_mysqld.inc -# The MySQL and InnoDB data dictionaries should now be out of sync. -# The select should print message to the error log -SELECT * FROM t1; -# MySQL have made a change (http://lists.mysql.com/commits/75268) that no -# longer results in the two data dictionaries being out of sync. If they -# revert their changes then this check for ER_AUTOINC_READ_FAILED will need -# to be enabled. Also, see http://bugs.mysql.com/bug.php?id=47621. -#-- error ER_AUTOINC_READ_FAILED,1467 +SHOW CREATE TABLE t1; INSERT INTO t1 VALUES(null); +SELECT * FROM t1; + # Before WL#5534, the following statement would copy the table, # and effectively set AUTO_INCREMENT to 4, because while copying # it would write values 1,2,3 to the column. diff --git a/mysql-test/suite/innodb/t/innodb.test b/mysql-test/suite/innodb/t/innodb.test index dfe1a0b4995e4..e5e4b45a8611a 100644 --- a/mysql-test/suite/innodb/t/innodb.test +++ b/mysql-test/suite/innodb/t/innodb.test @@ -1463,15 +1463,18 @@ select * from t1; drop table t1; # -# Test that update does not change internal auto-increment value +# Test that update does change internal auto-increment value # create table t1 (a int not null auto_increment primary key, val int) engine=InnoDB; insert into t1 (val) values (1); update t1 set a=2 where a=1; -# We should get the following error because InnoDB does not update the counter +# This should insert 3, since the counter has been updated to 2 already +insert into t1 (val) values (3); +select * from t1; +# We should get the following error because InnoDB does update the counter --error ER_DUP_ENTRY -insert into t1 (val) values (1); +insert into t1 values (2, 2); select * from t1; drop table t1; # diff --git a/mysql-test/suite/parts/r/partition_auto_increment_innodb.result b/mysql-test/suite/parts/r/partition_auto_increment_innodb.result index 635942a1f34b2..a6af2b924c065 100644 --- a/mysql-test/suite/parts/r/partition_auto_increment_innodb.result +++ b/mysql-test/suite/parts/r/partition_auto_increment_innodb.result @@ -43,7 +43,7 @@ UPDATE t1 SET c1 = 40 WHERE c1 = 50; SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; AUTO_INCREMENT -32 +52 UPDATE t1 SET c1 = NULL WHERE c1 = 4; Warnings: Warning 1048 Column 'c1' cannot be null @@ -62,10 +62,10 @@ c1 25 30 31 -32 -33 40 51 +52 +53 DROP TABLE t1; CREATE TABLE t1 ( c1 INT NOT NULL AUTO_INCREMENT, @@ -255,7 +255,7 @@ UPDATE t1 SET c1 = 140 WHERE c1 = 150; SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; AUTO_INCREMENT -141 +152 UPDATE t1 SET c1 = NULL WHERE c1 = 4; Warnings: Warning 1048 Column 'c1' cannot be null @@ -279,9 +279,9 @@ c1 90 91 140 -141 -142 151 +152 +153 DROP TABLE t1; # Test with auto_increment_increment and auto_increment_offset. CREATE TABLE t1 ( diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index e5284ee802aa9..9b99da2a684c0 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -50,6 +50,7 @@ Created 6/2/1994 Heikki Tuuri #include "gis0geo.h" #include "ut0new.h" #include "dict0boot.h" +#include "row0sel.h" /* row_search_max_autoinc() */ /**************************************************************//** Checks if the page in the cursor can be merged with given page. @@ -1405,6 +1406,117 @@ btr_free( btr_free_root(block, &mtr); mtr.commit(); } + +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@return the last used AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc(dict_index_t* index) +{ + ut_ad(dict_index_is_clust(index)); + ut_ad(index->table->persistent_autoinc); + ut_ad(!dict_table_is_temporary(index->table)); + + if (fil_space_t* space = fil_space_acquire(index->space)) { + mtr_t mtr; + mtr.start(); + ib_uint64_t autoinc; + if (buf_block_t* block = buf_page_get( + page_id_t(index->space, index->page), + page_size_t(space->flags), + RW_S_LATCH, &mtr)) { + autoinc = page_get_autoinc(block->frame); + } else { + autoinc = 0; + } + mtr.commit(); + fil_space_release(space); + return(autoinc); + } + + return(0); +} + +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, +or fall back to MAX(auto_increment_column). +@param[in] table table containing an AUTO_INCREMENT column +@param[in] col_no index of the AUTO_INCREMENT column +@return the AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc_with_fallback(const dict_table_t* table, unsigned col_no) +{ + ut_ad(table->persistent_autoinc); + ut_ad(!dict_table_is_temporary(table)); + + dict_index_t* index = dict_table_get_first_index(table); + + if (index == NULL) { + } else if (fil_space_t* space = fil_space_acquire(index->space)) { + mtr_t mtr; + mtr.start(); + buf_block_t* block = buf_page_get( + page_id_t(index->space, index->page), + page_size_t(space->flags), + RW_S_LATCH, &mtr); + + ib_uint64_t autoinc = block + ? page_get_autoinc(block->frame) : 0; + const bool retry = block && autoinc == 0 + && !page_is_empty(block->frame); + mtr.commit(); + fil_space_release(space); + + if (retry) { + /* This should be an old data file where + PAGE_ROOT_AUTO_INC was initialized to 0. + Fall back to reading MAX(autoinc_col). + There should be an index on it. */ + const dict_col_t* autoinc_col + = dict_table_get_nth_col(table, col_no); + while (index != NULL + && index->fields[0].col != autoinc_col) { + index = dict_table_get_next_index(index); + } + + if (index != NULL && index->space == space->id) { + autoinc = row_search_max_autoinc(index); + } + } + + return(autoinc); + } + + return(0); +} + +/** Write the next available AUTO_INCREMENT value to PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@param[in] autoinc the AUTO_INCREMENT value +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset) +{ + ut_ad(dict_index_is_clust(index)); + ut_ad(index->table->persistent_autoinc); + ut_ad(!dict_table_is_temporary(index->table)); + + if (fil_space_t* space = fil_space_acquire(index->space)) { + mtr_t mtr; + mtr.start(); + mtr.set_named_space(space); + page_set_autoinc(buf_page_get( + page_id_t(index->space, index->page), + page_size_t(space->flags), + RW_SX_LATCH, &mtr), + index, autoinc, &mtr, reset); + mtr.commit(); + fil_space_release(space); + } +} #endif /* !UNIV_HOTBACKUP */ /*************************************************************//** @@ -1499,21 +1611,28 @@ btr_page_reorganize_low( page_get_infimum_rec(temp_page), index, mtr); - /* Multiple transactions cannot simultaneously operate on the - same temp-table in parallel. - max_trx_id is ignored for temp tables because it not required - for MVCC. */ - if (dict_index_is_sec_or_ibuf(index) - && page_is_leaf(page) - && !dict_table_is_temporary(index->table)) { - /* Copy max trx id to recreated page */ - trx_id_t max_trx_id = page_get_max_trx_id(temp_page); - page_set_max_trx_id(block, NULL, max_trx_id, mtr); - /* In crash recovery, dict_index_is_sec_or_ibuf() always - holds, even for clustered indexes. max_trx_id is - unused in clustered index pages. */ - ut_ad(max_trx_id != 0 || recovery); - } + /* Copy the PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC. */ + memcpy(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), + temp_page + (PAGE_HEADER + PAGE_MAX_TRX_ID), 8); + /* PAGE_MAX_TRX_ID is unused in clustered index pages, + non-leaf pages, and in temporary tables. It was always + zero-initialized in page_create() in all InnoDB versions. + PAGE_MAX_TRX_ID must be nonzero on dict_index_is_sec_or_ibuf() + leaf pages. + + During redo log apply, dict_index_is_sec_or_ibuf() always + holds, even for clustered indexes. */ + ut_ad(recovery || dict_table_is_temporary(index->table) + || !page_is_leaf(temp_page) + || !dict_index_is_sec_or_ibuf(index) + || page_get_max_trx_id(page) != 0); + /* PAGE_MAX_TRX_ID must be zero on non-leaf pages other than + clustered index root pages. */ + ut_ad(recovery + || page_get_max_trx_id(page) == 0 + || (dict_index_is_sec_or_ibuf(index) + ? page_is_leaf(temp_page) + : page_is_root(temp_page))); /* If innodb_log_compressed_pages is ON, page reorganize should log the compressed page image.*/ diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 0d72b9d2b9275..d8a2e15d1eb24 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -751,7 +751,9 @@ btr_cur_search_to_nth_level( RW_S_LATCH, or 0 */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ - mtr_t* mtr) /*!< in: mtr */ + mtr_t* mtr, /*!< in: mtr */ + ib_uint64_t autoinc)/*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ { page_t* page = NULL; /* remove warning */ buf_block_t* block; @@ -888,6 +890,12 @@ btr_cur_search_to_nth_level( || latch_mode == BTR_SEARCH_TREE || latch_mode == BTR_MODIFY_LEAF); + ut_ad(autoinc == 0 || dict_index_is_clust(index)); + ut_ad(autoinc == 0 + || latch_mode == BTR_MODIFY_TREE + || latch_mode == BTR_MODIFY_LEAF); + ut_ad(autoinc == 0 || level == 0); + cursor->flag = BTR_CUR_BINARY; cursor->index = index; @@ -907,8 +915,7 @@ btr_cur_search_to_nth_level( # ifdef UNIV_SEARCH_PERF_STAT info->n_searches++; # endif - if (rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_NOT_LOCKED + if (autoinc == 0 && latch_mode <= BTR_MODIFY_LEAF && info->last_hash_succ # ifdef MYSQL_INDEX_DISABLE_AHI @@ -922,8 +929,10 @@ btr_cur_search_to_nth_level( /* If !has_search_latch, we do a dirty read of btr_search_enabled below, and btr_search_guess_on_hash() will have to check it again. */ - && UNIV_LIKELY(btr_search_enabled) + && btr_search_enabled && !modify_external + && rw_lock_get_writer(btr_get_search_latch(index)) + == RW_LOCK_NOT_LOCKED && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, has_search_latch, mtr)) { @@ -1071,16 +1080,16 @@ btr_cur_search_to_nth_level( if (height != 0) { /* We are about to fetch the root or a non-leaf page. */ - if ((latch_mode != BTR_MODIFY_TREE - || height == level) + if ((latch_mode != BTR_MODIFY_TREE || height == level) && !retrying_for_search_prev) { /* If doesn't have SX or X latch of index, each pages should be latched before reading. */ - if (modify_external - && height == ULINT_UNDEFINED - && upper_rw_latch == RW_S_LATCH) { + if (height == ULINT_UNDEFINED + && upper_rw_latch == RW_S_LATCH + && (modify_external || autoinc)) { /* needs sx-latch of root page - for fseg operation */ + for fseg operation or for writing + PAGE_ROOT_AUTO_INC */ rw_latch = RW_SX_LATCH; } else { rw_latch = upper_rw_latch; @@ -1271,11 +1280,13 @@ btr_cur_search_to_nth_level( && page_is_leaf(page) && rw_latch != RW_NO_LATCH && rw_latch != root_leaf_rw_latch) { - /* We should retry to get the page, because the root page - is latched with different level as a leaf page. */ + /* The root page is also a leaf page (root_leaf). + We should reacquire the page, because the root page + is latched differently from leaf pages. */ ut_ad(root_leaf_rw_latch != RW_NO_LATCH); ut_ad(rw_latch == RW_S_LATCH || rw_latch == RW_SX_LATCH); - ut_ad(rw_latch == RW_S_LATCH || modify_external); + ut_ad(rw_latch == RW_S_LATCH || modify_external || autoinc); + ut_ad(!autoinc || root_leaf_rw_latch == RW_X_LATCH); ut_ad(n_blocks == 0); mtr_release_block_at_savepoint( @@ -1365,6 +1376,7 @@ btr_cur_search_to_nth_level( /* release upper blocks */ if (retrying_for_search_prev) { + ut_ad(!autoinc); for (; prev_n_releases < prev_n_blocks; prev_n_releases++) { @@ -1378,8 +1390,9 @@ btr_cur_search_to_nth_level( } for (; n_releases < n_blocks; n_releases++) { - if (n_releases == 0 && modify_external) { - /* keep latch of root page */ + if (n_releases == 0 + && (modify_external || autoinc)) { + /* keep the root page latch */ ut_ad(mtr_memo_contains_flagged( mtr, tree_blocks[n_releases], MTR_MEMO_PAGE_SX_FIX @@ -1903,6 +1916,8 @@ btr_cur_search_to_nth_level( } if (level != 0) { + ut_ad(!autoinc); + if (upper_rw_latch == RW_NO_LATCH) { /* latch the page */ buf_block_t* child_block; @@ -1951,6 +1966,11 @@ btr_cur_search_to_nth_level( cursor->up_match = up_match; cursor->up_bytes = up_bytes; + if (autoinc) { + page_set_autoinc(tree_blocks[0], + index, autoinc, mtr, false); + } + #ifdef BTR_CUR_ADAPT /* We do a dirty read of btr_search_enabled here. We will properly check btr_search_enabled again in diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 73399f14081dd..377a332bd9f02 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -596,7 +597,7 @@ btr_pcur_open_on_user_rec_func( mtr_t* mtr) /*!< in: mtr */ { btr_pcur_open_low(index, 0, tuple, mode, latch_mode, cursor, - file, line, mtr); + file, line, 0, mtr); if ((mode == PAGE_CUR_GE) || (mode == PAGE_CUR_G)) { diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 4f77d2168f4f4..cabe649f0d4ee 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -810,20 +810,6 @@ dict_index_zip_pad_lock( mutex_enter(index->zip_pad.mutex); } - -/********************************************************************//** -Unconditionally set the autoinc counter. */ -void -dict_table_autoinc_initialize( -/*==========================*/ - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: next value to assign to a row */ -{ - ut_ad(dict_table_autoinc_own(table)); - - table->autoinc = value; -} - /** Get all the FTS indexes on a table. @param[in] table table @param[out] indexes all FTS indexes on this table @@ -849,75 +835,6 @@ dict_table_get_all_fts_indexes( return(ib_vector_size(indexes)); } -/** Store autoinc value when the table is evicted. -@param[in] table table evicted */ -void -dict_table_autoinc_store( - const dict_table_t* table) -{ - ut_ad(mutex_own(&dict_sys->mutex)); - - if (table->autoinc != 0) { - ut_ad(dict_sys->autoinc_map->find(table->id) - == dict_sys->autoinc_map->end()); - - dict_sys->autoinc_map->insert( - std::pair( - table->id, table->autoinc)); - } -} - -/** Restore autoinc value when the table is loaded. -@param[in] table table loaded */ -void -dict_table_autoinc_restore( - dict_table_t* table) -{ - ut_ad(mutex_own(&dict_sys->mutex)); - - autoinc_map_t::iterator it; - it = dict_sys->autoinc_map->find(table->id); - - if (it != dict_sys->autoinc_map->end()) { - table->autoinc = it->second; - ut_ad(table->autoinc != 0); - - dict_sys->autoinc_map->erase(it); - } -} - -/********************************************************************//** -Reads the next autoinc value (== autoinc counter value), 0 if not yet -initialized. -@return value for a new row, or 0 */ -ib_uint64_t -dict_table_autoinc_read( -/*====================*/ - const dict_table_t* table) /*!< in: table */ -{ - ut_ad(dict_table_autoinc_own(table)); - - return(table->autoinc); -} - -/********************************************************************//** -Updates the autoinc counter if the value supplied is greater than the -current value. */ -void -dict_table_autoinc_update_if_greater( -/*=================================*/ - - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: value which was assigned to a row */ -{ - ut_ad(dict_table_autoinc_own(table)); - - if (value > table->autoinc) { - - table->autoinc = value; - } -} - /********************************************************************//** Release the autoinc lock. */ void @@ -1212,8 +1129,6 @@ dict_init(void) } mutex_create(LATCH_ID_DICT_FOREIGN_ERR, &dict_foreign_err_mutex); - - dict_sys->autoinc_map = new autoinc_map_t(); } /**********************************************************************//** @@ -1477,8 +1392,6 @@ dict_table_add_to_cache( UT_LIST_ADD_FIRST(dict_sys->table_non_LRU, table); } - dict_table_autoinc_restore(table); - ut_ad(dict_lru_validate()); dict_sys->size += mem_heap_get_size(table->heap) @@ -2213,10 +2126,6 @@ dict_table_remove_from_cache_low( ut_ad(dict_lru_validate()); - if (lru_evict) { - dict_table_autoinc_store(table); - } - if (lru_evict && table->drop_aborted) { /* Do as dict_table_try_drop_aborted() does. */ @@ -7007,8 +6916,6 @@ dict_close(void) mutex_free(&dict_foreign_err_mutex); - delete dict_sys->autoinc_map; - ut_ad(dict_sys->size == 0); ut_free(dict_sys); diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index e73c61963866e..b150c1eb7a035 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -159,12 +159,6 @@ dict_mem_table_create( /* lazy creation of table autoinc latch */ dict_table_autoinc_create_lazy(table); - table->autoinc = 0; - - /* The number of transactions that are either waiting on the - AUTOINC lock or have been granted the lock. */ - table->n_waiting_or_granted_auto_inc_locks = 0; - /* If the table has an FTS index or we are in the process of building one, create the table->fts */ if (dict_table_has_fts_index(table) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 83dd0c17eaa9f..53d9672808bc2 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6739,33 +6739,35 @@ innobase_get_int_col_max_value( return(max_value); } -/************************************************************************ -Set the autoinc column max value. This should only be called once from -ha_innobase::open(). Therefore there's no need for a covering lock. */ +/** Initialize the AUTO_INCREMENT column metadata. + +Since a partial table definition for a persistent table can already be +present in the InnoDB dict_sys cache before it is accessed from SQL, +we have to initialize the AUTO_INCREMENT counter on the first +ha_innobase::open(). + +@param[in,out] table persistent table +@param[in] field the AUTO_INCREMENT column */ +static void -ha_innobase::innobase_initialize_autoinc() -/*======================================*/ +initialize_auto_increment(dict_table_t* table, const Field* field) { - ulonglong auto_inc; - const Field* field = table->found_next_number_field; + ut_ad(!dict_table_is_temporary(table)); - if (field != NULL) { - auto_inc = innobase_get_int_col_max_value(field); - ut_ad(!innobase_is_v_fld(field)); + const unsigned col_no = innodb_col_no(field); - /* autoinc column cannot be virtual column */ - ut_ad(!innobase_is_v_fld(field)); - } else { - /* We have no idea what's been passed in to us as the - autoinc column. We set it to the 0, effectively disabling - updates to the table. */ - auto_inc = 0; + dict_table_autoinc_lock(table); - ib::info() << "Unable to determine the AUTOINC column name"; - } + table->persistent_autoinc = 1 + + dict_table_get_nth_col_pos(table, col_no, NULL); - if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { + if (table->autoinc) { + /* Already initialized. Our caller checked + table->persistent_autoinc without + dict_table_autoinc_lock(), and there might be multiple + ha_innobase::open() executing concurrently. */ + } else if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { /* If the recovery level is set so high that writes are disabled we force the AUTOINC counter to 0 value effectively disabling writes to the table. @@ -6776,70 +6778,16 @@ ha_innobase::innobase_initialize_autoinc() tables can be dumped with minimal hassle. If an error were returned in this case, the first attempt to read the table would fail and subsequent SELECTs would succeed. */ - auto_inc = 0; - } else if (field == NULL) { - /* This is a far more serious error, best to avoid - opening the table and return failure. */ - my_error(ER_AUTOINC_READ_FAILED, MYF(0)); - } else { - dict_index_t* index; - const char* col_name; - ib_uint64_t read_auto_inc; - ulint err; - - update_thd(ha_thd()); - - col_name = field->field_name; - - index = innobase_get_index(table->s->next_number_index); - - /* Execute SELECT MAX(col_name) FROM TABLE; */ - err = row_search_max_autoinc(index, col_name, &read_auto_inc); - - switch (err) { - case DB_SUCCESS: { - ulonglong col_max_value; - - col_max_value = innobase_get_int_col_max_value(field); - - /* At the this stage we do not know the increment - nor the offset, so use a default increment of 1. */ - - auto_inc = innobase_next_autoinc( - read_auto_inc, 1, 1, 0, col_max_value); - - break; - } - case DB_RECORD_NOT_FOUND: - ib::error() << "MariaDB and InnoDB data dictionaries are" - " out of sync. Unable to find the AUTOINC" - " column " << col_name << " in the InnoDB" - " table " << index->table->name << ". We set" - " the next AUTOINC column value to 0, in" - " effect disabling the AUTOINC next value" - " generation."; - - ib::info() << "You can either set the next AUTOINC" - " value explicitly using ALTER TABLE or fix" - " the data dictionary by recreating the" - " table."; - - /* This will disable the AUTOINC generation. */ - auto_inc = 0; - - /* We want the open to succeed, so that the user can - take corrective action. ie. reads should succeed but - updates should fail. */ - err = DB_SUCCESS; - break; - default: - /* row_search_max_autoinc() should only return - one of DB_SUCCESS or DB_RECORD_NOT_FOUND. */ - ut_error; - } + } else if (table->persistent_autoinc) { + table->autoinc = innobase_next_autoinc( + btr_read_autoinc_with_fallback(table, col_no), + 1 /* need */, + 1 /* auto_increment_increment */, + 0 /* auto_increment_offset */, + innobase_get_int_col_max_value(field)); } - dict_table_autoinc_initialize(m_prebuilt->table, auto_inc); + dict_table_autoinc_unlock(table); } /*****************************************************************//** @@ -7199,22 +7147,12 @@ ha_innobase::open( dict_table_get_format(m_prebuilt->table)); } - /* Only if the table has an AUTOINC column. */ - if (m_prebuilt->table != NULL - && !m_prebuilt->table->ibd_file_missing - && table->found_next_number_field != NULL) { - dict_table_autoinc_lock(m_prebuilt->table); - - /* Since a table can already be "open" in InnoDB's internal - data dictionary, we only init the autoinc counter once, the - first time the table is loaded. We can safely reuse the - autoinc value from a previous MySQL open. */ - if (dict_table_autoinc_read(m_prebuilt->table) == 0) { - - innobase_initialize_autoinc(); - } - - dict_table_autoinc_unlock(m_prebuilt->table); + if (m_prebuilt->table == NULL + || dict_table_is_temporary(m_prebuilt->table) + || m_prebuilt->table->persistent_autoinc + || m_prebuilt->table->ibd_file_missing) { + } else if (const Field* ai = table->found_next_number_field) { + initialize_auto_increment(m_prebuilt->table, ai); } /* Set plugin parser for fulltext index */ @@ -7270,6 +7208,24 @@ ha_innobase::open( DBUG_RETURN(0); } +/** Convert MySQL column number to dict_table_t::cols[] offset. +@param[in] field non-virtual column +@return column number relative to dict_table_t::cols[] */ +unsigned +innodb_col_no(const Field* field) +{ + ut_ad(!innobase_is_s_fld(field)); + const TABLE* table = field->table; + unsigned col_no = 0; + ut_ad(field == table->field[field->field_index]); + for (unsigned i = 0; i < field->field_index; i++) { + if (table->field[i]->stored_in_db()) { + col_no++; + } + } + return(col_no); +} + /** Opens dictionary table object using table name. For partition, we need to try alternative lower/upper case names to support moving data files across platforms. @@ -9250,28 +9206,32 @@ innodb_fill_old_vcol_val( return(buf); } -/**********************************************************************//** -Checks which fields have changed in a row and stores information -of them to an update vector. +/** Calculate an update vector corresponding to the changes +between old_row and new_row. +@param[out] uvect update vector +@param[in] old_row current row in MySQL format +@param[in] new_row intended updated row in MySQL format +@param[in] table MySQL table handle +@param[in,out] upd_buff buffer to use for converted values +@param[in] buff_len length of upd_buff +@param[in,out] prebuilt InnoDB execution context +@param[out] auto_inc updated AUTO_INCREMENT value, or 0 if none @return DB_SUCCESS or error code */ static dberr_t calc_row_difference( -/*================*/ - upd_t* uvect, /*!< in/out: update vector */ - const uchar* old_row, /*!< in: old row in MySQL format */ - uchar* new_row, /*!< in: new row in MySQL format */ - TABLE* table, /*!< in: table in MySQL data - dictionary */ - uchar* upd_buff, /*!< in: buffer to use */ - ulint buff_len, /*!< in: buffer length */ - row_prebuilt_t* prebuilt, /*!< in: InnoDB prebuilt struct */ - THD* thd) /*!< in: user thread */ + upd_t* uvect, + const uchar* old_row, + uchar* new_row, + TABLE* table, + uchar* upd_buff, + ulint buff_len, + row_prebuilt_t* prebuilt, + ib_uint64_t& auto_inc) { uchar* original_upd_buff = upd_buff; Field* field; enum_field_types field_mysql_type; - uint n_fields; ulint o_len; ulint n_len; ulint col_pack_len; @@ -9288,19 +9248,19 @@ calc_row_difference( uint i; ibool changes_fts_column = FALSE; ibool changes_fts_doc_col = FALSE; - trx_t* trx = thd_to_trx(thd); + trx_t* const trx = prebuilt->trx; doc_id_t doc_id = FTS_NULL_DOC_ID; ulint num_v = 0; ut_ad(!srv_read_only_mode); - n_fields = table->s->fields; clust_index = dict_table_get_first_index(prebuilt->table); + auto_inc = 0; /* We use upd_buff to convert changed fields */ buf = (byte*) upd_buff; - for (i = 0; i < n_fields; i++) { + for (i = 0; i < table->s->fields; i++) { field = table->field[i]; bool is_virtual = innobase_is_v_fld(field); dict_col_t* col; @@ -9521,11 +9481,22 @@ calc_row_difference( dfield_set_null(vfield); } num_v++; + ut_ad(field != table->found_next_number_field); } else { ufield->field_no = dict_col_get_clust_pos( &prebuilt->table->cols[i - num_v], clust_index); ufield->old_v_val = NULL; + if (field != table->found_next_number_field + || dfield_is_null(&ufield->new_val)) { + } else { + auto_inc = row_parse_int( + static_cast( + ufield->new_val.data), + ufield->new_val.len, + col->mtype, + col->prtype & DATA_UNSIGNED); + } } n_changed++; @@ -9576,7 +9547,7 @@ calc_row_difference( other changes. We piggy back our changes on the normal UPDATE to reduce processing and IO overhead. */ if (!prebuilt->table->fts) { - trx->fts_next_doc_id = 0; + trx->fts_next_doc_id = 0; } else if (changes_fts_column || changes_fts_doc_col) { dict_table_t* innodb_table = prebuilt->table; @@ -9787,20 +9758,15 @@ ha_innobase::update_row( } } - upd_t* uvect; - - if (m_prebuilt->upd_node) { - uvect = m_prebuilt->upd_node->update; - } else { - uvect = row_get_prebuilt_update_vector(m_prebuilt); - } + upd_t* uvect = row_get_prebuilt_update_vector(m_prebuilt); + ib_uint64_t autoinc; /* Build an update vector from the modified fields in the rows (uses m_upd_buf of the handle) */ error = calc_row_difference( uvect, old_row, new_row, table, m_upd_buf, m_upd_buf_size, - m_prebuilt, m_user_thd); + m_prebuilt, autoinc); if (error != DB_SUCCESS) { goto func_exit; @@ -9821,42 +9787,29 @@ ha_innobase::update_row( error = row_update_for_mysql((byte*) old_row, m_prebuilt); - /* We need to do some special AUTOINC handling for the following case: - - INSERT INTO t (c1,c2) VALUES(x,y) ON DUPLICATE KEY UPDATE ... + if (error == DB_SUCCESS && autoinc) { + /* A value for an AUTO_INCREMENT column + was specified in the UPDATE statement. */ - We need to use the AUTOINC counter that was actually used by - MySQL in the UPDATE statement, which can be different from the - value used in the INSERT statement. */ + autoinc = innobase_next_autoinc( + autoinc, 1, + m_prebuilt->autoinc_increment, + m_prebuilt->autoinc_offset, + innobase_get_int_col_max_value( + table->found_next_number_field)); - if (error == DB_SUCCESS - && table->next_number_field - && new_row == table->record[0] - && thd_sql_command(m_user_thd) == SQLCOM_INSERT - && trx->duplicates) { - - ulonglong auto_inc; - ulonglong col_max_value; - - auto_inc = table->next_number_field->val_int(); + error = innobase_set_max_autoinc(autoinc); - /* We need the upper limit of the col type to check for - whether we update the table autoinc counter or not. */ - col_max_value = - innobase_get_int_col_max_value(table->next_number_field); - - if (auto_inc <= col_max_value && auto_inc != 0) { - - ulonglong offset; - ulonglong increment; - - offset = m_prebuilt->autoinc_offset; - increment = m_prebuilt->autoinc_increment; - - auto_inc = innobase_next_autoinc( - auto_inc, 1, increment, offset, col_max_value); - - error = innobase_set_max_autoinc(auto_inc); + if (m_prebuilt->table->persistent_autoinc) { + /* Update the PAGE_ROOT_AUTO_INC. Yes, we do + this even if dict_table_t::autoinc already was + greater than autoinc, because we cannot know + if any INSERT actually used (and wrote to + PAGE_ROOT_AUTO_INC) a value bigger than our + autoinc. */ + btr_write_autoinc(dict_table_get_first_index( + m_prebuilt->table), + autoinc); } } @@ -14271,34 +14224,40 @@ create_table_info_t::create_table_update_dict() } } - /* Note: We can't call update_thd() as m_prebuilt will not be - setup at this stage and so we use thd. */ - - /* We need to copy the AUTOINC value from the old table if - this is an ALTER|OPTIMIZE TABLE or CREATE INDEX because CREATE INDEX - does a table copy too. If query was one of : + if (const Field* ai = m_form->found_next_number_field) { + ut_ad(!innobase_is_v_fld(ai)); - CREATE TABLE ...AUTO_INCREMENT = x; or - ALTER TABLE...AUTO_INCREMENT = x; or - OPTIMIZE TABLE t; or - CREATE INDEX x on t(...); + ib_uint64_t autoinc = m_create_info->auto_increment_value; - Find out a table definition from the dictionary and get - the current value of the auto increment field. Set a new - value to the auto increment field if the value is greater - than the maximum value in the column. */ + if (autoinc == 0) { + autoinc = 1; + } - if (((m_create_info->used_fields & HA_CREATE_USED_AUTO) - || thd_sql_command(m_thd) == SQLCOM_ALTER_TABLE - || thd_sql_command(m_thd) == SQLCOM_OPTIMIZE - || thd_sql_command(m_thd) == SQLCOM_CREATE_INDEX) - && m_create_info->auto_increment_value > 0) { - ib_uint64_t auto_inc_value; + dict_table_autoinc_lock(innobase_table); + dict_table_autoinc_initialize(innobase_table, autoinc); - auto_inc_value = m_create_info->auto_increment_value; + if (dict_table_is_temporary(innobase_table)) { + /* AUTO_INCREMENT is not persistent for + TEMPORARY TABLE. Temporary tables are never + evicted. Keep the counter in memory only. */ + } else { + const unsigned col_no = innodb_col_no(ai); + + innobase_table->persistent_autoinc = 1 + + dict_table_get_nth_col_pos( + innobase_table, col_no, NULL); + + /* Persist the "last used" value, which + typically is AUTO_INCREMENT - 1. + In btr_create(), the value 0 was already written. */ + if (--autoinc) { + btr_write_autoinc( + dict_table_get_first_index( + innobase_table), + autoinc); + } + } - dict_table_autoinc_lock(innobase_table); - dict_table_autoinc_initialize(innobase_table, auto_inc_value); dict_table_autoinc_unlock(innobase_table); } @@ -16412,18 +16371,7 @@ ha_innobase::info_low( } if ((flag & HA_STATUS_AUTO) && table->found_next_number_field) { - - ulonglong auto_inc_val = innobase_peek_autoinc(); - /* Initialize autoinc value if not set. */ - if (auto_inc_val == 0) { - - dict_table_autoinc_lock(m_prebuilt->table); - innobase_initialize_autoinc(); - dict_table_autoinc_unlock(m_prebuilt->table); - - auto_inc_val = innobase_peek_autoinc(); - } - stats.auto_increment_value = auto_inc_val; + stats.auto_increment_value = innobase_peek_autoinc(); } func_exit: diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 99d50d82fd180..cef39e594b99d 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -457,7 +457,6 @@ class ha_innobase: public handler int end_stmt(); dberr_t innobase_get_autoinc(ulonglong* value); - void innobase_initialize_autoinc(); dberr_t innobase_lock_autoinc(); ulonglong innobase_peek_autoinc(); dberr_t innobase_set_max_autoinc(ulonglong auto_inc); @@ -1142,6 +1141,13 @@ innobase_build_v_templ_callback( the table virtual columns' template */ typedef void (*my_gcolumn_templatecallback_t)(const TABLE*, void*); +/** Convert MySQL column number to dict_table_t::cols[] offset. +@param[in] field non-virtual column +@return column number relative to dict_table_t::cols[] */ +unsigned +innodb_col_no(const Field* field) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + /********************************************************************//** Helper function to push frm mismatch error to error log and if needed to sql-layer. */ diff --git a/storage/innobase/handler/ha_innopart.cc b/storage/innobase/handler/ha_innopart.cc index 5fd02dfa0168b..09a60be157701 100644 --- a/storage/innobase/handler/ha_innopart.cc +++ b/storage/innobase/handler/ha_innopart.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -827,17 +828,6 @@ ha_innopart::alter_table_flags( return(HA_PARTITION_FUNCTION_SUPPORTED | HA_FAST_CHANGE_PARTITION); } -/** Internally called for initializing auto increment value. -Only called from ha_innobase::discard_or_import_table_space() -and should not do anything, since it is ha_innopart will initialize -it on first usage. */ -int -ha_innopart::innobase_initialize_autoinc() -{ - ut_ad(0); - return(0); -} - /** Set the autoinc column max value. This should only be called once from ha_innobase::open(). Therefore there's no need for a covering lock. @@ -892,78 +882,33 @@ ha_innopart::initialize_auto_increment( my_error(ER_AUTOINC_READ_FAILED, MYF(0)); error = HA_ERR_AUTOINC_READ_FAILED; } else { - dict_index_t* index; - const char* col_name; - ib_uint64_t read_auto_inc; - ib_uint64_t max_auto_inc = 0; - ulint err; - dict_table_t* ib_table; - ulonglong col_max_value; - - col_max_value = field->get_max_int_value(); + ib_uint64_t col_max_value = field->get_max_int_value(); update_thd(ha_thd()); - col_name = field->field_name; for (uint part = 0; part < m_tot_parts; part++) { - ib_table = m_part_share->get_table_part(part); + dict_table_t* ib_table + = m_part_share->get_table_part(part); dict_table_autoinc_lock(ib_table); - read_auto_inc = dict_table_autoinc_read(ib_table); - if (read_auto_inc != 0) { - set_if_bigger(max_auto_inc, read_auto_inc); - dict_table_autoinc_unlock(ib_table); - continue; - } - /* Execute SELECT MAX(col_name) FROM TABLE; */ - index = m_part_share->get_index( - part, table->s->next_number_index); - err = row_search_max_autoinc( - index, col_name, &read_auto_inc); - - switch (err) { - case DB_SUCCESS: { + ut_ad(ib_table->persistent_autoinc); + ib_uint64_t read_auto_inc + = dict_table_autoinc_read(ib_table); + if (read_auto_inc == 0) { + read_auto_inc = btr_read_autoinc( + dict_table_get_first_index(ib_table)); + /* At the this stage we do not know the increment nor the offset, so use a default increment of 1. */ - auto_inc = innobase_next_autoinc( + read_auto_inc = innobase_next_autoinc( read_auto_inc, 1, 1, 0, col_max_value); - set_if_bigger(max_auto_inc, auto_inc); dict_table_autoinc_initialize(ib_table, - auto_inc); - break; - } - case DB_RECORD_NOT_FOUND: - ib::error() << "MySQL and InnoDB data" - " dictionaries are out of sync. Unable" - " to find the AUTOINC column " - << col_name << " in the InnoDB table " - << index->table->name << ". We set the" - " next AUTOINC column value to 0, in" - " effect disabling the AUTOINC next" - " value generation."; - - ib::info() << "You can either set the next" - " AUTOINC value explicitly using ALTER" - " TABLE or fix the data dictionary by" - " recreating the table."; - - /* We want the open to succeed, so that the - user can take corrective action. ie. reads - should succeed but updates should fail. */ - - /* This will disable the AUTOINC generation. */ - auto_inc = 0; - goto done; - default: - /* row_search_max_autoinc() should only return - one of DB_SUCCESS or DB_RECORD_NOT_FOUND. */ - - ut_error; + read_auto_inc); } + set_if_bigger(auto_inc, read_auto_inc); dict_table_autoinc_unlock(ib_table); } - auto_inc = max_auto_inc; } done: @@ -1578,22 +1523,6 @@ ha_innopart::update_partition( DBUG_VOID_RETURN; } -/** Save currently highest auto increment value. -@param[in] nr Auto increment value to save. */ -void -ha_innopart::save_auto_increment( - ulonglong nr) -{ - - /* Store it in the shared dictionary of the partition. - TODO: When the new DD is done, store it in the table and make it - persistent! */ - - dict_table_autoinc_lock(m_prebuilt->table); - dict_table_autoinc_update_if_greater(m_prebuilt->table, nr + 1); - dict_table_autoinc_unlock(m_prebuilt->table); -} - /** Was the last returned row semi consistent read. In an UPDATE or DELETE, if the row under the cursor was locked by another transaction, and the engine used an optimistic read of the last diff --git a/storage/innobase/handler/ha_innopart.h b/storage/innobase/handler/ha_innopart.h index 8caa9cdd8d2d0..5f3f8eaa31896 100644 --- a/storage/innobase/handler/ha_innopart.h +++ b/storage/innobase/handler/ha_innopart.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -723,12 +724,6 @@ class ha_innopart: int next_partition_index(); - /** Internally called for initializing auto increment value. - Should never be called, but defined to catch such errors. - @return 0 on success else error code. */ - int - innobase_initialize_autoinc(); - /** Get the index for the current partition @param[in] keynr MySQL index number. @return InnoDB index or NULL. */ @@ -772,12 +767,6 @@ class ha_innopart: initialize_auto_increment( bool /* no_lock */); - /** Save currently highest auto increment value. - @param[in] nr Auto increment value to save. */ - void - save_auto_increment( - ulonglong nr); - /** Setup the ordered record buffer and the priority queue. @param[in] used_parts Number of used partitions in query. @return false for success, else true. */ diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 51aef715ee2e3..e09ed99937a6a 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -173,8 +173,6 @@ struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx const dtuple_t* add_cols; /** autoinc sequence to use */ ib_sequence_t sequence; - /** maximum auto-increment value */ - ulonglong max_autoinc; /** temporary table name to use for old table when renaming tables */ const char* tmp_name; /** whether the order of the clustered index is unchanged */ @@ -224,7 +222,6 @@ struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx add_cols (0), sequence(prebuilt->trx->mysql_thd, autoinc_col_min_value_arg, autoinc_col_max_value_arg), - max_autoinc (0), tmp_name (0), skip_pk_sort(false), num_to_add_vcol(0), @@ -4975,6 +4972,23 @@ prepare_inplace_alter_table_dict( DBUG_EXECUTE_IF("innodb_alter_table_pk_assert_no_sort", DBUG_ASSERT(ctx->skip_pk_sort);); + DBUG_ASSERT(!ctx->new_table->persistent_autoinc); + if (const Field* ai = altered_table->found_next_number_field) { + const unsigned col_no = innodb_col_no(ai); + + ctx->new_table->persistent_autoinc = 1 + + dict_table_get_nth_col_pos( + ctx->new_table, col_no, NULL); + + /* Initialize the AUTO_INCREMENT sequence + to the rebuilt table from the old one. */ + if (!old_table->found_next_number_field) { + } else if (ib_uint64_t autoinc + = btr_read_autoinc(clust_index)) { + btr_write_autoinc(new_clust_index, autoinc); + } + } + if (ctx->online) { /* Allocate a log for online table rebuild. */ rw_lock_x_lock(&clust_index->lock); @@ -7400,83 +7414,120 @@ innobase_rename_or_enlarge_columns_cache( } } -/** Get the auto-increment value of the table on commit. +/** Set the auto-increment value of the table on commit. @param ha_alter_info Data used during in-place alter @param ctx In-place ALTER TABLE context @param altered_table MySQL table that is being altered -@param old_table MySQL table as it is before the ALTER operation -@return the next auto-increment value (0 if not present) */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -ulonglong -commit_get_autoinc( -/*===============*/ +@param old_table MySQL table as it is before the ALTER operation */ +static MY_ATTRIBUTE((nonnull)) +void +commit_set_autoinc( Alter_inplace_info* ha_alter_info, ha_innobase_inplace_ctx*ctx, const TABLE* altered_table, const TABLE* old_table) { - ulonglong max_autoinc; - DBUG_ENTER("commit_get_autoinc"); if (!altered_table->found_next_number_field) { /* There is no AUTO_INCREMENT column in the table after the ALTER operation. */ - max_autoinc = 0; } else if (ctx->add_autoinc != ULINT_UNDEFINED) { + ut_ad(ctx->need_rebuild()); /* An AUTO_INCREMENT column was added. Get the last value from the sequence, which may be based on a supplied AUTO_INCREMENT value. */ - max_autoinc = ctx->sequence.last(); + ib_uint64_t autoinc = ctx->sequence.last(); + ctx->new_table->autoinc = autoinc; + /* Bulk index creation does not update + PAGE_ROOT_AUTO_INC, so we must persist the "last used" + value here. */ + btr_write_autoinc(dict_table_get_first_index(ctx->new_table), + autoinc - 1, true); } else if ((ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && (ha_alter_info->create_info->used_fields & HA_CREATE_USED_AUTO)) { - /* An AUTO_INCREMENT value was supplied, but the table was not - rebuilt. Get the user-supplied value or the last value from the - sequence. */ - ib_uint64_t max_value_table; - dberr_t err; - - Field* autoinc_field = - old_table->found_next_number_field; - - dict_index_t* index = dict_table_get_index_on_first_col( - ctx->old_table, autoinc_field->field_index, - autoinc_field->field_name); - - max_autoinc = ha_alter_info->create_info->auto_increment_value; - - dict_table_autoinc_lock(ctx->old_table); - - err = row_search_max_autoinc( - index, autoinc_field->field_name, &max_value_table); - - if (err != DB_SUCCESS) { - ut_ad(0); - max_autoinc = 0; - } else if (max_autoinc <= max_value_table) { - ulonglong col_max_value; - ulonglong offset; - - col_max_value = innobase_get_int_col_max_value(autoinc_field); - - offset = ctx->prebuilt->autoinc_offset; - max_autoinc = innobase_next_autoinc( - max_value_table, 1, 1, offset, - col_max_value); + /* An AUTO_INCREMENT value was supplied by the user. + It must be persisted to the data file. */ + const Field* ai = old_table->found_next_number_field; + ut_ad(!strcmp(dict_table_get_col_name(ctx->old_table, + innodb_col_no(ai)), + ai->field_name)); + + ib_uint64_t autoinc + = ha_alter_info->create_info->auto_increment_value; + if (autoinc == 0) { + autoinc = 1; + } + + if (autoinc >= ctx->old_table->autoinc) { + /* Persist the predecessor of the + AUTO_INCREMENT value as the last used one. */ + ctx->new_table->autoinc = autoinc--; + } else { + /* Mimic ALGORITHM=COPY in the following scenario: + + CREATE TABLE t (a SERIAL); + INSERT INTO t SET a=100; + ALTER TABLE t AUTO_INCREMENT = 1; + INSERT INTO t SET a=NULL; + SELECT * FROM t; + + By default, ALGORITHM=INPLACE would reset the + sequence to 1, while after ALGORITHM=COPY, the + last INSERT would use a value larger than 100. + + We could only search the tree to know current + max counter in the table and compare. */ + const dict_col_t* autoinc_col + = dict_table_get_nth_col(ctx->old_table, + innodb_col_no(ai)); + dict_index_t* index + = dict_table_get_first_index(ctx->old_table); + while (index != NULL + && index->fields[0].col != autoinc_col) { + index = dict_table_get_next_index(index); + } + + ut_ad(index); + + ib_uint64_t max_in_table = index + ? row_search_max_autoinc(index) + : 0; + + if (autoinc <= max_in_table) { + ctx->new_table->autoinc = innobase_next_autoinc( + max_in_table, 1, + ctx->prebuilt->autoinc_increment, + ctx->prebuilt->autoinc_offset, + innobase_get_int_col_max_value(ai)); + /* Persist the maximum value as the + last used one. */ + autoinc = max_in_table; + } else { + /* Persist the predecessor of the + AUTO_INCREMENT value as the last used one. */ + ctx->new_table->autoinc = autoinc--; + } } - dict_table_autoinc_unlock(ctx->old_table); - } else { - /* An AUTO_INCREMENT value was not specified. - Read the old counter value from the table. */ - ut_ad(old_table->found_next_number_field); - dict_table_autoinc_lock(ctx->old_table); - max_autoinc = ctx->old_table->autoinc; - dict_table_autoinc_unlock(ctx->old_table); + + btr_write_autoinc(dict_table_get_first_index(ctx->new_table), + autoinc, true); + } else if (ctx->need_rebuild()) { + /* No AUTO_INCREMENT value was specified. + Copy it from the old table. */ + ctx->new_table->autoinc = ctx->old_table->autoinc; + /* The persistent value was already copied in + prepare_inplace_alter_table_dict() when ctx->new_table + was created. If this was a LOCK=NONE operation, the + AUTO_INCREMENT values would be updated during + row_log_table_apply(). If this was LOCK!=NONE, + the table contents could not possibly have changed + between prepare_inplace and commit_inplace. */ } - DBUG_RETURN(max_autoinc); + DBUG_VOID_RETURN; } /** Add or drop foreign key constraints to the data dictionary tables, @@ -8557,8 +8608,7 @@ ha_innobase::commit_inplace_alter_table( DBUG_ASSERT(new_clustered == ctx->need_rebuild()); - ctx->max_autoinc = commit_get_autoinc( - ha_alter_info, ctx, altered_table, table); + commit_set_autoinc(ha_alter_info, ctx, altered_table, table); if (ctx->need_rebuild()) { ctx->tmp_name = dict_mem_create_temporary_tablename( @@ -8896,14 +8946,6 @@ ha_innobase::commit_inplace_alter_table( (*pctx); DBUG_ASSERT(ctx->need_rebuild() == new_clustered); - if (altered_table->found_next_number_field) { - dict_table_t* t = ctx->new_table; - - dict_table_autoinc_lock(t); - dict_table_autoinc_initialize(t, ctx->max_autoinc); - dict_table_autoinc_unlock(t); - } - bool add_fts = false; /* Publish the created fulltext index, if any. diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index c177f23824f72..c6abd02536dda 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -353,6 +353,34 @@ btr_free( const page_id_t& page_id, const page_size_t& page_size); +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@return the last used AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc(dict_index_t* index) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, +or fall back to MAX(auto_increment_column). +@param[in] table table containing an AUTO_INCREMENT column +@param[in] col_no index of the AUTO_INCREMENT column +@return the AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc_with_fallback(const dict_table_t* table, unsigned col_no) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + +/** Write the next available AUTO_INCREMENT value to PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@param[in] autoinc the AUTO_INCREMENT value +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset = false) + MY_ATTRIBUTE((nonnull)); + /*************************************************************//** Makes tree one level higher by splitting the root, and inserts the tuple. It is assumed that mtr contains an x-latch on the tree. diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index e445331b60c68..579f725f6bd2d 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -190,7 +190,10 @@ btr_cur_search_to_nth_level( RW_S_LATCH, or 0 */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ - mtr_t* mtr); /*!< in: mtr */ + mtr_t* mtr, /*!< in/out: mini-transaction */ + ib_uint64_t autoinc = 0); + /*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ /*****************************************************************//** Opens a cursor at either end of an index. diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index 02f4faf24a556..947316f0e4de9 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -114,9 +114,11 @@ btr_pcur_open_low( btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ + ib_uint64_t autoinc,/*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ mtr_t* mtr); /*!< in: mtr */ #define btr_pcur_open(i,t,md,l,c,m) \ - btr_pcur_open_low(i,0,t,md,l,c,__FILE__,__LINE__,m) + btr_pcur_open_low(i,0,t,md,l,c,__FILE__,__LINE__,0,m) /**************************************************************//** Opens an persistent cursor to an index tree without initializing the cursor. */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index e7ae85dd73093..60790bc13164d 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -434,6 +434,8 @@ btr_pcur_open_low( btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ + ib_uint64_t autoinc,/*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ mtr_t* mtr) /*!< in: mtr */ { btr_cur_t* btr_cursor; @@ -454,7 +456,7 @@ btr_pcur_open_low( err = btr_cur_search_to_nth_level( index, level, tuple, mode, latch_mode, - btr_cursor, 0, file, line, mtr); + btr_cursor, 0, file, line, mtr, autoinc); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 6ac696e75eb68..d42a5f6ee4bc8 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2015, MariaDB Corporation. +Copyright (c) 2013, 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -327,46 +327,52 @@ dict_table_autoinc_lock( /*====================*/ dict_table_t* table) /*!< in/out: table */ MY_ATTRIBUTE((nonnull)); -/********************************************************************//** -Unconditionally set the autoinc counter. */ +/** Unconditionally set the AUTO_INCREMENT counter. +@param[in,out] table table or partition +@param[in] value next available AUTO_INCREMENT value */ +MY_ATTRIBUTE((nonnull)) +UNIV_INLINE void -dict_table_autoinc_initialize( -/*==========================*/ - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: next value to assign to a row */ - MY_ATTRIBUTE((nonnull)); +dict_table_autoinc_initialize(dict_table_t* table, ib_uint64_t value) +{ + ut_ad(dict_table_autoinc_own(table)); + table->autoinc = value; +} -/** Store autoinc value when the table is evicted. -@param[in] table table evicted */ -void -dict_table_autoinc_store( - const dict_table_t* table); +/** +@param[in] table table or partition +@return the next AUTO_INCREMENT counter value +@retval 0 if AUTO_INCREMENT is not yet initialized */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) +UNIV_INLINE +ib_uint64_t +dict_table_autoinc_read(const dict_table_t* table) +{ + ut_ad(dict_table_autoinc_own(table)); + return(table->autoinc); +} -/** Restore autoinc value when the table is loaded. -@param[in] table table loaded */ -void -dict_table_autoinc_restore( - dict_table_t* table); +/** Update the AUTO_INCREMENT sequence if the value supplied is greater +than the current value. +@param[in,out] table table or partition +@param[in] value AUTO_INCREMENT value that was assigned to a row +@return whether the AUTO_INCREMENT sequence was updated */ +MY_ATTRIBUTE((nonnull)) +UNIV_INLINE +bool +dict_table_autoinc_update_if_greater(dict_table_t* table, ib_uint64_t value) +{ + ut_ad(dict_table_autoinc_own(table)); -/********************************************************************//** -Reads the next autoinc value (== autoinc counter value), 0 if not yet -initialized. -@return value for a new row, or 0 */ -ib_uint64_t -dict_table_autoinc_read( -/*====================*/ - const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); -/********************************************************************//** -Updates the autoinc counter if the value supplied is greater than the -current value. */ -void -dict_table_autoinc_update_if_greater( -/*=================================*/ + if (value > table->autoinc) { + + table->autoinc = value; + return(true); + } + + return(false); +} - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: value which was assigned to a row */ - MY_ATTRIBUTE((nonnull)); /********************************************************************//** Release the autoinc lock. */ void @@ -412,8 +418,9 @@ void dict_table_remove_from_cache_low( /*=============================*/ dict_table_t* table, /*!< in, own: table */ - ibool lru_evict); /*!< in: TRUE if table being evicted + ibool lru_evict) /*!< in: TRUE if table being evicted to make room in the table LRU list */ + MY_ATTRIBUTE((nonnull)); /**********************************************************************//** Renames a table object. @return TRUE if success */ @@ -1746,8 +1753,6 @@ extern dict_sys_t* dict_sys; /** the data dictionary rw-latch protecting dict_sys */ extern rw_lock_t* dict_operation_lock; -typedef std::map autoinc_map_t; - /* Dictionary system struct */ struct dict_sys_t{ DictSysMutex mutex; /*!< mutex protecting the data @@ -1783,8 +1788,6 @@ struct dict_sys_t{ UT_LIST_BASE_NODE_T(dict_table_t) table_non_LRU; /*!< List of tables that can't be evicted from the cache */ - autoinc_map_t* autoinc_map; /*!< Map to store table id and autoinc - when table is evicted */ }; #endif /* !UNIV_HOTBACKUP */ @@ -2052,18 +2055,6 @@ dict_index_node_ptr_max_size( /*=========================*/ const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((warn_unused_result)); -/*****************************************************************//** -Get index by first field of the index -@return index which is having first field matches -with the field present in field_index position of table */ -UNIV_INLINE -dict_index_t* -dict_table_get_index_on_first_col( -/*==============================*/ - const dict_table_t* table, /*!< in: table */ - ulint col_index, /*!< in: position of column - in table */ - const char* field_name); /*!< in: field name */ /** Check if a column is a virtual column @param[in] col column @return true if it is a virtual column, false otherwise */ diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index c44dc156aaa52..9bf6b8df4bc9b 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1859,42 +1859,6 @@ dict_table_is_file_per_table( return(is_file_per_table ); } -/**********************************************************************//** -Get index by first field of the index -@return index which is having first field matches -with the field present in field_index position of table */ -UNIV_INLINE -dict_index_t* -dict_table_get_index_on_first_col( -/*==============================*/ - const dict_table_t* table, /*!< in: table */ - ulint col_index, /*!< in: position of column - in table */ - const char* field_name) /*!< in: field name */ -{ - ut_ad(col_index < table->n_cols); - - dict_col_t* column = dict_table_get_nth_col(table, col_index); - - for (dict_index_t* index = dict_table_get_first_index(table); - index != NULL; index = dict_table_get_next_index(index)) { - - if (index->fields[0].col == column) { - return(index); - } - } - - /* If not yet found use field_name */ - for (dict_index_t* index = dict_table_get_first_index(table); - index != NULL; index = dict_table_get_next_index(index)) { - if (!strcmp(index->fields[0].name, field_name)) { - return (index); - } - } - ut_error; - return(0); -} - /** Get reference count. @return current value of n_ref_count */ inline diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index c81d893ed5a86..422404394bea4 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1466,6 +1466,11 @@ struct dict_table_t { /** Number of virtual columns. */ unsigned n_v_cols:10; + /** 1 + the position of autoinc counter field in clustered + index, or 0 if there is no persistent AUTO_INCREMENT column in + the table. */ + unsigned persistent_autoinc:10; + /** TRUE if it's not an InnoDB system table or a table that has no FK relationships. */ unsigned can_be_evicted:1; diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index 3bb622127fbde..b64602fe077e6 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -1,5 +1,4 @@ /***************************************************************************** - Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2016, MariaDB Corporation @@ -70,10 +69,12 @@ typedef byte page_header_t; #define PAGE_N_DIRECTION 14 /* number of consecutive inserts to the same direction */ #define PAGE_N_RECS 16 /* number of user records on the page */ -#define PAGE_MAX_TRX_ID 18 /* highest id of a trx which may have modified - a record on the page; trx_id_t; defined only - in secondary indexes and in the insert buffer - tree */ +/** The largest DB_TRX_ID that may have modified a record on the page; +Defined only in secondary index leaf pages and in change buffer leaf pages. +Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ +#define PAGE_MAX_TRX_ID 18 +/** The AUTO_INCREMENT value (on persistent clustered index root pages). */ +#define PAGE_ROOT_AUTO_INC PAGE_MAX_TRX_ID #define PAGE_HEADER_PRIV_END 26 /* end of private data structure of the page header which are set in a page create */ /*----*/ @@ -213,6 +214,32 @@ page_update_max_trx_id( uncompressed part will be updated, or NULL */ trx_id_t trx_id, /*!< in: transaction id */ mtr_t* mtr); /*!< in/out: mini-transaction */ + +/** Persist the AUTO_INCREMENT value on a clustered index root page. +@param[in,out] block clustered index root page +@param[in] index clustered index +@param[in] autoinc next available AUTO_INCREMENT value +@param[in,out] mtr mini-transaction +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +page_set_autoinc( + buf_block_t* block, + const dict_index_t* index MY_ATTRIBUTE((unused)), + ib_uint64_t autoinc, + mtr_t* mtr, + bool reset) + MY_ATTRIBUTE((nonnull)); + +/** Read the AUTO_INCREMENT value from a clustered index root page. +@param[in] page clustered index root page +@return the persisted AUTO_INCREMENT value */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) +UNIV_INLINE +ib_uint64_t +page_get_autoinc(const page_t* page); + /*************************************************************//** Returns the RTREE SPLIT SEQUENCE NUMBER (FIL_RTREE_SPLIT_SEQ_NUM). @return SPLIT SEQUENCE NUMBER */ diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 0a0ff41774c7b..a3bbc09e60f82 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -108,6 +109,17 @@ page_update_max_trx_id( } } +/** Read the AUTO_INCREMENT value from a clustered index root page. +@param[in] page clustered index root page +@return the persisted AUTO_INCREMENT value */ +UNIV_INLINE +ib_uint64_t +page_get_autoinc(const page_t* page) +{ + ut_ad(page_is_root(page)); + return(mach_read_from_8(PAGE_HEADER + PAGE_ROOT_AUTO_INC + page)); +} + /*************************************************************//** Returns the RTREE SPLIT SEQUENCE NUMBER (FIL_RTREE_SPLIT_SEQ_NUM). @return SPLIT SEQUENCE NUMBER */ diff --git a/storage/innobase/include/row0row.h b/storage/innobase/include/row0row.h index 93ff90d020eb7..9ae1387810aa8 100644 --- a/storage/innobase/include/row0row.h +++ b/storage/innobase/include/row0row.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -315,6 +316,22 @@ row_get_clust_rec( mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Parse the integer data from specified data, which could be +DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 +and the type is not unsigned then we reset the value to 0 +@param[in] data data to read +@param[in] len length of data +@param[in] mtype mtype of data +@param[in] unsigned_type if the data is unsigned +@return the integer value from the data */ +inline +ib_uint64_t +row_parse_int( + const byte* data, + ulint len, + ulint mtype, + bool unsigned_type); + /** Result of row_search_index_entry */ enum row_search_result { ROW_FOUND = 0, /*!< the record was found */ diff --git a/storage/innobase/include/row0row.ic b/storage/innobase/include/row0row.ic index 08c0f18e95b99..9243fae839ae6 100644 --- a/storage/innobase/include/row0row.ic +++ b/storage/innobase/include/row0row.ic @@ -173,3 +173,52 @@ row_build_row_ref_fast( } } } + +/** Parse the integer data from specified data, which could be +DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 +and the type is not unsigned then we reset the value to 0 +@param[in] data data to read +@param[in] len length of data +@param[in] mtype mtype of data +@param[in] unsigned_type if the data is unsigned +@return the integer value from the data */ +ib_uint64_t +row_parse_int( + const byte* data, + ulint len, + ulint mtype, + bool unsigned_type) +{ + ib_uint64_t value = 0; + + switch (mtype) { + case DATA_INT: + + ut_a(len <= sizeof value); + value = mach_read_int_type(data, len, unsigned_type); + break; + + case DATA_FLOAT: + + ut_a(len == sizeof(float)); + value = mach_float_read(data); + break; + + case DATA_DOUBLE: + + ut_a(len == sizeof(double)); + value = mach_double_read(data); + break; + + default: + ut_error; + + } + + if (!unsigned_type && static_cast(value) < 0) { + value = 0; + } + + return(value); +} + diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 1186aa6f26e67..3f87d78e25ae2 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -230,15 +230,12 @@ row_search_check_if_query_cache_permitted( trx_t* trx, /*!< in: transaction object */ const char* norm_name); /*!< in: concatenation of database name, '/' char, table name */ -/*******************************************************************//** -Read the max AUTOINC value from an index. -@return DB_SUCCESS if all OK else error code */ -dberr_t -row_search_max_autoinc( -/*===================*/ - dict_index_t* index, /*!< in: index to search */ - const char* col_name, /*!< in: autoinc column name */ - ib_uint64_t* value) /*!< out: AUTOINC value read */ +/** Read the max AUTOINC value from an index. +@param[in] index index starting with an AUTO_INCREMENT column +@return the largest AUTO_INCREMENT value +@retval 0 if no records were found */ +ib_uint64_t +row_search_max_autoinc(dict_index_t* index) MY_ATTRIBUTE((nonnull, warn_unused_result)); /** A structure for caching column values for prefetched rows */ diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 89669d09e89aa..d207371b21321 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -228,6 +228,40 @@ page_set_max_trx_id( } } +/** Persist the AUTO_INCREMENT value on a clustered index root page. +@param[in,out] block clustered index root page +@param[in] index clustered index +@param[in] autoinc next available AUTO_INCREMENT value +@param[in,out] mtr mini-transaction +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +page_set_autoinc( + buf_block_t* block, + const dict_index_t* index MY_ATTRIBUTE((unused)), + ib_uint64_t autoinc, + mtr_t* mtr, + bool reset) +{ + ut_ad(mtr_memo_contains_flagged( + mtr, block, MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX)); + ut_ad(dict_index_is_clust(index)); + ut_ad(index->page == block->page.id.page_no()); + ut_ad(index->space == block->page.id.space()); + + byte* field = PAGE_HEADER + PAGE_ROOT_AUTO_INC + + buf_block_get_frame(block); + if (!reset && mach_read_from_8(field) >= autoinc) { + /* nothing to update */ + } else if (page_zip_des_t* page_zip = buf_block_get_page_zip(block)) { + mach_write_to_8(field, autoinc); + page_zip_write_header(page_zip, field, 8, mtr); + } else { + mlog_write_ull(field, autoinc, mtr); + } +} + /************************************************************//** Allocates a block of memory from the heap of an index page. @return pointer to start of allocated buffer, or NULL if allocation fails */ diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 277a47bbad5d3..621540afcef97 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4753,19 +4753,19 @@ page_zip_reorganize( page_get_infimum_rec(temp_page), index, mtr); - /* Temp-Tables are not shared across connection and so we avoid - locking of temp-tables as there would be no 2 trx trying to - operate on same temp-table in parallel. - max_trx_id is use to track which all trxs wrote to the page - in parallel but in case of temp-table this can is not needed. */ - if (!dict_index_is_clust(index) - && !dict_table_is_temporary(index->table) - && page_is_leaf(temp_page)) { - /* Copy max trx id to recreated page */ - trx_id_t max_trx_id = page_get_max_trx_id(temp_page); - page_set_max_trx_id(block, NULL, max_trx_id, NULL); - ut_ad(max_trx_id != 0); - } + /* Copy the PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC. */ + memcpy(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), + temp_page + (PAGE_HEADER + PAGE_MAX_TRX_ID), 8); + /* PAGE_MAX_TRX_ID must be set on secondary index leaf pages. */ + ut_ad(dict_index_is_clust(index) || !page_is_leaf(temp_page) + || dict_table_is_temporary(index->table) + || page_get_max_trx_id(page) != 0); + /* PAGE_MAX_TRX_ID must be zero on non-leaf pages other than + clustered index root pages. */ + ut_ad(page_get_max_trx_id(page) == 0 + || (dict_index_is_clust(index) + ? page_is_root(temp_page) + : page_is_leaf(temp_page))); /* Restore logging. */ mtr_set_log_mode(mtr, log_mode); diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 1ae11204f69ee..4cdcf1372031c 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -4009,18 +4009,15 @@ row_import_for_mysql( table->ibd_file_missing = false; table->flags2 &= ~DICT_TF2_DISCARDED; - /* Set autoinc value read from cfg file. The value is set to zero - if the cfg file is missing and is initialized later from table - column value. */ - ib::info() << table->name << " autoinc value set to " - << autoinc; + /* Set autoinc value read from .cfg file, if one was specified. + Otherwise, keep the PAGE_ROOT_AUTO_INC as is. */ + if (autoinc) { + ib::info() << table->name << " autoinc value set to " + << autoinc; - dict_table_autoinc_lock(table); - dict_table_autoinc_initialize(table, autoinc); - dict_table_autoinc_unlock(table); - - ut_a(err == DB_SUCCESS); + table->autoinc = autoinc--; + btr_write_autoinc(dict_table_get_first_index(table), autoinc); + } return(row_import_cleanup(prebuilt, trx, err)); } - diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 271d70d4da9cd..1f884017dd395 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2473,6 +2474,7 @@ row_ins_clust_index_entry_low( dberr_t err = DB_SUCCESS; big_rec_t* big_rec = NULL; mtr_t mtr; + ib_uint64_t auto_inc = 0; mem_heap_t* offsets_heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; @@ -2487,7 +2489,6 @@ row_ins_clust_index_entry_low( ut_ad(!thr_get_trx(thr)->in_rollback); mtr_start(&mtr); - mtr.set_named_space(index->space); if (dict_table_is_temporary(index->table)) { /* Disable REDO logging as the lifetime of temp-tables is @@ -2496,23 +2497,41 @@ row_ins_clust_index_entry_low( Disable locking as temp-tables are local to a connection. */ ut_ad(flags & BTR_NO_LOCKING_FLAG); + ut_ad(!dict_index_is_online_ddl(index)); + ut_ad(!index->table->persistent_autoinc); mtr.set_log_mode(MTR_LOG_NO_REDO); - } + } else { + mtr.set_named_space(index->space); - if (mode == BTR_MODIFY_LEAF && dict_index_is_online_ddl(index)) { - mode = BTR_MODIFY_LEAF | BTR_ALREADY_S_LATCHED; - mtr_s_lock(dict_index_get_lock(index), &mtr); + if (mode == BTR_MODIFY_LEAF + && dict_index_is_online_ddl(index)) { + mode = BTR_MODIFY_LEAF | BTR_ALREADY_S_LATCHED; + mtr_s_lock(dict_index_get_lock(index), &mtr); + } + + if (unsigned ai = index->table->persistent_autoinc) { + /* Prepare to persist the AUTO_INCREMENT value + from the index entry to PAGE_ROOT_AUTO_INC. */ + const dfield_t* dfield = dtuple_get_nth_field( + entry, ai - 1); + auto_inc = dfield_is_null(dfield) + ? 0 + : row_parse_int(static_cast( + dfield->data), + dfield->len, + dfield->type.mtype, + dfield->type.prtype + & DATA_UNSIGNED); + } } /* Note that we use PAGE_CUR_LE as the search mode, because then the function will return in both low_match and up_match of the cursor sensible values */ - btr_pcur_open(index, entry, PAGE_CUR_LE, mode, &pcur, &mtr); + btr_pcur_open_low(index, 0, entry, PAGE_CUR_LE, mode, &pcur, + __FILE__, __LINE__, auto_inc, &mtr); cursor = btr_pcur_get_btr_cur(&pcur); - - if (cursor) { - cursor->thr = thr; - } + cursor->thr = thr; #ifdef UNIV_DEBUG { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 100e1bcb7083f..d2b0dbc0576d2 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1705,24 +1705,18 @@ row_get_prebuilt_update_vector( row_prebuilt_t* prebuilt) /*!< in: prebuilt struct in MySQL handle */ { - dict_table_t* table = prebuilt->table; - upd_node_t* node; - - ut_ad(prebuilt && table && prebuilt->trx); - if (prebuilt->upd_node == NULL) { /* Not called before for this handle: create an update node and query graph to the prebuilt struct */ - node = row_create_update_node_for_mysql(table, prebuilt->heap); - - prebuilt->upd_node = node; + prebuilt->upd_node = row_create_update_node_for_mysql( + prebuilt->table, prebuilt->heap); prebuilt->upd_graph = static_cast( que_node_get_parent( pars_complete_graph_for_exec( - static_cast(node), + prebuilt->upd_node, prebuilt->trx, prebuilt->heap, prebuilt))); diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 5daa26319a560..967d0ac98cd4f 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -6045,29 +6045,7 @@ row_search_autoinc_read_column( data = rec_get_nth_field(rec, offsets, col_no, &len); - switch (mtype) { - case DATA_INT: - ut_a(len <= sizeof value); - value = mach_read_int_type(data, len, unsigned_type); - break; - - case DATA_FLOAT: - ut_a(len == sizeof(float)); - value = (ib_uint64_t) mach_float_read(data); - break; - - case DATA_DOUBLE: - ut_a(len == sizeof(double)); - value = (ib_uint64_t) mach_double_read(data); - break; - - default: - ut_error; - } - - if (!unsigned_type && static_cast(value) < 0) { - value = 0; - } + value = row_parse_int(data, len, mtype, unsigned_type); func_exit: if (UNIV_LIKELY_NULL(heap)) { @@ -6113,42 +6091,27 @@ row_search_get_max_rec( return(rec); } -/*******************************************************************//** -Read the max AUTOINC value from an index. -@return DB_SUCCESS if all OK else error code, DB_RECORD_NOT_FOUND if -column name can't be found in index */ -dberr_t -row_search_max_autoinc( -/*===================*/ - dict_index_t* index, /*!< in: index to search */ - const char* col_name, /*!< in: name of autoinc column */ - ib_uint64_t* value) /*!< out: AUTOINC value read */ +/** Read the max AUTOINC value from an index. +@param[in] index index starting with an AUTO_INCREMENT column +@return the largest AUTO_INCREMENT value +@retval 0 if no records were found */ +ib_uint64_t +row_search_max_autoinc(dict_index_t* index) { - dict_field_t* dfield = dict_index_get_nth_field(index, 0); - dberr_t error = DB_SUCCESS; - *value = 0; - - if (strcmp(col_name, dfield->name) != 0) { - error = DB_RECORD_NOT_FOUND; - } else { - mtr_t mtr; - const rec_t* rec; + const dict_field_t* dfield = dict_index_get_nth_field(index, 0); - mtr_start(&mtr); - - rec = row_search_get_max_rec(index, &mtr); + ib_uint64_t value = 0; - if (rec != NULL) { - ibool unsigned_type = ( - dfield->col->prtype & DATA_UNSIGNED); - - *value = row_search_autoinc_read_column( - index, rec, 0, - dfield->col->mtype, unsigned_type); - } + mtr_t mtr; + mtr.start(); - mtr_commit(&mtr); + if (const rec_t* rec = row_search_get_max_rec(index, &mtr)) { + value = row_search_autoinc_read_column( + index, rec, 0, + dfield->col->mtype, + dfield->col->prtype & DATA_UNSIGNED); } - return(error); + mtr.commit(); + return(value); } From cb0ce5c2e9cafb8f3c41d45a28292f1898ccdbd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 16 Dec 2016 10:19:14 +0200 Subject: [PATCH 274/295] MDEV-6076: Optimize the test. Remove unnecessary restarts by testing multiple tables across a restart. This change almost halves the execution time. Some further restarts could be removed with additional effort. --- .../innodb/include/autoinc_persist_alter.inc | 14 +- .../suite/innodb/r/autoinc_persist.result | 339 ++++-------------- .../suite/innodb/t/autoinc_persist.test | 117 +++--- 3 files changed, 129 insertions(+), 341 deletions(-) diff --git a/mysql-test/suite/innodb/include/autoinc_persist_alter.inc b/mysql-test/suite/innodb/include/autoinc_persist_alter.inc index ddff573b0565f..bbf5f26508594 100644 --- a/mysql-test/suite/innodb/include/autoinc_persist_alter.inc +++ b/mysql-test/suite/innodb/include/autoinc_persist_alter.inc @@ -41,21 +41,17 @@ eval SELECT MAX(a) AS `Expect 124` FROM $table; eval OPTIMIZE TABLE $table; -eval SHOW CREATE TABLE $table; - ---source include/restart_mysqld.inc - ---echo # We expect the counter to still be 125 -eval SHOW CREATE TABLE $table; - eval DELETE FROM $table WHERE a >= 123; -eval CREATE UNIQUE INDEX idx_aa ON $table(a); +eval CREATE TABLE i$table(a INT AUTO_INCREMENT, INDEX(a)) AUTO_INCREMENT=125 ENGINE=InnoDB; +eval CREATE UNIQUE INDEX idx_aa ON i$table(a); --source include/restart_mysqld.inc eval INSERT INTO $table VALUES(0), (0); +eval INSERT INTO i$table VALUES(0), (0); eval SELECT MAX(a) AS `Expect 126` FROM $table; +eval SELECT MAX(a) AS `Expect 126` FROM i$table; -eval DROP TABLE $table; +eval DROP TABLE $table, i$table; diff --git a/mysql-test/suite/innodb/r/autoinc_persist.result b/mysql-test/suite/innodb/r/autoinc_persist.result index b9eefe30ddc58..0494a256f934e 100644 --- a/mysql-test/suite/innodb/r/autoinc_persist.result +++ b/mysql-test/suite/innodb/r/autoinc_persist.result @@ -204,7 +204,6 @@ a 20 30 31 -set global innodb_flush_log_at_trx_commit=1; CREATE TABLE t12(a DOUBLE AUTO_INCREMENT KEY) ENGINE = InnoDB; INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); @@ -220,185 +219,9 @@ a 20 30 31 -CREATE TABLE t13(a INT AUTO_INCREMENT PRIMARY KEY) ENGINE = InnoDB, -AUTO_INCREMENT = 1234; # Scenario 1: Normal restart, to test if the counters are persisted -# We expect these results should be equal to above SELECTs -SELECT * FROM t1; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -SELECT * FROM t2; -a -1 -2 -3 -4 -8 -10 -11 -20 -30 -31 -SELECT * FROM t3; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -1024 -4096 -SELECT * FROM t4; -a -1 -2 -3 -4 -8 -10 -11 -20 -30 -31 -1024 -4096 -SELECT * FROM t5; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -1000000 -1000005 -SELECT * FROM t6; -a -1 -2 -3 -4 -8 -10 -11 -20 -30 -31 -1000000 -1000005 -SELECT * FROM t7; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -100000000 -100000008 -SELECT * FROM t8; -a -1 -2 -3 -4 -8 -10 -11 -20 -30 -31 -100000000 -100000008 -SELECT * FROM t9; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -100000000000 -100000000006 -SELECT * FROM t10; -a -1 -2 -3 -4 -8 -10 -11 -20 -30 -31 -100000000000 -100000000006 -SELECT * FROM t11; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -SELECT * FROM t12; -a --10 --1 -1 -2 -3 -4 -5 -20 -30 -31 -SELECT * FROM t13; -a -SHOW CREATE TABLE t13; -Table Create Table -t13 CREATE TABLE `t13` ( - `a` int(11) NOT NULL AUTO_INCREMENT, - PRIMARY KEY (`a`) -) ENGINE=InnoDB AUTO_INCREMENT=1234 DEFAULT CHARSET=latin1 -INSERT INTO t13 VALUES(0); -SELECT a AS `Expect 1234` FROM t13; -Expect 1234 -1234 # Scenario 2: Delete some values, to test the counters should not be the # one which is the largest in current table -set global innodb_flush_log_at_trx_commit=1; DELETE FROM t1 WHERE a > 30; SELECT MAX(a) AS `Expect 30` FROM t1; Expect 30 @@ -419,6 +242,19 @@ DELETE FROM t9 WHERE a > 100000000000; SELECT MAX(a) AS `Expect 100000000000` FROM t9; Expect 100000000000 100000000000 +CREATE TABLE t13(a INT AUTO_INCREMENT PRIMARY KEY) ENGINE = InnoDB, +AUTO_INCREMENT = 1234; +SHOW CREATE TABLE t13; +Table Create Table +t13 CREATE TABLE `t13` ( + `a` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) +) ENGINE=InnoDB AUTO_INCREMENT=1234 DEFAULT CHARSET=latin1 +INSERT INTO t13 VALUES(0); +SELECT a AS `Expect 1234` FROM t13; +Expect 1234 +1234 +DROP TABLE t13; INSERT INTO t1 VALUES(0), (0); SELECT MAX(a) AS `Expect 33` FROM t1; Expect 33 @@ -543,7 +379,6 @@ INSERT INTO t7 VALUES(0); SELECT MAX(a) AS `Expect 100000109` FROM t7; Expect 100000109 100000109 -set global innodb_flush_log_at_trx_commit=1; INSERT INTO t9 VALUES(100), (200); UPDATE t9 SET a = 100000000105 WHERE a = 5; UPDATE t9 SET a = 100000000100 WHERE a = 100; @@ -593,6 +428,7 @@ INSERT INTO t5 VALUES(1100200); DELETE FROM t5 WHERE a = 1100200; INSERT INTO t7 VALUES(100000200); DELETE FROM t7 WHERE a = 100000200; +# Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; INSERT INTO t9 VALUES(100000000200); DELETE FROM t9 WHERE a = 100000000200; @@ -648,12 +484,20 @@ SELECT * FROM t9; a 1 2 +# Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; TRUNCATE TABLE t1; TRUNCATE TABLE t3; TRUNCATE TABLE t5; TRUNCATE TABLE t7; TRUNCATE TABLE t9; +# Scenario 7: Test explicit rename table won't change the counter +RENAME TABLE t9 to t19; +INSERT INTO t19 VALUES(0), (0); +SELECT * FROM t19; +a +1 +2 # Kill and restart INSERT INTO t1 VALUES(0), (0); SELECT * FROM t1; @@ -675,14 +519,6 @@ SELECT * FROM t7; a 1 2 -INSERT INTO t9 VALUES(0), (0); -SELECT * FROM t9; -a -1 -2 -# Scenario 7: Test explicit rename table won't change the counter -set global innodb_flush_log_at_trx_commit=1; -RENAME TABLE t9 to t19; INSERT INTO t19 VALUES(0), (0); SELECT * FROM t19; a @@ -691,7 +527,6 @@ a 3 4 DELETE FROM t19 WHERE a = 4; -# Kill and restart RENAME TABLE t19 to t9; INSERT INTO t9 VALUES(0), (0); SELECT * FROM t9; @@ -799,20 +634,19 @@ INSERT INTO t3 VALUES(0); SELECT MAX(a) AS `Expect 120` FROM t3; Expect 120 120 -INSERT INTO mdev6076a SET b=0; +INSERT INTO mdev6076a SET b=NULL; SELECT * FROM mdev6076a; a b 1 2 2 1 -3 0 -INSERT INTO mdev6076b SET b=0; +3 NULL +INSERT INTO mdev6076b SET b=NULL; SELECT * FROM mdev6076b; a b 100 2 101 1 -102 0 +102 NULL DROP TABLE mdev6076a, mdev6076b; -set global innodb_flush_log_at_trx_commit=1; INSERT INTO t3 VALUES(0), (0), (200), (210); # Test the different algorithms in ALTER TABLE CREATE TABLE t_inplace LIKE t3; @@ -890,26 +724,18 @@ OPTIMIZE TABLE t_inplace; Table Op Msg_type Msg_text test.t_inplace optimize note Table does not support optimize, doing recreate + analyze instead test.t_inplace optimize status OK -SHOW CREATE TABLE t_inplace; -Table Create Table -t_inplace CREATE TABLE `t_inplace` ( - `a` smallint(6) NOT NULL AUTO_INCREMENT, - PRIMARY KEY (`a`) -) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 -# We expect the counter to still be 125 -SHOW CREATE TABLE t_inplace; -Table Create Table -t_inplace CREATE TABLE `t_inplace` ( - `a` smallint(6) NOT NULL AUTO_INCREMENT, - PRIMARY KEY (`a`) -) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 DELETE FROM t_inplace WHERE a >= 123; -CREATE UNIQUE INDEX idx_aa ON t_inplace(a); +CREATE TABLE it_inplace(a INT AUTO_INCREMENT, INDEX(a)) AUTO_INCREMENT=125 ENGINE=InnoDB; +CREATE UNIQUE INDEX idx_aa ON it_inplace(a); INSERT INTO t_inplace VALUES(0), (0); +INSERT INTO it_inplace VALUES(0), (0); SELECT MAX(a) AS `Expect 126` FROM t_inplace; Expect 126 126 -DROP TABLE t_inplace; +SELECT MAX(a) AS `Expect 126` FROM it_inplace; +Expect 126 +126 +DROP TABLE t_inplace, it_inplace; CREATE TABLE t_copy LIKE t3; INSERT INTO t_copy SELECT * FROM t3; SELECT * FROM t_copy; @@ -985,26 +811,18 @@ OPTIMIZE TABLE t_copy; Table Op Msg_type Msg_text test.t_copy optimize note Table does not support optimize, doing recreate + analyze instead test.t_copy optimize status OK -SHOW CREATE TABLE t_copy; -Table Create Table -t_copy CREATE TABLE `t_copy` ( - `a` smallint(6) NOT NULL AUTO_INCREMENT, - PRIMARY KEY (`a`) -) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 -# We expect the counter to still be 125 -SHOW CREATE TABLE t_copy; -Table Create Table -t_copy CREATE TABLE `t_copy` ( - `a` smallint(6) NOT NULL AUTO_INCREMENT, - PRIMARY KEY (`a`) -) ENGINE=InnoDB AUTO_INCREMENT=125 DEFAULT CHARSET=latin1 DELETE FROM t_copy WHERE a >= 123; -CREATE UNIQUE INDEX idx_aa ON t_copy(a); +CREATE TABLE it_copy(a INT AUTO_INCREMENT, INDEX(a)) AUTO_INCREMENT=125 ENGINE=InnoDB; +CREATE UNIQUE INDEX idx_aa ON it_copy(a); INSERT INTO t_copy VALUES(0), (0); +INSERT INTO it_copy VALUES(0), (0); SELECT MAX(a) AS `Expect 126` FROM t_copy; Expect 126 126 -DROP TABLE t_copy; +SELECT MAX(a) AS `Expect 126` FROM it_copy; +Expect 126 +126 +DROP TABLE t_copy, it_copy; # Scenario 9: Test the sql_mode = NO_AUTO_VALUE_ON_ZERO CREATE TABLE t30 (a INT NOT NULL AUTO_INCREMENT PRIMARY KEY, b INT, key(b)) ENGINE = InnoDB; set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; @@ -1029,6 +847,7 @@ a b 202 5 203 6 204 7 +# Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; CREATE TABLE t31 (a INT) ENGINE = InnoDB; INSERT INTO t31 VALUES(1), (2); @@ -1043,6 +862,43 @@ a b 2 2 4 3 5 4 +SET SQL_MODE = 0; +# Scenario 10: Rollback would not rollback the counter +CREATE TABLE t32 ( +a BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; +INSERT INTO t32 VALUES(0), (0); +# Ensure that all changes before the server is killed are persisted. +set global innodb_flush_log_at_trx_commit=1; +START TRANSACTION; +INSERT INTO t32 VALUES(0), (0); +SELECT MAX(a) AS `Expect 4` FROM t32; +Expect 4 +4 +DELETE FROM t32 WHERE a >= 2; +ROLLBACK; +# Scenario 11: Test duplicate primary key/secondary key will not stop +# increasing the counter +CREATE TABLE t33 ( +a BIGINT NOT NULL PRIMARY KEY, +b BIGINT NOT NULL AUTO_INCREMENT, +KEY(b)) ENGINE = InnoDB; +INSERT INTO t33 VALUES(1, NULL); +INSERT INTO t33 VALUES(2, NULL); +INSERT INTO t33 VALUES(2, NULL); +ERROR 23000: Duplicate entry '2' for key 'PRIMARY' +INSERT INTO t33 VALUES(3, NULL); +SELECT MAX(b) AS `Expect 4` FROM t33; +Expect 4 +4 +TRUNCATE TABLE t33; +INSERT INTO t33 VALUES(1, NULL); +INSERT INTO t33 VALUES(2, NULL); +set global innodb_flush_log_at_trx_commit=1; +START TRANSACTION; +UPDATE t33 SET a = 10 WHERE a = 1; +INSERT INTO t33 VALUES(2, NULL); +ERROR 23000: Duplicate entry '2' for key 'PRIMARY' +COMMIT; # Kill and restart # This will not insert 0 INSERT INTO t31(a) VALUES(6), (0); @@ -1080,19 +936,6 @@ a b 0 6 300 7 SET SQL_MODE = 0; -# Scenario 10: Rollback would not rollback the counter -CREATE TABLE t32 ( -a BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; -INSERT INTO t32 VALUES(0), (0); -set global innodb_flush_log_at_trx_commit=1; -START TRANSACTION; -INSERT INTO t32 VALUES(0), (0); -SELECT MAX(a) AS `Expect 4` FROM t32; -Expect 4 -4 -DELETE FROM t32 WHERE a >= 2; -ROLLBACK; -# Kill and restart SELECT MAX(a) AS `Expect 2` FROM t32; Expect 2 2 @@ -1100,32 +943,8 @@ INSERT INTO t32 VALUES(0), (0); SELECT MAX(a) AS `Expect 6` FROM t32; Expect 6 6 -# Scenario 11: Test duplicate primary key/secondary key will not stop -# increasing the counter -CREATE TABLE t33 ( -a BIGINT NOT NULL PRIMARY KEY, -b BIGINT NOT NULL AUTO_INCREMENT, -KEY(b)) ENGINE = InnoDB; -INSERT INTO t33 VALUES(1, NULL); -INSERT INTO t33 VALUES(2, NULL); -INSERT INTO t33 VALUES(2, NULL); -ERROR 23000: Duplicate entry '2' for key 'PRIMARY' -INSERT INTO t33 VALUES(3, NULL); -SELECT MAX(b) AS `Expect 4` FROM t33; -Expect 4 -4 -TRUNCATE TABLE t33; -INSERT INTO t33 VALUES(1, NULL); -INSERT INTO t33 VALUES(2, NULL); -set global innodb_flush_log_at_trx_commit=1; -START TRANSACTION; -UPDATE t33 SET a = 10 WHERE a = 1; -INSERT INTO t33 VALUES(2, NULL); -ERROR 23000: Duplicate entry '2' for key 'PRIMARY' -COMMIT; -# Kill and restart INSERT INTO t33 VALUES(3, NULL); SELECT MAX(b) AS `Expect 4` FROM t33; Expect 4 4 -DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t30, t32, t33; +DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t30, t32, t33; diff --git a/mysql-test/suite/innodb/t/autoinc_persist.test b/mysql-test/suite/innodb/t/autoinc_persist.test index e4ea2db6bd2c7..9031f64317a9f 100644 --- a/mysql-test/suite/innodb/t/autoinc_persist.test +++ b/mysql-test/suite/innodb/t/autoinc_persist.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +# Restarting is not supported when testing the embedded server. --source include/not_embedded.inc --echo # @@ -84,45 +85,15 @@ INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); SELECT * FROM t11; -# Since autoinc counter is persisted by redo logs, we don't want to -# lose them on kill and restart, so to make the result after restart stable. -set global innodb_flush_log_at_trx_commit=1; - CREATE TABLE t12(a DOUBLE AUTO_INCREMENT KEY) ENGINE = InnoDB; INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); SELECT * FROM t12; -CREATE TABLE t13(a INT AUTO_INCREMENT PRIMARY KEY) ENGINE = InnoDB, -AUTO_INCREMENT = 1234; - --echo # Scenario 1: Normal restart, to test if the counters are persisted ---source include/restart_mysqld.inc - ---echo # We expect these results should be equal to above SELECTs -SELECT * FROM t1; -SELECT * FROM t2; -SELECT * FROM t3; -SELECT * FROM t4; -SELECT * FROM t5; -SELECT * FROM t6; -SELECT * FROM t7; -SELECT * FROM t8; -SELECT * FROM t9; -SELECT * FROM t10; -SELECT * FROM t11; -SELECT * FROM t12; - -SELECT * FROM t13; -SHOW CREATE TABLE t13; -INSERT INTO t13 VALUES(0); -SELECT a AS `Expect 1234` FROM t13; - --echo # Scenario 2: Delete some values, to test the counters should not be the --echo # one which is the largest in current table -set global innodb_flush_log_at_trx_commit=1; - DELETE FROM t1 WHERE a > 30; SELECT MAX(a) AS `Expect 30` FROM t1; DELETE FROM t3 WHERE a > 2000; @@ -134,8 +105,16 @@ SELECT MAX(a) AS `Expect 100000000` FROM t7; DELETE FROM t9 WHERE a > 100000000000; SELECT MAX(a) AS `Expect 100000000000` FROM t9; +CREATE TABLE t13(a INT AUTO_INCREMENT PRIMARY KEY) ENGINE = InnoDB, +AUTO_INCREMENT = 1234; + --source include/restart_mysqld.inc +SHOW CREATE TABLE t13; +INSERT INTO t13 VALUES(0); +SELECT a AS `Expect 1234` FROM t13; +DROP TABLE t13; + INSERT INTO t1 VALUES(0), (0); SELECT MAX(a) AS `Expect 33` FROM t1; INSERT INTO t3 VALUES(0), (0); @@ -222,8 +201,6 @@ DELETE FROM t7 WHERE a > 100000105; INSERT INTO t7 VALUES(0); SELECT MAX(a) AS `Expect 100000109` FROM t7; -set global innodb_flush_log_at_trx_commit=1; - INSERT INTO t9 VALUES(100), (200); # Updating to bigger value will update the auto-increment counter UPDATE t9 SET a = 100000000105 WHERE a = 5; @@ -268,6 +245,7 @@ DELETE FROM t5 WHERE a = 1100200; INSERT INTO t7 VALUES(100000200); DELETE FROM t7 WHERE a = 100000200; +--echo # Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; INSERT INTO t9 VALUES(100000000200); @@ -313,6 +291,7 @@ SELECT * FROM t7; INSERT INTO t9 VALUES(0), (0); SELECT * FROM t9; +--echo # Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; TRUNCATE TABLE t1; @@ -321,6 +300,12 @@ TRUNCATE TABLE t5; TRUNCATE TABLE t7; TRUNCATE TABLE t9; +--echo # Scenario 7: Test explicit rename table won't change the counter + +RENAME TABLE t9 to t19; +INSERT INTO t19 VALUES(0), (0); +SELECT * FROM t19; + --source include/kill_and_restart_mysqld.inc INSERT INTO t1 VALUES(0), (0); @@ -335,20 +320,10 @@ SELECT * FROM t5; INSERT INTO t7 VALUES(0), (0); SELECT * FROM t7; -INSERT INTO t9 VALUES(0), (0); -SELECT * FROM t9; - ---echo # Scenario 7: Test explicit rename table won't change the counter - -set global innodb_flush_log_at_trx_commit=1; - -RENAME TABLE t9 to t19; INSERT INTO t19 VALUES(0), (0); SELECT * FROM t19; DELETE FROM t19 WHERE a = 4; ---source include/kill_and_restart_mysqld.inc - RENAME TABLE t19 to t9; INSERT INTO t9 VALUES(0), (0); SELECT * FROM t9; @@ -413,14 +388,12 @@ ALGORITHM=INPLACE; INSERT INTO t3 VALUES(0); SELECT MAX(a) AS `Expect 120` FROM t3; -INSERT INTO mdev6076a SET b=0; +INSERT INTO mdev6076a SET b=NULL; SELECT * FROM mdev6076a; -INSERT INTO mdev6076b SET b=0; +INSERT INTO mdev6076b SET b=NULL; SELECT * FROM mdev6076b; DROP TABLE mdev6076a, mdev6076b; -set global innodb_flush_log_at_trx_commit=1; - INSERT INTO t3 VALUES(0), (0), (200), (210); --echo # Test the different algorithms in ALTER TABLE @@ -445,6 +418,7 @@ SELECT * FROM t30 ORDER BY b; ALTER TABLE t30 MODIFY b MEDIUMINT; SELECT * FROM t30 ORDER BY b; +--echo # Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; CREATE TABLE t31 (a INT) ENGINE = InnoDB; @@ -455,25 +429,6 @@ INSERT INTO t31 VALUES(3, 0), (4, NULL), (5, NULL); INSERT INTO t31 VALUES(6, 0); SELECT * FROM t31; ---source include/kill_and_restart_mysqld.inc - ---echo # This will not insert 0 -INSERT INTO t31(a) VALUES(6), (0); -SELECT * FROM t31; -DROP TABLE t31; - -set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; - -DELETE FROM t30 WHERE a = 0; -UPDATE t30 set a = 0 where b = 5; -SELECT * FROM t30 ORDER BY b; -DELETE FROM t30 WHERE a = 0; - -UPDATE t30 SET a = NULL WHERE b = 6; -UPDATE t30 SET a = 300 WHERE b = 7; - -SELECT * FROM t30 ORDER BY b; - SET SQL_MODE = 0; --echo # Scenario 10: Rollback would not rollback the counter @@ -482,6 +437,7 @@ a BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; INSERT INTO t32 VALUES(0), (0); +--echo # Ensure that all changes before the server is killed are persisted. set global innodb_flush_log_at_trx_commit=1; START TRANSACTION; @@ -490,12 +446,6 @@ SELECT MAX(a) AS `Expect 4` FROM t32; DELETE FROM t32 WHERE a >= 2; ROLLBACK; ---source include/kill_and_restart_mysqld.inc - -SELECT MAX(a) AS `Expect 2` FROM t32; -INSERT INTO t32 VALUES(0), (0); -SELECT MAX(a) AS `Expect 6` FROM t32; - --echo # Scenario 11: Test duplicate primary key/secondary key will not stop --echo # increasing the counter @@ -527,7 +477,30 @@ COMMIT; --source include/kill_and_restart_mysqld.inc +--echo # This will not insert 0 +INSERT INTO t31(a) VALUES(6), (0); +SELECT * FROM t31; +DROP TABLE t31; + +set SQL_MODE = NO_AUTO_VALUE_ON_ZERO; + +DELETE FROM t30 WHERE a = 0; +UPDATE t30 set a = 0 where b = 5; +SELECT * FROM t30 ORDER BY b; +DELETE FROM t30 WHERE a = 0; + +UPDATE t30 SET a = NULL WHERE b = 6; +UPDATE t30 SET a = 300 WHERE b = 7; + +SELECT * FROM t30 ORDER BY b; + +SET SQL_MODE = 0; + +SELECT MAX(a) AS `Expect 2` FROM t32; +INSERT INTO t32 VALUES(0), (0); +SELECT MAX(a) AS `Expect 6` FROM t32; + INSERT INTO t33 VALUES(3, NULL); SELECT MAX(b) AS `Expect 4` FROM t33; -DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t30, t32, t33; +DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t30, t32, t33; From c64edc6b83f1cf1ed21d93bdba5d13dd2afd8dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 15 Dec 2016 18:51:41 +0200 Subject: [PATCH 275/295] MDEV-6076: Preserve PAGE_ROOT_AUTO_INC when emptying pages. Thanks to Zhangyuan from Alibaba for pointing out this bug. btr_page_empty(): When a clustered index root page is emptied, preserve PAGE_ROOT_AUTO_INC. This would occur during a page split. page_create_empty(): Preserve PAGE_ROOT_AUTO_INC when a clustered index root page becomes empty. Use a faster method for writing the field. page_zip_copy_recs(): Reset PAGE_MAX_TRX_ID when copying clustered index pages. We must clear the field when the root page was a leaf page and it is being split, so that PAGE_MAX_TRX_ID will continue to be 0 in clustered index non-root pages. page_create_zip(): Add debug assertions for validating PAGE_MAX_TRX_ID and PAGE_ROOT_AUTO_INC. --- .../suite/innodb/r/autoinc_persist.result | 16 +++++++++- .../suite/innodb/t/autoinc_persist.test | 21 +++++++++++++- storage/innobase/btr/btr0btr.cc | 13 ++++++++- storage/innobase/page/page0page.cc | 29 ++++++++++++++++--- storage/innobase/page/page0zip.cc | 20 ++++++++----- 5 files changed, 85 insertions(+), 14 deletions(-) diff --git a/mysql-test/suite/innodb/r/autoinc_persist.result b/mysql-test/suite/innodb/r/autoinc_persist.result index 0494a256f934e..814f3d32e60cd 100644 --- a/mysql-test/suite/innodb/r/autoinc_persist.result +++ b/mysql-test/suite/innodb/r/autoinc_persist.result @@ -629,6 +629,16 @@ ERROR 0A000: LOCK=NONE is not supported. Reason: Adding an auto-increment column ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, ALGORITHM=INPLACE; ALTER TABLE mdev6076b ADD COLUMN a SERIAL FIRST, AUTO_INCREMENT=100, ALGORITHM=INPLACE; +# MDEV-6076: Test root page split and page_create_empty() +CREATE TABLE mdev6076empty (b SERIAL, pad CHAR(255) NOT NULL DEFAULT '') +ENGINE=InnoDB; +BEGIN; +# Insert records in descending order of AUTO_INCREMENT, +# causing a page split on the very last insert. +# Without the fix in btr_page_empty() this would lose the counter value. +# Without the fix in page_create_empty() the counter value would be lost +# when ROLLBACK deletes the last row. +ROLLBACK; # Kill and restart INSERT INTO t3 VALUES(0); SELECT MAX(a) AS `Expect 120` FROM t3; @@ -646,7 +656,11 @@ a b 100 2 101 1 102 NULL -DROP TABLE mdev6076a, mdev6076b; +INSERT INTO mdev6076empty SET b=NULL; +SELECT * FROM mdev6076empty; +b pad +56 +DROP TABLE mdev6076a, mdev6076b, mdev6076empty; INSERT INTO t3 VALUES(0), (0), (200), (210); # Test the different algorithms in ALTER TABLE CREATE TABLE t_inplace LIKE t3; diff --git a/mysql-test/suite/innodb/t/autoinc_persist.test b/mysql-test/suite/innodb/t/autoinc_persist.test index 9031f64317a9f..45a96f85fe114 100644 --- a/mysql-test/suite/innodb/t/autoinc_persist.test +++ b/mysql-test/suite/innodb/t/autoinc_persist.test @@ -382,6 +382,23 @@ ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, LOCK=NONE; ALTER TABLE mdev6076a ADD COLUMN a SERIAL FIRST, ALGORITHM=INPLACE; ALTER TABLE mdev6076b ADD COLUMN a SERIAL FIRST, AUTO_INCREMENT=100, ALGORITHM=INPLACE; +--echo # MDEV-6076: Test root page split and page_create_empty() +CREATE TABLE mdev6076empty (b SERIAL, pad CHAR(255) NOT NULL DEFAULT '') +ENGINE=InnoDB; +BEGIN; +--echo # Insert records in descending order of AUTO_INCREMENT, +--echo # causing a page split on the very last insert. +--echo # Without the fix in btr_page_empty() this would lose the counter value. +--echo # Without the fix in page_create_empty() the counter value would be lost +--echo # when ROLLBACK deletes the last row. +--disable_query_log +let $i= 55; +while ($i) { + eval INSERT INTO mdev6076empty SET b=$i; + dec $i; +} +--enable_query_log +ROLLBACK; --source include/kill_and_restart_mysqld.inc @@ -392,7 +409,9 @@ INSERT INTO mdev6076a SET b=NULL; SELECT * FROM mdev6076a; INSERT INTO mdev6076b SET b=NULL; SELECT * FROM mdev6076b; -DROP TABLE mdev6076a, mdev6076b; +INSERT INTO mdev6076empty SET b=NULL; +SELECT * FROM mdev6076empty; +DROP TABLE mdev6076a, mdev6076b, mdev6076empty; INSERT INTO t3 VALUES(0), (0), (200), (210); diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 9b99da2a684c0..1171a40ffd77a 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -1867,12 +1867,23 @@ btr_page_empty( /* Recreate the page: note that global data on page (possible segment headers, next page-field, etc.) is preserved intact */ + /* Preserve PAGE_ROOT_AUTO_INC when creating a clustered index + root page. */ + const ib_uint64_t autoinc + = dict_index_is_clust(index) && page_is_root(page) + ? page_get_autoinc(page) + : 0; + if (page_zip) { - page_create_zip(block, index, level, 0, NULL, mtr); + page_create_zip(block, index, level, autoinc, NULL, mtr); } else { page_create(block, mtr, dict_table_is_comp(index->table), dict_index_is_spatial(index)); btr_page_set_level(page, NULL, level, mtr); + if (autoinc) { + mlog_write_ull(PAGE_HEADER + PAGE_MAX_TRX_ID + page, + autoinc, mtr); + } } } diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index d207371b21321..9bcc28f598070 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -488,6 +488,22 @@ page_create_zip( is_spatial = index ? dict_index_is_spatial(index) : page_comp_info->type & DICT_SPATIAL; + /* PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC are always 0 for + temporary tables. */ + ut_ad(!dict_table_is_temporary(index->table) || max_trx_id == 0); + /* In secondary indexes and the change buffer, PAGE_MAX_TRX_ID + must be zero on non-leaf pages. max_trx_id can be 0 when the + index consists of an empty root (leaf) page. */ + ut_ad(max_trx_id == 0 + || level == 0 + || !dict_index_is_sec_or_ibuf(index) + || dict_table_is_temporary(index->table)); + /* In the clustered index, PAGE_ROOT_AUTOINC or + PAGE_MAX_TRX_ID must be 0 on other pages than the root. */ + ut_ad(level == 0 || max_trx_id == 0 + || !dict_index_is_sec_or_ibuf(index) + || dict_table_is_temporary(index->table)); + page = page_create_low(block, TRUE, is_spatial); mach_write_to_2(PAGE_HEADER + PAGE_LEVEL + page, level); mach_write_to_8(PAGE_HEADER + PAGE_MAX_TRX_ID + page, max_trx_id); @@ -521,8 +537,8 @@ page_create_empty( dict_index_t* index, /*!< in: the index of the page */ mtr_t* mtr) /*!< in/out: mini-transaction */ { - trx_id_t max_trx_id = 0; - const page_t* page = buf_block_get_frame(block); + trx_id_t max_trx_id; + page_t* page = buf_block_get_frame(block); page_zip_des_t* page_zip= buf_block_get_page_zip(block); ut_ad(fil_page_index_page_check(page)); @@ -536,6 +552,11 @@ page_create_empty( && page_is_leaf(page)) { max_trx_id = page_get_max_trx_id(page); ut_ad(max_trx_id); + } else if (page_is_root(page)) { + /* Preserve PAGE_ROOT_AUTO_INC. */ + max_trx_id = page_get_max_trx_id(page); + } else { + max_trx_id = 0; } if (page_zip) { @@ -547,8 +568,8 @@ page_create_empty( dict_index_is_spatial(index)); if (max_trx_id) { - page_update_max_trx_id( - block, page_zip, max_trx_id, mtr); + mlog_write_ull(PAGE_HEADER + PAGE_MAX_TRX_ID + page, + max_trx_id, mtr); } } } diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 621540afcef97..798aeeed74f0a 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4821,13 +4821,6 @@ page_zip_copy_recs( ut_a(dict_index_is_clust(index)); } - /* The PAGE_MAX_TRX_ID must be set on leaf pages of secondary - indexes. It does not matter on other pages. */ - ut_a(dict_index_is_clust(index) - || dict_table_is_temporary(index->table) - || !page_is_leaf(src) - || page_get_max_trx_id(src)); - UNIV_MEM_ASSERT_W(page, UNIV_PAGE_SIZE); UNIV_MEM_ASSERT_W(page_zip->data, page_zip_get_size(page_zip)); UNIV_MEM_ASSERT_RW(src, UNIV_PAGE_SIZE); @@ -4849,6 +4842,19 @@ page_zip_copy_recs( memcpy(PAGE_DATA + page_zip->data, PAGE_DATA + src_zip->data, page_zip_get_size(page_zip) - PAGE_DATA); + if (dict_index_is_clust(index)) { + /* Reset the PAGE_ROOT_AUTO_INC field when copying + from a root page. */ + memset(PAGE_HEADER + PAGE_ROOT_AUTO_INC + page, 0, 8); + memset(PAGE_HEADER + PAGE_ROOT_AUTO_INC + page_zip->data, + 0, 8); + } else { + /* The PAGE_MAX_TRX_ID must be nonzero on leaf pages + of secondary indexes, and 0 on others. */ + ut_ad(dict_table_is_temporary(index->table) + || !page_is_leaf(src) == !page_get_max_trx_id(src)); + } + /* Copy all fields of src_zip to page_zip, except the pointer to the compressed data page. */ { From 6f4f9f2843d452d29883fe8d136e938980e6b376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 16 Dec 2016 17:15:40 +0200 Subject: [PATCH 276/295] MDEV-6076 Adjust a test result. This test and result had been modified in WL#6204, but I missed it, because the test had been renamed from innodb_fts.innodb_fts_plugin to innodb_fts.plugin. --- mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result b/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result index c7c86290f3c78..f057db1d2849e 100644 --- a/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result +++ b/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result @@ -139,8 +139,8 @@ INSERT INTO articles (title, body) VALUES SELECT * FROM articles WHERE MATCH(title, body) AGAINST('Tricks'); id title body -4 1001 MySQL Tricks How to use full-text search engine -5 Go MySQL Tricks How to use full text search engine +9 1001 MySQL Tricks How to use full-text search engine +10 Go MySQL Tricks How to use full text search engine SELECT COUNT(*) FROM articles; COUNT(*) 5 From 9f863a15b037c0ddffc2d12c1f7d841466f9f345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 19 Dec 2016 15:57:41 +0200 Subject: [PATCH 277/295] MDEV-11602 InnoDB leaks foreign key metadata on DDL operations Essentially revert MDEV-6759, which addressed a double free of memory by removing the freeing altogether, introducing the memory leaks. No double free was observed when running the test suite -DWITH_ASAN. Replace some mem_heap_free(foreign->heap) with dict_foreign_free(foreign) so that the calls can be located and instrumented more easily when needed. --- storage/innobase/dict/dict0dict.cc | 8 ++++---- storage/innobase/dict/dict0load.cc | 2 +- storage/xtradb/dict/dict0dict.cc | 8 ++++---- storage/xtradb/dict/dict0load.cc | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index f75d584e70d28..2e40946224ee9 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1614,6 +1614,7 @@ struct dict_foreign_remove_partial if (table != NULL) { table->referenced_set.erase(foreign); } + dict_foreign_free(foreign); } }; @@ -3535,8 +3536,7 @@ dict_foreign_add_to_cache( } if (for_in_cache) { - /* Free the foreign object */ - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } else { for_in_cache = foreign; } @@ -3564,7 +3564,7 @@ dict_foreign_add_to_cache( " the ones in table."); if (for_in_cache == foreign) { - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); @@ -3620,7 +3620,7 @@ dict_foreign_add_to_cache( be one */ } - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index e68bbe7d10a7a..77a31e5de63c2 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -489,7 +489,7 @@ dict_process_sys_foreign_rec( } /* This recieves a dict_foreign_t* that points to a stack variable. - So mem_heap_free(foreign->heap) is not used as elsewhere. + So dict_foreign_free(foreign) is not used as elsewhere. Since the heap used here is freed elsewhere, foreign->heap is not assigned. */ foreign->id = mem_heap_strdupl(heap, (const char*) field, len); diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 5339ecfaed613..3845138c8bf60 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -1614,6 +1614,7 @@ struct dict_foreign_remove_partial if (table != NULL) { table->referenced_set.erase(foreign); } + dict_foreign_free(foreign); } }; @@ -3539,8 +3540,7 @@ dict_foreign_add_to_cache( } if (for_in_cache) { - /* Free the foreign object */ - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } else { for_in_cache = foreign; } @@ -3564,7 +3564,7 @@ dict_foreign_add_to_cache( " the ones in table."); if (for_in_cache == foreign) { - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); @@ -3620,7 +3620,7 @@ dict_foreign_add_to_cache( be one */ } - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index ca7de72c9b907..9ea4f4d873aac 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -491,7 +491,7 @@ dict_process_sys_foreign_rec( } /* This recieves a dict_foreign_t* that points to a stack variable. - So mem_heap_free(foreign->heap) is not used as elsewhere. + So dict_foreign_free(foreign) is not used as elsewhere. Since the heap used here is freed elsewhere, foreign->heap is not assigned. */ foreign->id = mem_heap_strdupl(heap, (const char*) field, len); From 195241e125f58c9cfcb0fd28f5f2bf78e97fe7d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Dec 2016 15:03:56 +0200 Subject: [PATCH 278/295] Port the test innodb.doublewrite from MySQL 5.7. --- mysql-test/include/kill_mysqld.inc | 7 + .../innodb/include/no_checkpoint_end.inc | 35 ++ .../innodb/include/no_checkpoint_start.inc | 5 + mysql-test/suite/innodb/r/doublewrite.result | 241 ++++++++++++ mysql-test/suite/innodb/t/doublewrite.test | 368 ++++++++++++++++++ 5 files changed, 656 insertions(+) create mode 100644 mysql-test/include/kill_mysqld.inc create mode 100644 mysql-test/suite/innodb/include/no_checkpoint_end.inc create mode 100644 mysql-test/suite/innodb/include/no_checkpoint_start.inc create mode 100644 mysql-test/suite/innodb/r/doublewrite.result create mode 100644 mysql-test/suite/innodb/t/doublewrite.test diff --git a/mysql-test/include/kill_mysqld.inc b/mysql-test/include/kill_mysqld.inc new file mode 100644 index 0000000000000..86ee048a0f18c --- /dev/null +++ b/mysql-test/include/kill_mysqld.inc @@ -0,0 +1,7 @@ +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect + +--echo # Kill the server +--exec echo "wait" > $_expect_file_name +--shutdown_server 0 +--source include/wait_until_disconnected.inc diff --git a/mysql-test/suite/innodb/include/no_checkpoint_end.inc b/mysql-test/suite/innodb/include/no_checkpoint_end.inc new file mode 100644 index 0000000000000..7ca81f8ade033 --- /dev/null +++ b/mysql-test/suite/innodb/include/no_checkpoint_end.inc @@ -0,0 +1,35 @@ +# Check that the latest checkpoint in the redo log files +# is not newer than the checkpoint sampled by no_checkpoint_start.inc + +--source include/kill_mysqld.inc + +perl; +my $cp = $ENV{CHECKPOINT_LSN}; +$cp =~ s/^InnoDB\t\t//; +my $log = "$ENV{MYSQLD_DATADIR}ib_logfile0"; +open(LOG, "<$log") || die "Unable to open $log"; +seek(LOG, 512, 0) || die "Unable to seek $log"; +die unless read(LOG, $_, 16) == 16; +my ($no1hi,$no1lo,$cp1hi,$cp1lo) = unpack("N*", $_); +seek(LOG, 3 * 512, 0) || die "Unable to seek $log"; +die unless read(LOG, $_, 16) == 16; +my ($no2hi,$no2lo,$cp2hi,$cp2lo) = unpack("N*", $_); +close(LOG); + +my $cp1 = $cp1hi << 32 | $cp1lo; +my $cp2 = $cp2hi << 32 | $cp2lo; + +open(OUT, ">$ENV{MYSQLTEST_VARDIR}/log/check.txt") || die; + +if ($cp1 > $cp || $cp2 > $cp) { + print OUT "--source include/start_mysqld.inc\n"; + print OUT "$ENV{CLEANUP_IF_CHECKPOINT}\n"; + print OUT "--skip Extra checkpoint 1 after $cp"; + print OUT " ($no1hi:$no1lo=$cp1,$no2hi:$no2lo=$cp2)\n"; +} + +close(OUT); +EOF + +--source $MYSQLTEST_VARDIR/log/check.txt +--remove_file $MYSQLTEST_VARDIR/log/check.txt diff --git a/mysql-test/suite/innodb/include/no_checkpoint_start.inc b/mysql-test/suite/innodb/include/no_checkpoint_start.inc new file mode 100644 index 0000000000000..a903fee67d425 --- /dev/null +++ b/mysql-test/suite/innodb/include/no_checkpoint_start.inc @@ -0,0 +1,5 @@ +# Preparation for using no_checkpoint_end.inc + +let MYSQLD_DATADIR= `select @@datadir`; +--replace_regex /.*Last checkpoint at[ ]*([0-9]+).*/\1/ +let CHECKPOINT_LSN=`SHOW ENGINE INNODB STATUS`; diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result new file mode 100644 index 0000000000000..aa96a5f2d9321 --- /dev/null +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -0,0 +1,241 @@ +# +# Bug #17335427 INNODB CAN NOT USE THE DOUBLEWRITE BUFFER PROPERLY +# Bug #18144349 INNODB CANNOT USE THE DOUBLEWRITE BUFFER FOR THE FIRST +# PAGE OF SYSTEM TABLESPACE +# +SET GLOBAL innodb_fast_shutdown = 0; +show variables like 'innodb_doublewrite'; +Variable_name Value +innodb_doublewrite ON +show variables like 'innodb_fil_make_page_dirty_debug'; +Variable_name Value +innodb_fil_make_page_dirty_debug 0 +show variables like 'innodb_saved_page_number_debug'; +Variable_name Value +innodb_saved_page_number_debug 0 +create table t1 (f1 int primary key, f2 blob) engine=innodb; +start transaction; +insert into t1 values(1, repeat('#',12)); +insert into t1 values(2, repeat('+',12)); +insert into t1 values(3, repeat('/',12)); +insert into t1 values(4, repeat('-',12)); +insert into t1 values(5, repeat('.',12)); +commit work; +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if first page of user +# tablespace is full of zeroes. +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; +# Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; +begin; +insert into t1 values (6, repeat('%', 12)); +# Make the first page dirty for table t1 +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = @space_id; +# Ensure that dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Make the first page (page_no=0) of the user tablespace +# full of zeroes. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if first page of user +# tablespace is corrupted. +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; +# Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; +begin; +insert into t1 values (6, repeat('%', 12)); +# Make the first page dirty for table t1 +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = @space_id; +# Ensure that dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Corrupt the first page (page_no=0) of the user tablespace. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if 2nd page of user +# tablespace is full of zeroes. +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; +# Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; +begin; +insert into t1 values (6, repeat('%', 400)); +# Make the 2nd page dirty for table t1 +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = @space_id; +# Ensure that dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Make the 2nd page (page_no=1) of the tablespace all zeroes. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if 2nd page of user +# tablespace is corrupted. +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; +# Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; +begin; +insert into t1 values (6, repeat('%', 400)); +# Make the 2nd page dirty for table t1 +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = @space_id; +# Ensure that the dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Corrupt the 2nd page (page_no=1) of the user tablespace. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if first page of +# system tablespace is full of zeroes. +begin; +insert into t1 values (6, repeat('%', 400)); +# Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; +# Make the first page dirty for system tablespace +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = 0; +# Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Make the first page (page_no=0) of the system tablespace +# all zeroes. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if first page of +# system tablespace is corrupted. +begin; +insert into t1 values (6, repeat('%', 400)); +# Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; +# Make the first page dirty for system tablespace +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = 0; +# Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Corrupt the first page (page_no=0) of the system tablespace. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if 2nd page of +# system tablespace is full of zeroes. +begin; +insert into t1 values (6, repeat('%', 400)); +# Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; +# Make the second page dirty for system tablespace +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = 0; +# Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Make the 2nd page (page_no=1) of the system tablespace +# all zeroes. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +# Test Begin: Test if recovery works if 2nd page of +# system tablespace is corrupted. +begin; +insert into t1 values (6, repeat('%', 400)); +# Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; +# Make the second page dirty for system tablespace +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = 0; +# Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; +# Kill the server +# Make the 2nd page (page_no=1) of the system tablespace +# all zeroes. +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ +# Test End +# --------------------------------------------------------------- +drop table t1; diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test new file mode 100644 index 0000000000000..5bd4551aa1edb --- /dev/null +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -0,0 +1,368 @@ +--echo # +--echo # Bug #17335427 INNODB CAN NOT USE THE DOUBLEWRITE BUFFER PROPERLY +--echo # Bug #18144349 INNODB CANNOT USE THE DOUBLEWRITE BUFFER FOR THE FIRST +--echo # PAGE OF SYSTEM TABLESPACE +--echo # + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/not_embedded.inc + +# Slow shutdown and restart to make sure ibuf merge is finished +SET GLOBAL innodb_fast_shutdown = 0; +--source include/restart_mysqld.inc + +--disable_query_log +call mtr.add_suppression("InnoDB: Database page [0-9]+:1 contained only zeroes."); +call mtr.add_suppression("Header page consists of zero bytes"); +call mtr.add_suppression("Checksum mismatch in tablespace.*table test/t1"); +call mtr.add_suppression("but the innodb_page_size start-up parameter is"); +call mtr.add_suppression("Database page corruption"); +--enable_query_log + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; + +show variables like 'innodb_doublewrite'; +show variables like 'innodb_fil_make_page_dirty_debug'; +show variables like 'innodb_saved_page_number_debug'; + +create table t1 (f1 int primary key, f2 blob) engine=innodb; + +start transaction; +insert into t1 values(1, repeat('#',12)); +insert into t1 values(2, repeat('+',12)); +insert into t1 values(3, repeat('/',12)); +insert into t1 values(4, repeat('-',12)); +insert into t1 values(5, repeat('.',12)); +commit work; + +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if first page of user +--echo # tablespace is full of zeroes. + +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; + +--echo # Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; + +begin; +insert into t1 values (6, repeat('%', 12)); + +--source ../include/no_checkpoint_start.inc + +--echo # Make the first page dirty for table t1 +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = @space_id; + +--echo # Ensure that dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; + +--let CLEANUP_IF_CHECKPOINT=drop table t1; +--source ../include/no_checkpoint_end.inc + +--echo # Make the first page (page_no=0) of the user tablespace +--echo # full of zeroes. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if first page of user +--echo # tablespace is corrupted. + +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; + +--echo # Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; + +begin; +insert into t1 values (6, repeat('%', 12)); + +--source ../include/no_checkpoint_start.inc + +--echo # Make the first page dirty for table t1 +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = @space_id; + +--echo # Ensure that dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; + +--source include/no_checkpoint_end.inc + +--echo # Corrupt the first page (page_no=0) of the user tablespace. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if 2nd page of user +--echo # tablespace is full of zeroes. + +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; + +--echo # Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; + +begin; +insert into t1 values (6, repeat('%', 400)); + +--source ../include/no_checkpoint_start.inc + +--echo # Make the 2nd page dirty for table t1 +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = @space_id; + +--echo # Ensure that dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; + +--source include/no_checkpoint_end.inc + +--echo # Make the 2nd page (page_no=1) of the tablespace all zeroes. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET); +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if 2nd page of user +--echo # tablespace is corrupted. + +select space from information_schema.innodb_sys_tables +where name = 'test/t1' into @space_id; + +--echo # Ensure that dirty pages of table t1 is flushed. +flush tables t1 for export; +unlock tables; + +begin; +insert into t1 values (6, repeat('%', 400)); + +--source ../include/no_checkpoint_start.inc + +--echo # Make the 2nd page dirty for table t1 +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = @space_id; + +--echo # Ensure that the dirty pages of table t1 are flushed. +set global innodb_buf_flush_list_now = 1; + +--source include/no_checkpoint_end.inc + +--echo # Corrupt the 2nd page (page_no=1) of the user tablespace. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET); +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if first page of +--echo # system tablespace is full of zeroes. + +begin; +insert into t1 values (6, repeat('%', 400)); + +--echo # Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; + +--echo # Make the first page dirty for system tablespace +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = 0; + +--echo # Ensure that the dirty page of system tablespace is also flushed. +# We do this after the transaction starts and all dirty pages have been flushed +# already. So flushing of this specified dirty page will surely keep the +# copy in doublewrite buffer, and no more writes to doublewrite buffer would +# overwrite the copy. Thus, we can safely modify the original page when server +# is down. So do the following testings. +set global innodb_buf_flush_list_now = 1; + +--source include/kill_mysqld.inc + +--echo # Make the first page (page_no=0) of the system tablespace +--echo # all zeroes. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if first page of +--echo # system tablespace is corrupted. + +begin; +insert into t1 values (6, repeat('%', 400)); + +--echo # Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; + +--echo # Make the first page dirty for system tablespace +set global innodb_saved_page_number_debug = 0; +set global innodb_fil_make_page_dirty_debug = 0; + +--echo # Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; + +--source include/kill_mysqld.inc + +--echo # Corrupt the first page (page_no=0) of the system tablespace. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if 2nd page of +--echo # system tablespace is full of zeroes. + +begin; +insert into t1 values (6, repeat('%', 400)); + +--echo # Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; + +--echo # Make the second page dirty for system tablespace +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = 0; + +--echo # Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; + +--source include/kill_mysqld.inc + +--echo # Make the 2nd page (page_no=1) of the system tablespace +--echo # all zeroes. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET); +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- +--echo # Test Begin: Test if recovery works if 2nd page of +--echo # system tablespace is corrupted. + +begin; +insert into t1 values (6, repeat('%', 400)); + +--echo # Ensure that all dirty pages in the system are flushed. +set global innodb_buf_flush_list_now = 1; + +--echo # Make the second page dirty for system tablespace +set global innodb_saved_page_number_debug = 1; +set global innodb_fil_make_page_dirty_debug = 0; + +--echo # Ensure that the dirty page of system tablespace is also flushed. +set global innodb_buf_flush_list_now = 1; + +--source include/kill_mysqld.inc + +--echo # Make the 2nd page (page_no=1) of the system tablespace +--echo # all zeroes. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1"; +open(FILE, "+<", $fname) or die; +FILE->autoflush(1); +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET); +print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2); +close FILE; +EOF + +--source include/start_mysqld.inc + +check table t1; +select f1, f2 from t1; + +--echo # Test End +--echo # --------------------------------------------------------------- + +drop table t1; From 75ab65aecee1db8ea402a834935f433021992cb3 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 20 Dec 2016 15:31:02 -0500 Subject: [PATCH 279/295] Fix failing galera tests. --- .../suite/galera/r/galera_split_brain.result | 1 - .../r/galera_var_cluster_address.result | 35 +++----- .../suite/galera/t/galera_split_brain.test | 5 +- .../galera/t/galera_var_cluster_address.test | 79 ++++++------------- 4 files changed, 34 insertions(+), 86 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_split_brain.result b/mysql-test/suite/galera/r/galera_split_brain.result index 615615040feb7..6473f95735b94 100644 --- a/mysql-test/suite/galera/r/galera_split_brain.result +++ b/mysql-test/suite/galera/r/galera_split_brain.result @@ -2,4 +2,3 @@ call mtr.add_suppression("WSREP: TO isolation failed for: "); Killing server ... CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; ERROR 40001: Deadlock found when trying to get lock; try restarting transaction -SET GLOBAL wsrep_cluster_address = ''; diff --git a/mysql-test/suite/galera/r/galera_var_cluster_address.result b/mysql-test/suite/galera/r/galera_var_cluster_address.result index f8bd869f8fe9e..55d216eda006d 100644 --- a/mysql-test/suite/galera/r/galera_var_cluster_address.result +++ b/mysql-test/suite/galera/r/galera_var_cluster_address.result @@ -1,6 +1,6 @@ SET GLOBAL wsrep_cluster_address = 'foo://'; SET SESSION wsrep_sync_wait=0; -SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS; +SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS; ERROR 08S01: WSREP has not yet prepared node for application use SHOW STATUS LIKE 'wsrep_ready'; Variable_name Value @@ -20,32 +20,19 @@ VARIABLE_VALUE = 1 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; VARIABLE_VALUE = 'Primary' 1 -SET GLOBAL wsrep_cluster_address = @@wsrep_cluster_address; -SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; -VARIABLE_VALUE = 'Primary' -1 -SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; -VARIABLE_VALUE = 2 -1 -SET GLOBAL wsrep_cluster_address = 'gcomm://192.0.2.1'; -SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS; -ERROR 08S01: WSREP has not yet prepared node for application use -SHOW STATUS LIKE 'wsrep_ready'; -Variable_name Value -wsrep_ready OFF -SHOW STATUS LIKE 'wsrep_cluster_status'; -Variable_name Value -wsrep_cluster_status non-Primary -SHOW STATUS LIKE 'wsrep_local_state'; -Variable_name Value -wsrep_local_state 0 -SHOW STATUS LIKE 'wsrep_local_state_comment'; -Variable_name Value -wsrep_local_state_comment Initialized -SET GLOBAL wsrep_cluster_address = @@wsrep_cluster_address; SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; VARIABLE_VALUE = 'Primary' 1 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; VARIABLE_VALUE = 2 1 +CALL mtr.add_suppression("Backend not supported: foo"); +CALL mtr.add_suppression("Failed to initialize backend using 'foo"); +CALL mtr.add_suppression("Failed to open channel 'my_wsrep_cluster' at 'foo"); +CALL mtr.add_suppression("gcs connect failed: Socket type not supported"); +CALL mtr.add_suppression("wsrep::connect\\(\\) failed: 7"); +CALL mtr.add_suppression("gcs_caused\\(\\) returned -103 \\(Software caused connection abort\\)"); +CALL mtr.add_suppression("failed to open gcomm backend connection: 110: failed to reach primary view: 110"); +CALL mtr.add_suppression("Failed to open backend connection: -110 \\(Connection timed out\\)"); +CALL mtr.add_suppression("gcs connect failed: Connection timed out"); +CALL mtr.add_suppression("WSREP: wsrep::connect\\(foo://\\) failed: 7"); diff --git a/mysql-test/suite/galera/t/galera_split_brain.test b/mysql-test/suite/galera/t/galera_split_brain.test index 22f6370241ce6..a85a2ad9b8deb 100644 --- a/mysql-test/suite/galera/t/galera_split_brain.test +++ b/mysql-test/suite/galera/t/galera_split_brain.test @@ -25,11 +25,8 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; # Reset the master and restart the slave so that post-test checks can run -SET GLOBAL wsrep_cluster_address = ''; ---disable_query_log ---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig'; ---enable_query_log +--connection node_2 --source include/start_mysqld.inc --sleep 5 --source include/wait_until_connected_again.inc diff --git a/mysql-test/suite/galera/t/galera_var_cluster_address.test b/mysql-test/suite/galera/t/galera_var_cluster_address.test index dfd84002dd65d..4e5d138ae0a13 100644 --- a/mysql-test/suite/galera/t/galera_var_cluster_address.test +++ b/mysql-test/suite/galera/t/galera_var_cluster_address.test @@ -6,17 +6,15 @@ --source include/have_innodb.inc # Save original auto_increment_offset values. ---connection node_1 -let $auto_increment_offset_node_1 = `SELECT @@global.auto_increment_offset`; ---connection node_2 -let $auto_increment_offset_node_2 = `SELECT @@global.auto_increment_offset`; - +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc # # Set to invalid value # ---connection node_1 ---let $wsrep_cluster_address_node1 = `SELECT @@wsrep_cluster_address` +--connection node_2 +--let $wsrep_cluster_address_node2 = `SELECT @@wsrep_cluster_address` SET GLOBAL wsrep_cluster_address = 'foo://'; # With wsrep_sync_wait, this returns an error @@ -26,7 +24,7 @@ SET GLOBAL wsrep_cluster_address = 'foo://'; SET SESSION wsrep_sync_wait=0; --error ER_UNKNOWN_COM_ERROR -SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS; +SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS; # Must return 'OFF' SHOW STATUS LIKE 'wsrep_ready'; @@ -38,9 +36,9 @@ SHOW STATUS LIKE 'wsrep_cluster_status'; SHOW STATUS LIKE 'wsrep_local_state'; SHOW STATUS LIKE 'wsrep_local_state_comment'; ---connection node_2 +--connection node_1 --sleep 1 -# Node #2 thinks that it is now part of a single-node primary cluster +# Node #1 thinks that it is now part of a single-node primary cluster SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; @@ -48,63 +46,30 @@ SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VA # Reset everything as it was # ---connection node_1 +--connection node_2 --disable_query_log ---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_node1'; +--eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_node2'; --enable_query_log ---connection node_2 -SET GLOBAL wsrep_cluster_address = @@wsrep_cluster_address; - --source include/wait_until_connected_again.inc --connection node_1 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; -# -# Set to invalid host -# - ---connection node_1 -SET GLOBAL wsrep_cluster_address = 'gcomm://192.0.2.1'; - ---error ER_UNKNOWN_COM_ERROR -SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS; - -# Must return 'OFF' -SHOW STATUS LIKE 'wsrep_ready'; - -# Must return 'Non-primary' -SHOW STATUS LIKE 'wsrep_cluster_status'; - -# Must return 0 = 'Initialized' -SHOW STATUS LIKE 'wsrep_local_state'; -SHOW STATUS LIKE 'wsrep_local_state_comment'; - -# -# Reset everything as it was -# - ---connection node_1 ---disable_query_log ---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_node1'; ---enable_query_log - --connection node_2 -SET GLOBAL wsrep_cluster_address = @@wsrep_cluster_address; ---sleep 1 - ---connection node_1 -SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; -SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +CALL mtr.add_suppression("Backend not supported: foo"); +CALL mtr.add_suppression("Failed to initialize backend using 'foo"); +CALL mtr.add_suppression("Failed to open channel 'my_wsrep_cluster' at 'foo"); +CALL mtr.add_suppression("gcs connect failed: Socket type not supported"); +CALL mtr.add_suppression("wsrep::connect\\(\\) failed: 7"); +CALL mtr.add_suppression("gcs_caused\\(\\) returned -103 \\(Software caused connection abort\\)"); +CALL mtr.add_suppression("failed to open gcomm backend connection: 110: failed to reach primary view: 110"); +CALL mtr.add_suppression("Failed to open backend connection: -110 \\(Connection timed out\\)"); +CALL mtr.add_suppression("gcs connect failed: Connection timed out"); +CALL mtr.add_suppression("WSREP: wsrep::connect\\(foo://\\) failed: 7"); # Restore original auto_increment_offset values. ---disable_query_log ---connection node_1 ---eval SET @@global.auto_increment_offset = $auto_increment_offset_node_1; ---connection node_2 ---eval SET @@global.auto_increment_offset = $auto_increment_offset_node_2; ---enable_query_log - +--source include/auto_increment_offset_restore.inc +--source include/galera_end.inc From 9e032d6150b39cf36a22e7c1503e06fae9c4016d Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 21 Dec 2016 09:34:37 +0530 Subject: [PATCH 280/295] MDEV-11490 Galera_3nodes test suite does not suppress Warnings. Problem:- While running individual tests of Galera_3nodes , We get warnings like '[Warning] WSREP: Could not open state file for reading: '. And because of this individual tests fails. Solution:- We change suite.pm of Galera_3nodes to supress these warnings. --- mysql-test/suite/galera_3nodes/suite.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera_3nodes/suite.pm b/mysql-test/suite/galera_3nodes/suite.pm index 39d5acbcc1b6b..c91e6e07d76ae 100644 --- a/mysql-test/suite/galera_3nodes/suite.pm +++ b/mysql-test/suite/galera_3nodes/suite.pm @@ -26,6 +26,7 @@ push @::global_suppressions, ( qr(WSREP: wsrep_sst_receive_address is set to '127.0.0.1), qr(WSREP: Could not open saved state file for reading: ), + qr(WSREP: Could not open state file for reading: ), qr(WSREP: Gap in state sequence. Need state transfer.), qr(WSREP: Failed to prepare for incremental state transfer:), qr(WSREP:.*down context.*), @@ -33,13 +34,14 @@ push @::global_suppressions, qr(WSREP: last inactive check more than .* skipping check), qr(WSREP: SQL statement was ineffective), qr(WSREP: Releasing seqno [0-9]* before [0-9]* was assigned.), - qr|WSREP: access file\(gvwstate.dat\) failed\(No such file or directory\)|, + qr|WSREP: access file\(.*gvwstate.dat\) failed\(No such file or directory\)|, qr(WSREP: Quorum: No node with complete state), qr(WSREP: Initial position was provided by configuration or SST, avoiding override), qr|WSREP: discarding established \(time wait\) .*|, qr(WSREP: There are no nodes in the same segment that will ever be able to become donors, yet there is a suitable donor outside. Will use that one.), qr(WSREP: evs::proto.*), qr|WSREP: Ignoring possible split-brain (allowed by configuration) from view:.*|, + qr(WSREP: Member .* requested state transfer from .* but it is impossible to select State Transfer donor: Resource temporarily unavailable), qr(WSREP: Could not find peer:), qr(WSREP: Protocol violation. JOIN message sender .*), qr(WSREP: JOIN message from member [0-9]* in non-primary configuration. Ignored.), From c33c638f39eb626ca7e77cbdd38eef6aee53f25e Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 21 Dec 2016 22:40:52 +0200 Subject: [PATCH 281/295] MDEV-7558 analyze_stmt_slow_query_log fails sporadically in buildbot The reason was that the test was reusing the same log file without deleting it between tests. Fixed by creating a new log file as part of the test --- mysql-test/r/analyze_stmt_slow_query_log.result | 3 +++ mysql-test/t/analyze_stmt_slow_query_log.test | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/mysql-test/r/analyze_stmt_slow_query_log.result b/mysql-test/r/analyze_stmt_slow_query_log.result index 7d280e66b6c8c..a0c4b45dee09a 100644 --- a/mysql-test/r/analyze_stmt_slow_query_log.result +++ b/mysql-test/r/analyze_stmt_slow_query_log.result @@ -1,4 +1,7 @@ drop table if exists t1; +SET @@global.slow_query_log = OFF; +FLUSH SLOW LOGS; +SET @@global.slow_query_log = ON; create table t1 (a int); INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); select * from t1 where a<3; diff --git a/mysql-test/t/analyze_stmt_slow_query_log.test b/mysql-test/t/analyze_stmt_slow_query_log.test index 7346ac3b2c5b8..44865b652cb99 100644 --- a/mysql-test/t/analyze_stmt_slow_query_log.test +++ b/mysql-test/t/analyze_stmt_slow_query_log.test @@ -2,6 +2,21 @@ drop table if exists t1; --enable_warnings +# +# Remove old log file +# +let SLOW_LOG_FILE= `select @@slow_query_log_file`; + +SET @@global.slow_query_log = OFF; + +perl; + my $slow_log_file= $ENV{'SLOW_LOG_FILE'} or die "SLOW_LOG_FILE not set"; + unlink($slow_log_file); +EOF + +FLUSH SLOW LOGS; +SET @@global.slow_query_log = ON; + create table t1 (a int); INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); select * from t1 where a<3; From c51c885dee098b09b9eeaef16813dffac95ef994 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 21 Dec 2016 22:41:07 +0200 Subject: [PATCH 282/295] Fixed compiler warning --- storage/xtradb/row/row0ftsort.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 9f182fc5e7095..fb78808ae8057 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -668,7 +668,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; @@ -691,9 +690,6 @@ fts_parallel_tokenization( doc.charset = fts_index_get_charset( psort_info->psort_common->dup->index); - idx_field = dict_index_get_nth_field( - psort_info->psort_common->dup->index, 0); - block = psort_info->merge_block; crypt_block = psort_info->crypt_block; crypt_data = psort_info->psort_common->crypt_data; From 55eb7120a04e23519ff495e7b5be0086f842b78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 22 Dec 2016 14:02:51 +0200 Subject: [PATCH 283/295] MDEV-11218: encryption.innodb_encryption_discard_import failed in buildbot Try to stabilize test cases. These test behave badly when run in certain order. --- .../r/innodb-bad-key-change3.result | 11 ++- .../encryption/r/innodb-discard-import.result | 52 ++++++-------- .../r/innodb_encryption_discard_import.result | 11 +-- .../encryption/t/innodb-bad-key-change3.test | 35 ++++++---- .../encryption/t/innodb-discard-import.test | 68 +++++-------------- .../t/innodb_encryption_discard_import.test | 33 ++++----- 6 files changed, 82 insertions(+), 128 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change3.result b/mysql-test/suite/encryption/r/innodb-bad-key-change3.result index 68d8552a0a307..c83fc5b5fcbba 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change3.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change3.result @@ -1,4 +1,6 @@ -call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded."); +call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; @@ -18,15 +20,12 @@ FLUSH TABLE t1 FOR EXPORT; t1.cfg t1.frm t1.ibd +backup: t1 UNLOCK TABLES; -# Tablespaces should be still encrypted -# t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd ALTER TABLE t1 DISCARD TABLESPACE; +restore: t1 .ibd and .cfg files SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; -# List after t1 DISCARD -t1.frm ALTER TABLE t1 IMPORT TABLESPACE; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW CREATE TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-discard-import.result b/mysql-test/suite/encryption/r/innodb-discard-import.result index 195b82f748899..06f4abab9f40b 100644 --- a/mysql-test/suite/encryption/r/innodb-discard-import.result +++ b/mysql-test/suite/encryption/r/innodb-discard-import.result @@ -1,4 +1,5 @@ -call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); +call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; @@ -57,31 +58,32 @@ t3.ibd t4.cfg t4.frm t4.ibd +backup: t1 +backup: t2 +backup: t3 +backup: t4 +t1.cfg +t1.frm +t1.ibd +t2.cfg +t2.frm +t2.ibd +t3.cfg +t3.frm +t3.ibd +t4.cfg +t4.frm +t4.ibd UNLOCK TABLES; -# tables should be either encrypted and/or compressed -# t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd -# t2 yes on expecting NOT FOUND -NOT FOUND /barfoo/ in t2.ibd -# t3 yes on expecting NOT FOUND -NOT FOUND /tmpres/ in t3.ibd -# t4 yes on expecting NOT FOUND -NOT FOUND /mysql/ in t4.ibd ALTER TABLE t1 DISCARD TABLESPACE; ALTER TABLE t2 DISCARD TABLESPACE; ALTER TABLE t3 DISCARD TABLESPACE; ALTER TABLE t4 DISCARD TABLESPACE; -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; -SET GLOBAL innodb_compression_algorithm = 1; -# List after t1 DISCARD -t1.frm -t2.frm -t3.frm -t4.frm +restore: t1 .ibd and .cfg files +restore: t2 .ibd and .cfg files +restore: t3 .ibd and .cfg files +restore: t4 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't1' SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -92,8 +94,6 @@ SELECT COUNT(*) FROM t1; COUNT(*) 2000 ALTER TABLE t2 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't2' SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( @@ -104,8 +104,6 @@ SELECT COUNT(*) FROM t2; COUNT(*) 2000 ALTER TABLE t3 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't3' SHOW CREATE TABLE t3; Table Create Table t3 CREATE TABLE `t3` ( @@ -116,8 +114,6 @@ SELECT COUNT(*) FROM t3; COUNT(*) 2000 ALTER TABLE t4 IMPORT TABLESPACE; -Warnings: -Warning 1814 Tablespace has been discarded for table 't4' SHOW CREATE TABLE t4; Table Create Table t4 CREATE TABLE `t4` ( @@ -127,10 +123,6 @@ t4 CREATE TABLE `t4` ( SELECT COUNT(*) FROM t4; COUNT(*) 2000 -flush data to disk -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; -SET GLOBAL innodb_compression_algorithm = 1; # tables should be still either encrypted and/or compressed # t1 yes on expecting NOT FOUND NOT FOUND /foobar/ in t1.ibd diff --git a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result index 40284dbf71d72..d08a6e943f4c6 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result @@ -97,14 +97,6 @@ t3 CREATE TABLE `t3` ( `a` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes -# Wait max 10 min for key encryption threads to encrypt all spaces -# Tablespaces should be encrypted after alter table -# t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd -# t2 ... on expecting NOT FOUND -NOT FOUND /temp/ in t2.ibd -# t3 ... on expecting NOT FOUND -NOT FOUND /barfoo/ in t3.ibd # Restarting server # Done restarting server # Verify that tables are still usable @@ -124,5 +116,8 @@ NOT FOUND /foobar/ in t1.ibd NOT FOUND /temp/ in t2.ibd # t3 ... on expecting NOT FOUND NOT FOUND /barfoo/ in t3.ibd +# Wait max 10 min for key encryption threads to encrypt all spaces +# Success! +# Restart mysqld --innodb_encrypt_tables=0 --innodb_encryption_threads=0 DROP PROCEDURE innodb_insert_proc; DROP TABLE t1, t2, t3; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test index 20d63b1064928..f6204e1ddfed8 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test @@ -8,7 +8,10 @@ # # MDEV-8772: Assertion failure in file ha_innodb.cc line 20027 when importing page compressed and encrypted tablespace using incorrect keys # -call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); + +call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded."); +call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); --disable_query_log let $innodb_file_format_orig = `SELECT @@innodb_file_format`; @@ -32,28 +35,26 @@ EOF --enable_reconnect --source include/wait_until_connected_again.inc +--disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; +--enable_warnings CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(255)) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; SHOW WARNINGS; SHOW CREATE TABLE t1; INSERT INTO t1 VALUES (1,'foobar'),(2,'barfoo'); +let MYSQLD_DATADIR =`SELECT @@datadir`; FLUSH TABLE t1 FOR EXPORT; --echo # List before copying files --list_files $MYSQLD_DATADIR/test ---copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg ---copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1"); +EOF UNLOCK TABLES; ---sleep 5 ---echo # Tablespaces should be still encrypted ---let SEARCH_PATTERN=foobar ---echo # t1 yes on expecting NOT FOUND --- let SEARCH_FILE=$t1_IBD --- source include/search_pattern_in_file.inc - ALTER TABLE t1 DISCARD TABLESPACE; --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect @@ -65,17 +66,21 @@ ALTER TABLE t1 DISCARD TABLESPACE; 4;770A8A65DA156D24EE2A093277530144 EOF +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1"); +ib_restore_tablespaces("test", "t1"); +EOF + --exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --source include/restart_mysqld.inc +--disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; ---echo # List after t1 DISCARD ---list_files $MYSQLD_DATADIR/test ---copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg ---copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd +--enable_warnings --error ER_GET_ERRMSG ALTER TABLE t1 IMPORT TABLESPACE; @@ -105,10 +110,12 @@ EOF DROP TABLE t1; # reset system +--disable_warnings --disable_query_log EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; --enable_query_log +--enable_warnings --remove_file $MYSQLTEST_VARDIR/keys1.txt --remove_file $MYSQLTEST_VARDIR/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-discard-import.test b/mysql-test/suite/encryption/t/innodb-discard-import.test index 6d9f6c5dbb35b..9feaacc41e51b 100644 --- a/mysql-test/suite/encryption/t/innodb-discard-import.test +++ b/mysql-test/suite/encryption/t/innodb-discard-import.test @@ -10,7 +10,8 @@ # MDEV-8770: Incorrect error message when importing page compressed tablespace # -call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); +call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); --disable_query_log let $innodb_file_format_orig = `SELECT @@innodb_file_format`; @@ -18,9 +19,11 @@ let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; --enable_query_log +--disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; +--enable_warnings --let $MYSQLD_TMPDIR = `SELECT @@tmpdir` --let $MYSQLD_DATADIR = `SELECT @@datadir` @@ -65,58 +68,28 @@ select count(*) from t2; select count(*) from t3; select count(*) from t4; +let MYSQLD_DATADIR =`SELECT @@datadir`; FLUSH TABLE t1,t2,t3,t4 FOR EXPORT; --echo # List before copying files --list_files $MYSQLD_DATADIR/test ---copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg ---copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd ---copy_file $MYSQLD_DATADIR/test/t2.cfg $MYSQLD_TMPDIR/t2.cfg ---copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_TMPDIR/t2.ibd ---copy_file $MYSQLD_DATADIR/test/t3.cfg $MYSQLD_TMPDIR/t3.cfg ---copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_TMPDIR/t3.ibd ---copy_file $MYSQLD_DATADIR/test/t4.cfg $MYSQLD_TMPDIR/t4.cfg ---copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_TMPDIR/t4.ibd -UNLOCK TABLES; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1","t2","t3","t4"); +EOF +--list_files $MYSQLD_DATADIR/test ---echo # tables should be either encrypted and/or compressed ---let SEARCH_PATTERN=foobar ---echo # t1 yes on expecting NOT FOUND --- let SEARCH_FILE=$t1_IBD --- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=barfoo ---echo # t2 yes on expecting NOT FOUND --- let SEARCH_FILE=$t2_IBD --- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=tmpres ---echo # t3 yes on expecting NOT FOUND --- let SEARCH_FILE=$t3_IBD --- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=mysql ---echo # t4 yes on expecting NOT FOUND --- let SEARCH_FILE=$t4_IBD --- source include/search_pattern_in_file.inc +UNLOCK TABLES; ALTER TABLE t1 DISCARD TABLESPACE; ALTER TABLE t2 DISCARD TABLESPACE; ALTER TABLE t3 DISCARD TABLESPACE; ALTER TABLE t4 DISCARD TABLESPACE; ---source include/restart_mysqld.inc - -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; -SET GLOBAL innodb_compression_algorithm = 1; - ---echo # List after t1 DISCARD ---list_files $MYSQLD_DATADIR/test ---copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg ---copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd ---copy_file $MYSQLD_TMPDIR/t2.cfg $MYSQLD_DATADIR/test/t2.cfg ---copy_file $MYSQLD_TMPDIR/t2.ibd $MYSQLD_DATADIR/test/t2.ibd ---copy_file $MYSQLD_TMPDIR/t3.cfg $MYSQLD_DATADIR/test/t3.cfg ---copy_file $MYSQLD_TMPDIR/t3.ibd $MYSQLD_DATADIR/test/t3.ibd ---copy_file $MYSQLD_TMPDIR/t4.cfg $MYSQLD_DATADIR/test/t4.cfg ---copy_file $MYSQLD_TMPDIR/t4.ibd $MYSQLD_DATADIR/test/t4.ibd +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1","t2","t3","t4"); +ib_restore_tablespaces("test", "t1","t2","t3","t4"); +EOF ALTER TABLE t1 IMPORT TABLESPACE; SHOW CREATE TABLE t1; @@ -131,13 +104,6 @@ ALTER TABLE t4 IMPORT TABLESPACE; SHOW CREATE TABLE t4; SELECT COUNT(*) FROM t4; ---echo flush data to disk ---source include/restart_mysqld.inc - -SET GLOBAL innodb_file_format = `Barracuda`; -SET GLOBAL innodb_file_per_table = ON; -SET GLOBAL innodb_compression_algorithm = 1; - --echo # tables should be still either encrypted and/or compressed --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -160,8 +126,10 @@ DROP PROCEDURE innodb_insert_proc; DROP TABLE t1,t2,t3,t4; # reset system +--disable_warnings --disable_query_log EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; --enable_query_log +--enable_warnings diff --git a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test index a3789ea7ca718..192233a535fdb 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test @@ -19,8 +19,10 @@ let $innodb_file_format_orig = `SELECT @@innodb_file_format`; let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; --enable_query_log +--disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; +--enable_warnings CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB; @@ -115,26 +117,6 @@ SHOW CREATE TABLE t2; ALTER TABLE t3 ENGINE InnoDB; SHOW CREATE TABLE t3; ---echo # Wait max 10 min for key encryption threads to encrypt all spaces ---let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 ---source include/wait_condition.inc - ---sleep 5 ---echo # Tablespaces should be encrypted after alter table ---let SEARCH_PATTERN=foobar ---echo # t1 yes on expecting NOT FOUND --- let SEARCH_FILE=$t1_IBD --- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp ---echo # t2 ... on expecting NOT FOUND --- let SEARCH_FILE=$t2_IBD --- source include/search_pattern_in_file.inc ---echo # t3 ... on expecting NOT FOUND ---let SEARCH_PATTERN=barfoo --- let SEARCH_FILE=$t3_IBD --- source include/search_pattern_in_file.inc - --echo # Restarting server -- source include/restart_mysqld.inc --echo # Done restarting server @@ -159,12 +141,23 @@ SELECT COUNT(1) FROM t3; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--echo # Wait max 10 min for key encryption threads to encrypt all spaces +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--source include/wait_condition.inc + +--echo # Success! +--echo # Restart mysqld --innodb_encrypt_tables=0 --innodb_encryption_threads=0 +-- let $restart_parameters=--innodb_encrypt_tables=0 --innodb_encryption_threads=0 +-- source include/restart_mysqld.inc DROP PROCEDURE innodb_insert_proc; DROP TABLE t1, t2, t3; # reset system +--disable_warnings --disable_query_log EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; --enable_query_log +--enable_warnings From 7e02fd1f710e39e3cfccc507f24ceacee76b9439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Dec 2016 14:20:47 +0200 Subject: [PATCH 284/295] MDEV-11630 Call mutex_free() before freeing the mutex list Make some global fil_crypt_ variables static. fil_close(): Call mutex_free(&fil_system->mutex) also in InnoDB, not only in XtraDB. In InnoDB, sync_close() was called before fil_close(). innobase_shutdown_for_mysql(): Call fil_close() before sync_close(), similar to XtraDB shutdown. fil_space_crypt_cleanup(): Call mutex_free() to pair with fil_space_crypt_init(). fil_crypt_threads_cleanup(): Call mutex_free() to pair with fil_crypt_threads_init(). --- storage/innobase/fil/fil0crypt.cc | 23 ++++++++++++----------- storage/innobase/fil/fil0fil.cc | 5 +---- storage/innobase/srv/srv0start.cc | 2 +- storage/xtradb/fil/fil0crypt.cc | 23 ++++++++++++----------- storage/xtradb/srv/srv0start.cc | 2 +- 5 files changed, 27 insertions(+), 28 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 3d044e82849f7..eb65e1758edc0 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -41,12 +41,12 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include /** Mutex for keys */ -UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; +static ib_mutex_t fil_crypt_key_mutex; static bool fil_crypt_threads_inited = false; #ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_key_mutex_key; +static mysql_pfs_key_t fil_crypt_key_mutex_key; #endif /** Is encryption enabled/disabled */ @@ -62,19 +62,19 @@ static uint srv_n_fil_crypt_threads_started = 0; UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; /** Event to signal FROM the key rotation threads. */ -UNIV_INTERN os_event_t fil_crypt_event; +static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ -UNIV_INTERN os_event_t fil_crypt_threads_event; +static os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ -UNIV_INTERN os_event_t fil_crypt_throttle_sleep_event; +static os_event_t fil_crypt_throttle_sleep_event; /** Mutex for key rotation threads */ -UNIV_INTERN ib_mutex_t fil_crypt_threads_mutex; +static ib_mutex_t fil_crypt_threads_mutex; #ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_threads_mutex_key; +static mysql_pfs_key_t fil_crypt_threads_mutex_key; #endif /** Variable ensuring only 1 thread at time does initial conversion */ @@ -96,13 +96,11 @@ static fil_crypt_stat_t crypt_stat; static ib_mutex_t crypt_stat_mutex; #ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_stat_mutex_key; -#endif +static mysql_pfs_key_t fil_crypt_stat_mutex_key; /** * key for crypt data mutex */ -#ifdef UNIV_PFS_MUTEX UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; #endif @@ -140,6 +138,8 @@ fil_space_crypt_cleanup() /*=====================*/ { os_event_free(fil_crypt_throttle_sleep_event); + mutex_free(&fil_crypt_key_mutex); + mutex_free(&crypt_stat_mutex); } /** @@ -362,7 +362,7 @@ fil_space_destroy_crypt_data( fil_space_crypt_t* c = *crypt_data; c->~fil_space_crypt_struct(); mem_free(c); - (*crypt_data) = NULL; + *crypt_data = NULL; } } @@ -2483,6 +2483,7 @@ fil_crypt_threads_cleanup() { os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); + mutex_free(&fil_crypt_threads_mutex); fil_crypt_threads_inited = false; } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 1aa2555f1b36b..03f6b9148ba1b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6449,10 +6449,7 @@ fil_close(void) { fil_space_crypt_cleanup(); -#ifndef UNIV_HOTBACKUP - /* The mutex should already have been freed. */ - ut_ad(fil_system->mutex.magic_n == 0); -#endif /* !UNIV_HOTBACKUP */ + mutex_free(&fil_system->mutex); hash_table_free(fil_system->spaces); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 135846384f689..1f5d6eef0e8e4 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3244,9 +3244,9 @@ innobase_shutdown_for_mysql(void) que_close(); row_mysql_close(); srv_mon_free(); + fil_close(); sync_close(); srv_free(); - fil_close(); /* 4. Free the os_conc_mutex and all os_events and os_mutexes */ diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 3d044e82849f7..eb65e1758edc0 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -41,12 +41,12 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include /** Mutex for keys */ -UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; +static ib_mutex_t fil_crypt_key_mutex; static bool fil_crypt_threads_inited = false; #ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_key_mutex_key; +static mysql_pfs_key_t fil_crypt_key_mutex_key; #endif /** Is encryption enabled/disabled */ @@ -62,19 +62,19 @@ static uint srv_n_fil_crypt_threads_started = 0; UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; /** Event to signal FROM the key rotation threads. */ -UNIV_INTERN os_event_t fil_crypt_event; +static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ -UNIV_INTERN os_event_t fil_crypt_threads_event; +static os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ -UNIV_INTERN os_event_t fil_crypt_throttle_sleep_event; +static os_event_t fil_crypt_throttle_sleep_event; /** Mutex for key rotation threads */ -UNIV_INTERN ib_mutex_t fil_crypt_threads_mutex; +static ib_mutex_t fil_crypt_threads_mutex; #ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_threads_mutex_key; +static mysql_pfs_key_t fil_crypt_threads_mutex_key; #endif /** Variable ensuring only 1 thread at time does initial conversion */ @@ -96,13 +96,11 @@ static fil_crypt_stat_t crypt_stat; static ib_mutex_t crypt_stat_mutex; #ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_stat_mutex_key; -#endif +static mysql_pfs_key_t fil_crypt_stat_mutex_key; /** * key for crypt data mutex */ -#ifdef UNIV_PFS_MUTEX UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; #endif @@ -140,6 +138,8 @@ fil_space_crypt_cleanup() /*=====================*/ { os_event_free(fil_crypt_throttle_sleep_event); + mutex_free(&fil_crypt_key_mutex); + mutex_free(&crypt_stat_mutex); } /** @@ -362,7 +362,7 @@ fil_space_destroy_crypt_data( fil_space_crypt_t* c = *crypt_data; c->~fil_space_crypt_struct(); mem_free(c); - (*crypt_data) = NULL; + *crypt_data = NULL; } } @@ -2483,6 +2483,7 @@ fil_crypt_threads_cleanup() { os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); + mutex_free(&fil_crypt_threads_mutex); fil_crypt_threads_inited = false; } diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 52bbd61e2b2e0..ada91c645240b 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2015, MariaDB Corporation +Copyright (c) 2013, 2016, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described From 545c9126963b26a093d5c8b6225cc52e360892e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Dec 2016 12:03:36 +0200 Subject: [PATCH 285/295] Remove an unnecessary comparison. --- storage/innobase/fil/fil0crypt.cc | 5 ----- storage/innobase/include/fil0crypt.h | 3 --- storage/xtradb/fil/fil0crypt.cc | 5 ----- storage/xtradb/include/fil0crypt.h | 3 --- 4 files changed, 16 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index eb65e1758edc0..6483cdc5a5396 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -283,11 +283,6 @@ fil_space_read_crypt_data( const byte* page, /*!< in: page 0 */ ulint offset) /*!< in: offset */ { - if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { - /* Crypt data is not stored. */ - return NULL; - } - if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { /* Crypt data is not stored. */ return NULL; diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 8bb0ce65a6bee..9a35f6591e751 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -34,9 +34,6 @@ Created 04/01/2015 Jan Lindström static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { 's', 0xE, 0xC, 'R', 'E', 't' }; -static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; - /* This key will be used if nothing else is given */ #define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index eb65e1758edc0..6483cdc5a5396 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -283,11 +283,6 @@ fil_space_read_crypt_data( const byte* page, /*!< in: page 0 */ ulint offset) /*!< in: offset */ { - if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { - /* Crypt data is not stored. */ - return NULL; - } - if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { /* Crypt data is not stored. */ return NULL; diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 8bb0ce65a6bee..9a35f6591e751 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -34,9 +34,6 @@ Created 04/01/2015 Jan Lindström static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { 's', 0xE, 0xC, 'R', 'E', 't' }; -static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; - /* This key will be used if nothing else is given */ #define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA From dc9f5dfcbb8efbc7c1f178313f945981b01d229b Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Tue, 27 Dec 2016 20:41:32 +0200 Subject: [PATCH 286/295] Replication tests fail on valgrind due to waiting-related timeouts MTR raises default wait_for_pos_timeout from 300 to 1500 when tests are run with valgrind. The same needs to be done for other replication-related waits --- mysql-test/include/sync_slave_sql_with_io.inc | 4 ++++ mysql-test/include/wait_for_slave_param.inc | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mysql-test/include/sync_slave_sql_with_io.inc b/mysql-test/include/sync_slave_sql_with_io.inc index 8048f7a177c20..9efede9a61f1d 100644 --- a/mysql-test/include/sync_slave_sql_with_io.inc +++ b/mysql-test/include/sync_slave_sql_with_io.inc @@ -26,6 +26,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } --let $_master_log_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1) diff --git a/mysql-test/include/wait_for_slave_param.inc b/mysql-test/include/wait_for_slave_param.inc index d3f7ec56614be..25020d46ed966 100644 --- a/mysql-test/include/wait_for_slave_param.inc +++ b/mysql-test/include/wait_for_slave_param.inc @@ -50,6 +50,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } if ($slave_error_param == '') From d50cf42bc05b1faa5d39c766389ac345e119037e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Dec 2016 15:54:24 +0200 Subject: [PATCH 287/295] MDEV-9282 Debian: the Lintian complains about "shlib-calls-exit" in ha_innodb.so Replace all exit() calls in InnoDB with abort() [possibly via ut_a()]. Calling exit() in a multi-threaded program is problematic also for the reason that other threads could see corrupted data structures while some data structures are being cleaned up by atexit() handlers or similar. In the long term, all these calls should be replaced with something that returns an error all the way up the call stack. --- storage/innobase/api/api0misc.cc | 4 ---- storage/innobase/buf/buf0dblwr.cc | 15 +++------------ storage/innobase/fil/fil0fil.cc | 2 +- storage/innobase/include/fts0ast.h | 3 +++ storage/innobase/log/log0log.cc | 20 +++----------------- storage/innobase/os/os0file.cc | 2 +- storage/innobase/os/os0thread.cc | 6 +----- storage/innobase/row/row0mysql.cc | 4 +--- storage/innobase/row/row0undo.cc | 3 +-- storage/innobase/srv/srv0start.cc | 13 ++----------- storage/xtradb/api/api0misc.cc | 4 ---- storage/xtradb/buf/buf0dblwr.cc | 15 +++------------ storage/xtradb/fil/fil0fil.cc | 2 +- storage/xtradb/include/fts0ast.h | 3 +++ storage/xtradb/log/log0log.cc | 30 +++++------------------------- storage/xtradb/log/log0online.cc | 19 ++++++------------- storage/xtradb/os/os0file.cc | 2 +- storage/xtradb/os/os0thread.cc | 6 +----- storage/xtradb/row/row0mysql.cc | 4 +--- storage/xtradb/row/row0undo.cc | 3 +-- storage/xtradb/srv/srv0start.cc | 13 ++----------- 21 files changed, 40 insertions(+), 133 deletions(-) diff --git a/storage/innobase/api/api0misc.cc b/storage/innobase/api/api0misc.cc index a980d32c33f7f..5daee5de4c941 100644 --- a/storage/innobase/api/api0misc.cc +++ b/storage/innobase/api/api0misc.cc @@ -184,10 +184,6 @@ ib_handle_errors( trx_rollback_for_mysql(trx); break; - case DB_MUST_GET_MORE_FILE_SPACE: - - exit(1); - case DB_CORRUPTION: case DB_FOREIGN_EXCEED_MAX_CASCADE: break; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 16877818ba95e..cc5fe52f80aaa 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -224,12 +224,10 @@ buf_dblwr_create(void) + FSP_EXTENT_SIZE / 2 + 100) * UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your buffer pool size. Cannot continue " "operation."); - - exit(EXIT_FAILURE); } block2 = fseg_create(TRX_SYS_SPACE, TRX_SYS_PAGE_NO, @@ -242,15 +240,10 @@ buf_dblwr_create(void) buf_block_dbg_add_level(block2, SYNC_NO_ORDER_CHECK); if (block2 == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - /* We exit without committing the mtr to prevent - its modifications to the database getting to disk */ - - exit(EXIT_FAILURE); } fseg_header = doublewrite + TRX_SYS_DOUBLEWRITE_FSEG; @@ -261,12 +254,10 @@ buf_dblwr_create(void) new_block = fseg_alloc_free_page( fseg_header, prev_page_no + 1, FSP_UP, &mtr); if (new_block == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - exit(EXIT_FAILURE); } /* We read the allocated pages to the buffer pool; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 03f6b9148ba1b..22352d94332d2 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -4599,7 +4599,7 @@ fil_load_single_table_tablespace( return; } - exit(1); + abort(); } if (def.success && remote.success) { diff --git a/storage/innobase/include/fts0ast.h b/storage/innobase/include/fts0ast.h index 50f62063893e0..6229869e8d071 100644 --- a/storage/innobase/include/fts0ast.h +++ b/storage/innobase/include/fts0ast.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +30,8 @@ Created 2007/03/16/03 Sunny Bains #include "mem0mem.h" #include "ha_prototypes.h" +#define exit(x) abort() + /* The type of AST Node */ enum fts_ast_type_t { FTS_AST_OPER, /*!< Operator */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 89b616aba0196..a832657136684 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -815,24 +815,10 @@ log_calc_max_ages(void) mutex_exit(&(log_sys->mutex)); if (!success) { - fprintf(stderr, - "InnoDB: Error: ib_logfiles are too small" - " for innodb_thread_concurrency %lu.\n" - "InnoDB: The combined size of ib_logfiles" + ib_logf(IB_LOG_LEVEL_FATAL, + "The combined size of ib_logfiles" " should be bigger than\n" - "InnoDB: 200 kB * innodb_thread_concurrency.\n" - "InnoDB: To get mysqld to start up, set" - " innodb_thread_concurrency in my.cnf\n" - "InnoDB: to a lower value, for example, to 8." - " After an ERROR-FREE shutdown\n" - "InnoDB: of mysqld you can adjust the size of" - " ib_logfiles, as explained in\n" - "InnoDB: " REFMAN "adding-and-removing.html\n" - "InnoDB: Cannot continue operation." - " Calling exit(1).\n", - (ulong) srv_thread_concurrency); - - exit(1); + "InnoDB: 200 kB * innodb_thread_concurrency."); } return(success); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 2db53c25b04e5..46518ef8d350f 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -814,7 +814,7 @@ os_file_handle_error_cond_exit( } if (should_exit) { - exit(1); + abort(); } } diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index a5b0f7de6ae7c..88f4292d6fe93 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -171,11 +171,7 @@ os_thread_create_func( #else ret = pthread_create(&pthread, &attr, func, arg); #endif - if (ret) { - fprintf(stderr, - "InnoDB: Error: pthread_create returned %d\n", ret); - exit(1); - } + ut_a(ret == 0); #ifndef UNIV_HPUX10 pthread_attr_destroy(&attr); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 3501124710551..1c02590933f87 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -660,9 +660,7 @@ row_mysql_handle_errors( "InnoDB: lack of space. You must add" " a new data file to\n" "InnoDB: my.cnf and restart the database.\n", stderr); - - ut_ad(0); - exit(1); + abort(); case DB_CORRUPTION: fputs("InnoDB: We detected index corruption" diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc index 149dc6719301e..82b1ab049facf 100644 --- a/storage/innobase/row/row0undo.cc +++ b/storage/innobase/row/row0undo.cc @@ -363,8 +363,7 @@ row_undo_step( "InnoDB: Out of tablespace.\n" "InnoDB: Consider increasing" " your tablespace.\n"); - - exit(1); + abort(); } ut_error; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 1f5d6eef0e8e4..bbb9dc0205ed8 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -600,7 +600,7 @@ create_log_file( fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ srv_force_recovery_crash); \ fflush(stderr); \ - exit(3); \ + abort(); \ } \ } while (0) #endif @@ -2912,16 +2912,7 @@ innobase_start_or_create_for_mysql(void) /* Check that os_fast_mutexes work as expected */ os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - if (0 != os_fast_mutex_trylock(&srv_os_test_mutex)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: pthread_mutex_trylock returns" - " an unexpected value on\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: success! Cannot continue.\n"); - exit(1); - } + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); os_fast_mutex_unlock(&srv_os_test_mutex); diff --git a/storage/xtradb/api/api0misc.cc b/storage/xtradb/api/api0misc.cc index a980d32c33f7f..5daee5de4c941 100644 --- a/storage/xtradb/api/api0misc.cc +++ b/storage/xtradb/api/api0misc.cc @@ -184,10 +184,6 @@ ib_handle_errors( trx_rollback_for_mysql(trx); break; - case DB_MUST_GET_MORE_FILE_SPACE: - - exit(1); - case DB_CORRUPTION: case DB_FOREIGN_EXCEED_MAX_CASCADE: break; diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index d8d85c25289db..62ed17296f5d1 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -224,12 +224,10 @@ buf_dblwr_create(void) + FSP_EXTENT_SIZE / 2 + 100) * UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your buffer pool size. Cannot continue " "operation."); - - exit(EXIT_FAILURE); } block2 = fseg_create(TRX_SYS_SPACE, TRX_SYS_PAGE_NO, @@ -242,15 +240,10 @@ buf_dblwr_create(void) buf_block_dbg_add_level(block2, SYNC_NO_ORDER_CHECK); if (block2 == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - /* We exit without committing the mtr to prevent - its modifications to the database getting to disk */ - - exit(EXIT_FAILURE); } fseg_header = doublewrite + TRX_SYS_DOUBLEWRITE_FSEG; @@ -261,12 +254,10 @@ buf_dblwr_create(void) new_block = fseg_alloc_free_page( fseg_header, prev_page_no + 1, FSP_UP, &mtr); if (new_block == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - exit(EXIT_FAILURE); } /* We read the allocated pages to the buffer pool; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 28f262b50c711..5900bdb21402c 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -4618,7 +4618,7 @@ fil_load_single_table_tablespace( return; } - exit(1); + abort(); } if (def.success && remote.success) { diff --git a/storage/xtradb/include/fts0ast.h b/storage/xtradb/include/fts0ast.h index 50f62063893e0..6229869e8d071 100644 --- a/storage/xtradb/include/fts0ast.h +++ b/storage/xtradb/include/fts0ast.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +30,8 @@ Created 2007/03/16/03 Sunny Bains #include "mem0mem.h" #include "ha_prototypes.h" +#define exit(x) abort() + /* The type of AST Node */ enum fts_ast_type_t { FTS_AST_OPER, /*!< Operator */ diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index a01a2ed957070..f3a3486017cc4 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -920,24 +920,10 @@ log_calc_max_ages(void) mutex_exit(&(log_sys->mutex)); if (!success) { - fprintf(stderr, - "InnoDB: Error: ib_logfiles are too small" - " for innodb_thread_concurrency %lu.\n" - "InnoDB: The combined size of ib_logfiles" + ib_logf(IB_LOG_LEVEL_FATAL, + "The combined size of ib_logfiles" " should be bigger than\n" - "InnoDB: 200 kB * innodb_thread_concurrency.\n" - "InnoDB: To get mysqld to start up, set" - " innodb_thread_concurrency in my.cnf\n" - "InnoDB: to a lower value, for example, to 8." - " After an ERROR-FREE shutdown\n" - "InnoDB: of mysqld you can adjust the size of" - " ib_logfiles, as explained in\n" - "InnoDB: " REFMAN "adding-and-removing.html\n" - "InnoDB: Cannot continue operation." - " Calling exit(1).\n", - (ulong) srv_thread_concurrency); - - exit(1); + "InnoDB: 200 kB * innodb_thread_concurrency."); } return(success); @@ -2861,15 +2847,9 @@ log_group_archive( } if (!ret) { - fprintf(stderr, + ib_logf(IB_LOG_LEVEL_FATAL, "InnoDB: Cannot create or open" - " archive log file %s.\n" - "InnoDB: Cannot continue operation.\n" - "InnoDB: Check that the log archive" - " directory exists,\n" - "InnoDB: you have access rights to it, and\n" - "InnoDB: there is space available.\n", name); - exit(1); + " archive log file %s.\n", name); } #ifdef UNIV_DEBUG diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 2a1ac63dc5bd1..4e6ad65a906c8 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011-2012 Percona Inc. All Rights Reserved. +Copyright (C) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -405,12 +406,11 @@ log_online_can_track_missing( last_tracked_lsn = ut_max(last_tracked_lsn, MIN_TRACKED_LSN); if (last_tracked_lsn > tracking_start_lsn) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "last tracked LSN " LSN_PF " is ahead of tracking " "start LSN " LSN_PF ". This can be caused by " "mismatched bitmap files.", last_tracked_lsn, tracking_start_lsn); - exit(1); } return (last_tracked_lsn == tracking_start_lsn) @@ -450,9 +450,7 @@ log_online_track_missing_on_startup( log_bmp_sys->start_lsn = ut_max(last_tracked_lsn, MIN_TRACKED_LSN); log_set_tracked_lsn(log_bmp_sys->start_lsn); - if (!log_online_follow_redo_log()) { - exit(1); - } + ut_a(log_online_follow_redo_log()); ut_ad(log_bmp_sys->end_lsn >= tracking_start_lsn); ib_logf(IB_LOG_LEVEL_INFO, @@ -677,9 +675,8 @@ log_online_read_init(void) if (os_file_closedir(bitmap_dir)) { os_file_get_last_error(TRUE); - ib_logf(IB_LOG_LEVEL_ERROR, "cannot close \'%s\'", + ib_logf(IB_LOG_LEVEL_FATAL, "cannot close \'%s\'", log_bmp_sys->bmp_file_home); - exit(1); } if (!log_bmp_sys->out_seq_num) { @@ -699,9 +696,7 @@ log_online_read_init(void) if (!success) { /* New file, tracking from scratch */ - if (!log_online_start_bitmap_file()) { - exit(1); - } + ut_a(log_online_start_bitmap_file()); } else { @@ -738,9 +733,7 @@ log_online_read_init(void) } else { file_start_lsn = tracking_start_lsn; } - if (!log_online_rotate_bitmap_file(file_start_lsn)) { - exit(1); - } + ut_a(log_online_rotate_bitmap_file(file_start_lsn)); if (last_tracked_lsn < tracking_start_lsn) { diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 007b100285dd0..5e107dac0eb67 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -920,7 +920,7 @@ os_file_handle_error_cond_exit( } if (should_exit) { - exit(1); + abort(); } } diff --git a/storage/xtradb/os/os0thread.cc b/storage/xtradb/os/os0thread.cc index af826027efc36..5ddc40b0eeb92 100644 --- a/storage/xtradb/os/os0thread.cc +++ b/storage/xtradb/os/os0thread.cc @@ -192,11 +192,7 @@ os_thread_create_func( #else ret = pthread_create(&pthread, &attr, func, arg); #endif - if (ret) { - fprintf(stderr, - "InnoDB: Error: pthread_create returned %d\n", ret); - exit(1); - } + ut_a(ret == 0); #ifndef UNIV_HPUX10 pthread_attr_destroy(&attr); diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 0bdee1282f8a6..e53a0ea9586a1 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1196,9 +1196,7 @@ row_mysql_handle_errors( "InnoDB: lack of space. You must add" " a new data file to\n" "InnoDB: my.cnf and restart the database.\n", stderr); - - ut_ad(0); - exit(1); + abort(); case DB_CORRUPTION: fputs("InnoDB: We detected index corruption" diff --git a/storage/xtradb/row/row0undo.cc b/storage/xtradb/row/row0undo.cc index 149dc6719301e..82b1ab049facf 100644 --- a/storage/xtradb/row/row0undo.cc +++ b/storage/xtradb/row/row0undo.cc @@ -363,8 +363,7 @@ row_undo_step( "InnoDB: Out of tablespace.\n" "InnoDB: Consider increasing" " your tablespace.\n"); - - exit(1); + abort(); } ut_error; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ada91c645240b..2dd0285d03f05 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -630,7 +630,7 @@ create_log_file( fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ srv_force_recovery_crash); \ fflush(stderr); \ - exit(3); \ + abort(); \ } \ } while (0) #endif @@ -2979,16 +2979,7 @@ innobase_start_or_create_for_mysql(void) /* Check that os_fast_mutexes work as expected */ os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - if (0 != os_fast_mutex_trylock(&srv_os_test_mutex)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: pthread_mutex_trylock returns" - " an unexpected value on\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: success! Cannot continue.\n"); - exit(1); - } + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); os_fast_mutex_unlock(&srv_os_test_mutex); From 283e9cf4cbb34e1325699707068ab72ec3accfff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 28 Dec 2016 16:14:28 +0200 Subject: [PATCH 288/295] MDEV-11656: 'Data structure corruption' IMPORT TABLESPACE doesn't work for encrypted InnoDB tables if space_id changed Problem was that for encryption we use temporary scratch area for reading and writing tablespace pages. But if page was not really decrypted the correct updated page was not moved to scratch area that was then written. This can happen e.g. for page 0 as it is newer encrypted even if encryption is enabled and as we write the contents of old page 0 to tablespace it contained naturally incorrect space_id that is then later noted and error message was written. Updated page with correct space_id was lost. If tablespace is encrypted we use additional temporary scratch area where pages are read for decrypting readptr == crypt_io_buffer != io_buffer. Destination for decryption is a buffer pool block block->frame == dst == io_buffer that is updated. Pages that did not require decryption even when tablespace is marked as encrypted are not copied instead block->frame is set to src == readptr. If tablespace was encrypted we copy updated page to writeptr != io_buffer. This fixes above bug. For encryption we again use temporary scratch area writeptr != io_buffer == dst that is then written to the tablespace (1) For normal tables src == dst == writeptr ut_ad(!encrypted && !page_compressed ? src == dst && dst == writeptr + (i * size):1); (2) For page compressed tables src == dst == writeptr ut_ad(page_compressed && !encrypted ? src == dst && dst == writeptr + (i * size):1); (3) For encrypted tables src != dst != writeptr ut_ad(encrypted ? src != dst && dst != writeptr + (i * size):1); --- .../r/innodb-discard-import-change.result | 105 ++++++++++++++ .../t/innodb-discard-import-change.test | 131 ++++++++++++++++++ storage/innobase/fil/fil0fil.cc | 61 ++++++-- storage/xtradb/fil/fil0fil.cc | 61 ++++++-- 4 files changed, 342 insertions(+), 16 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-discard-import-change.result create mode 100644 mysql-test/suite/encryption/t/innodb-discard-import-change.test diff --git a/mysql-test/suite/encryption/r/innodb-discard-import-change.result b/mysql-test/suite/encryption/r/innodb-discard-import-change.result new file mode 100644 index 0000000000000..7071f9eaf2008 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-discard-import-change.result @@ -0,0 +1,105 @@ +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +insert into t1 values (NULL, 'verysecretmessage'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +insert into t5 select * from t1; +FLUSH TABLE t1,t2,t3,t4,t5 FOR EXPORT; +backup: t1 +backup: t2 +backup: t3 +backup: t4 +backup: t5 +t1.cfg +t1.frm +t1.ibd +t2.cfg +t2.frm +t2.ibd +t3.cfg +t3.frm +t3.ibd +t4.cfg +t4.frm +t4.ibd +t5.cfg +t5.frm +t5.ibd +UNLOCK TABLES; +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; +create table t6(a int) engine=innodb; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; +restore: t1 .ibd and .cfg files +restore: t2 .ibd and .cfg files +restore: t3 .ibd and .cfg files +restore: t4 .ibd and .cfg files +restore: t5 .ibd and .cfg files +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` bigint(20) NOT NULL AUTO_INCREMENT, + `b` char(200) DEFAULT NULL, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=377 DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=4 +SELECT COUNT(*) FROM t1; +COUNT(*) +256 +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t2; +COUNT(*) +256 +ALTER TABLE t3 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t3; +COUNT(*) +256 +ALTER TABLE t4 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t4; +COUNT(*) +256 +ALTER TABLE t5 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t5; +COUNT(*) +256 +# t1 encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t1.ibd +# t2 encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t2.ibd +# t3 page compressed expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t3.ibd +# t4 page compressed and encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t4.ibd +# t5 normal expecting FOUND +FOUND /verysecretmessage/ in t5.ibd +DROP TABLE t1,t2,t3,t4,t5,t6; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import-change.test b/mysql-test/suite/encryption/t/innodb-discard-import-change.test new file mode 100644 index 0000000000000..a278a8fba2910 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-discard-import-change.test @@ -0,0 +1,131 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# +# MDEV-11656: 'Data structure corruption' IMPORT TABLESPACE doesn't work for encrypted InnoDB tables if space_id changed +# + +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; +--enable_query_log + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +--enable_warnings + +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; + +insert into t1 values (NULL, 'verysecretmessage'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +insert into t5 select * from t1; + +let MYSQLD_DATADIR =`SELECT @@datadir`; +FLUSH TABLE t1,t2,t3,t4,t5 FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1","t2","t3","t4","t5"); +EOF +--list_files $MYSQLD_DATADIR/test +UNLOCK TABLES; + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; + +# +# Now intentionally change space_id for t1,t3,t4,t5 +# +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; + +create table t6(a int) engine=innodb; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; + +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1","t2","t3","t4","t5"); +ib_restore_tablespaces("test", "t1","t2","t3","t4","t5"); +EOF + +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +SELECT COUNT(*) FROM t1; +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t2; +ALTER TABLE t3 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t3; +ALTER TABLE t4 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t4; +ALTER TABLE t5 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t5; + +# +# Verify +# +--let $MYSQLD_TMPDIR = `SELECT @@tmpdir` +--let $MYSQLD_DATADIR = `SELECT @@datadir` +--let SEARCH_RANGE = 10000000 +--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd +--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd +--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd +--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd +--let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd +--let SEARCH_PATTERN=verysecretmessage +--echo # t1 encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 page compressed expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 page compressed and encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc +--echo # t5 normal expecting FOUND +-- let SEARCH_FILE=$t5_IBD +-- source include/search_pattern_in_file.inc + +DROP TABLE t1,t2,t3,t4,t5,t6; + +# reset system +--disable_warnings +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; +--enable_query_log +--enable_warnings + diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 22352d94332d2..ce5c62a8c8b73 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6533,6 +6533,7 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { byte* io_buffer = iter.io_buffer; + bool row_compressed = false; block->frame = io_buffer; @@ -6545,6 +6546,7 @@ fil_iterate( /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; + row_compressed = true; } else { io_buffer = iter.io_buffer; } @@ -6585,8 +6587,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; ++i) { ulint size = iter.page_size; dberr_t err = DB_SUCCESS; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6610,8 +6613,12 @@ fil_iterate( if (decrypted) { updated = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, size); + if (!page_compressed && !row_compressed) { + block->frame = src; + frame_changed = true; + } else { + memcpy(dst, src, size); + } } } @@ -6636,7 +6643,45 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; @@ -6657,7 +6702,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6839,9 +6884,9 @@ fil_tablespace_iterate( void* crypt_io_buffer = NULL; if (iter.crypt_data != NULL) { crypt_io_buffer = mem_alloc( - iter.n_io_buffers * UNIV_PAGE_SIZE); + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); iter.crypt_io_buffer = static_cast( - crypt_io_buffer); + ut_align(crypt_io_buffer, UNIV_PAGE_SIZE)); } err = fil_iterate(iter, &block, callback); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 5900bdb21402c..e7da4569f0d1f 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -6594,6 +6594,7 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { byte* io_buffer = iter.io_buffer; + bool row_compressed = false; block->frame = io_buffer; @@ -6606,6 +6607,7 @@ fil_iterate( /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; + row_compressed = true; } else { io_buffer = iter.io_buffer; } @@ -6646,8 +6648,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; ++i) { ulint size = iter.page_size; dberr_t err = DB_SUCCESS; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6671,8 +6674,12 @@ fil_iterate( if (decrypted) { updated = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, size); + if (!page_compressed && !row_compressed) { + block->frame = src; + frame_changed = true; + } else { + memcpy(dst, src, size); + } } } @@ -6697,7 +6704,45 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; @@ -6718,7 +6763,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6900,9 +6945,9 @@ fil_tablespace_iterate( void* crypt_io_buffer = NULL; if (iter.crypt_data != NULL) { crypt_io_buffer = mem_alloc( - iter.n_io_buffers * UNIV_PAGE_SIZE); + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); iter.crypt_io_buffer = static_cast( - crypt_io_buffer); + ut_align(crypt_io_buffer, UNIV_PAGE_SIZE)); } err = fil_iterate(iter, &block, callback); From 100f721c0aeb96f687a658586afd22a67098c4a8 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 21 Dec 2016 17:41:48 +0100 Subject: [PATCH 289/295] MDEV-11584: GRANT inside an SP does not work well on 2nd execution Allocate password hash in statment memory --- mysql-test/r/sp.result | 9 +++++++++ mysql-test/t/sp.test | 11 +++++++++++ sql/sql_acl.cc | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index fefcdae539eba..2b50585aa5b77 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8040,3 +8040,12 @@ v_name v_total c 1 DROP PROCEDURE p1; DROP TABLE t1; +# +# MDEV-11584: GRANT inside an SP does not work well on 2nd execution +# +CREATE PROCEDURE sp1() +GRANT ALL PRIVILEGES ON *.* TO 'foo'@'%' IDENTIFIED BY 'pass'; +CALL sp1(); +CALL sp1(); +drop user 'foo'@'%'; +drop procedure sp1; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index aaab59bcb89d9..acb6099cea108 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9484,3 +9484,14 @@ CALL p1(); DROP PROCEDURE p1; DROP TABLE t1; + +--echo # +--echo # MDEV-11584: GRANT inside an SP does not work well on 2nd execution +--echo # + +CREATE PROCEDURE sp1() + GRANT ALL PRIVILEGES ON *.* TO 'foo'@'%' IDENTIFIED BY 'pass'; +CALL sp1(); +CALL sp1(); +drop user 'foo'@'%'; +drop procedure sp1; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 2ae252c223227..6334d3fa1dd6a 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -1064,7 +1064,12 @@ static bool fix_lex_user(THD *thd, LEX_USER *user) make_scramble= my_make_scrambled_password; } + Query_arena *arena, backup; + arena= thd->activate_stmt_arena_if_needed(&backup); char *buff= (char *) thd->alloc(scramble_length + 1); + if (arena) + thd->restore_active_arena(arena, &backup); + if (buff == NULL) return true; make_scramble(buff, user->pwtext.str, user->pwtext.length); From 23cc1be270c7304963643947d8e5ab88f4e312ee Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 21 Dec 2016 20:11:14 +0100 Subject: [PATCH 290/295] MDEV-11584: GRANT inside an SP does not work well on 2nd execution Allocate password hash in statment memory --- mysql-test/r/sp.result | 10 ++++++++++ mysql-test/t/sp.test | 14 ++++++++++++++ sql/sql_acl.cc | 5 +++++ 3 files changed, 29 insertions(+) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index 820701f844c13..4142fc0b79655 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8032,3 +8032,13 @@ return 1; end | ERROR 0A000: Not allowed to return a result set from a function drop table t1,t2; +# +# MDEV-11584: GRANT inside an SP does not work well on 2nd execution +# +CREATE PROCEDURE sp1() +GRANT ALL PRIVILEGES ON *.* TO 'foo'@'%' IDENTIFIED BY 'pass'; +CALL sp1(); +CALL sp1(); +drop user 'foo'@'%'; +drop procedure sp1; +#End of 10.1 tests diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index bcddbd6f97ed1..3034f34d763b8 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9504,3 +9504,17 @@ end | --delimiter ; drop table t1,t2; + +--echo # +--echo # MDEV-11584: GRANT inside an SP does not work well on 2nd execution +--echo # + +CREATE PROCEDURE sp1() + GRANT ALL PRIVILEGES ON *.* TO 'foo'@'%' IDENTIFIED BY 'pass'; +CALL sp1(); +CALL sp1(); +drop user 'foo'@'%'; +drop procedure sp1; + + +--echo #End of 10.1 tests diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 3f26bc2e08a2b..2accb3abc910b 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -1067,7 +1067,12 @@ static bool fix_lex_user(THD *thd, LEX_USER *user) make_scramble= my_make_scrambled_password; } + Query_arena *arena, backup; + arena= thd->activate_stmt_arena_if_needed(&backup); char *buff= (char *) thd->alloc(scramble_length + 1); + if (arena) + thd->restore_active_arena(arena, &backup); + if (buff == NULL) return true; make_scramble(buff, user->pwtext.str, user->pwtext.length); From 48dc7cc66ef5b69fcf28ec0b2ecf0338c188cf4e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Dec 2016 18:08:18 +0100 Subject: [PATCH 291/295] cleanup: redundant memcmp() --- storage/innobase/fil/fil0crypt.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 75efcdfdab02f..f68eabba5795a 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -251,11 +251,6 @@ fil_space_read_crypt_data( const byte* page, /*!< in: page 0 */ ulint offset) /*!< in: offset */ { - if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { - /* Crypt data is not stored. */ - return NULL; - } - if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { /* Crypt data is not stored. */ return NULL; From 0d897c2cebaff9d039514880d46752ab41c89c70 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Dec 2016 17:56:40 +0100 Subject: [PATCH 292/295] cleanup: binlog.binlog_row_annotate --- .../binlog_tests/binlog_row_annotate.inc | 37 +- .../suite/binlog/r/binlog_row_annotate.result | 412 +++++++----------- .../binlog_row_annotate.result | 269 +++++------- 3 files changed, 282 insertions(+), 436 deletions(-) diff --git a/mysql-test/extra/binlog_tests/binlog_row_annotate.inc b/mysql-test/extra/binlog_tests/binlog_row_annotate.inc index b3784ea960698..e53f49149f1e9 100644 --- a/mysql-test/extra/binlog_tests/binlog_row_annotate.inc +++ b/mysql-test/extra/binlog_tests/binlog_row_annotate.inc @@ -4,7 +4,7 @@ # Please check all dependent tests after modifying it # # Usage: -# --let $use_remote_mysqlbinlog= 1 # optional +# --let use_remote_mysqlbinlog= 1 # optional # --source extra/binlog_tests/binlog_row_annotate.inc # # By default, the script uses mysqlbinlog both with direct access to files @@ -35,13 +35,12 @@ set @old_binlog_checksum=@@binlog_checksum; set global binlog_checksum=NONE; +--let datadir= `select @@datadir` --source include/have_log_bin.inc --source include/binlog_start_pos.inc --source include/have_binlog_format_row.inc ---disable_query_log - set sql_mode=""; # Fix timestamp to avoid varying results @@ -50,14 +49,6 @@ SET timestamp=1000000000; # Delete all existing binary logs RESET MASTER; ---disable_warnings -DROP DATABASE IF EXISTS test1; -DROP DATABASE IF EXISTS test2; -DROP DATABASE IF EXISTS test3; -DROP DATABASE IF EXISTS xtest1; -DROP DATABASE IF EXISTS xtest2; ---enable_warnings - CREATE DATABASE test1; CREATE TABLE test1.t1(a int); @@ -108,7 +99,6 @@ DELETE xtest1.xt1, xtest2.xt2 WHERE xtest1.xt1.a=xtest2.xt2.a AND xtest2.xt2.a=test3.t3.a; FLUSH LOGS; ---enable_query_log --echo ##################################################################################### --echo # The following Annotate_rows events should appear below: @@ -119,11 +109,7 @@ FLUSH LOGS; --echo # - DELETE xtest1.xt1, test2.t2 FROM <...> --echo ##################################################################################### -let $start_pos= `select @binlog_start_pos`; ---replace_column 2 # 5 # ---replace_result $start_pos ---replace_regex /table_id: [0-9]+/table_id: #/ /\/\* xid=.* \*\//\/* xid= *\// ---eval show binlog events in 'master-bin.000001' from $start_pos +--source include/show_binlog_events.inc if (!$use_remote_mysqlbinlog) { @@ -138,9 +124,8 @@ if (!$use_remote_mysqlbinlog) --echo # - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) --echo ##################################################################################### - let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ - --exec $MYSQL_BINLOG --base64-output=decode-rows -v -v $MYSQLD_DATADIR/master-bin.000001 + --exec $MYSQL_BINLOG --base64-output=decode-rows -v -v $datadir/master-bin.000001 --echo # --echo ##################################################################################### @@ -149,9 +134,8 @@ if (!$use_remote_mysqlbinlog) --echo # - DELETE test1.t1, test2.t2 FROM <...> --echo ##################################################################################### - let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ - --exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v $MYSQLD_DATADIR/master-bin.000001 + --exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v $datadir/master-bin.000001 --echo # --echo ##################################################################################### @@ -159,11 +143,10 @@ if (!$use_remote_mysqlbinlog) --echo # No Annotates should appear in this output --echo ##################################################################################### - let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ - --exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v $MYSQLD_DATADIR/master-bin.000001 + --exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v $datadir/master-bin.000001 - --let $use_remote_mysqlbinlog= 0 + --let use_remote_mysqlbinlog= 0 } --echo # @@ -177,7 +160,6 @@ if (!$use_remote_mysqlbinlog) --echo # - DELETE xtest1.xt1, test2.t2 FROM <...> (with one subsequent Table map) --echo ##################################################################################### -let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ --exec $MYSQL_BINLOG --base64-output=decode-rows -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 @@ -188,7 +170,6 @@ let $MYSQLD_DATADIR= `select @@datadir`; --echo # - DELETE test1.t1, test2.t2 FROM <...> --echo ##################################################################################### -let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ --exec $MYSQL_BINLOG --base64-output=decode-rows --database=test1 -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 @@ -198,18 +179,14 @@ let $MYSQLD_DATADIR= `select @@datadir`; --echo # No Annotates should appear in this output --echo ##################################################################################### -let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ --exec $MYSQL_BINLOG --base64-output=decode-rows --skip-annotate-row-events -v -v --read-from-remote-server --user=root --host=localhost --port=$MASTER_MYPORT master-bin.000001 # Clean-up ---disable_query_log set global binlog_checksum=@old_binlog_checksum; DROP DATABASE test1; DROP DATABASE test2; DROP DATABASE test3; DROP DATABASE xtest1; DROP DATABASE xtest2; ---enable_query_log - diff --git a/mysql-test/suite/binlog/r/binlog_row_annotate.result b/mysql-test/suite/binlog/r/binlog_row_annotate.result index e7b695c00d929..3da1d9877f4ed 100644 --- a/mysql-test/suite/binlog/r/binlog_row_annotate.result +++ b/mysql-test/suite/binlog/r/binlog_row_annotate.result @@ -1,5 +1,37 @@ set @old_binlog_checksum=@@binlog_checksum; set global binlog_checksum=NONE; +set sql_mode=""; +SET timestamp=1000000000; +RESET MASTER; +CREATE DATABASE test1; +CREATE TABLE test1.t1(a int); +CREATE DATABASE test2; +CREATE TABLE test2.t2(a int); +CREATE VIEW test2.v2 AS SELECT * FROM test2.t2; +CREATE DATABASE test3; +CREATE TABLE test3.t3(a int); +CREATE DATABASE xtest1; +CREATE TABLE xtest1.xt1(a int); +CREATE DATABASE xtest2; +CREATE TABLE xtest2.xt2(a int); +INSERT INTO test1.t1 VALUES (1), (2), (3); +SET SESSION binlog_annotate_row_events = ON; +INSERT INTO test2.t2 VALUES (1), (2), (3); +INSERT INTO test3.t3 VALUES (1), (2), (3); +DELETE test1.t1, test2.t2 +FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 +WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a; +INSERT INTO xtest1.xt1 VALUES (1), (2), (3); +INSERT INTO test2.v2 VALUES (1), (2), (3); +DELETE xtest1.xt1, test2.t2 +FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 +WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a; +INSERT INTO xtest1.xt1 VALUES (1), (2), (3); +INSERT INTO xtest2.xt2 VALUES (1), (2), (3); +DELETE xtest1.xt1, xtest2.xt2 +FROM xtest1.xt1 INNER JOIN xtest2.xt2 INNER JOIN test3.t3 +WHERE xtest1.xt1.a=xtest2.xt2.a AND xtest2.xt2.a=test3.t3.a; +FLUSH LOGS; ##################################################################################### # The following Annotate_rows events should appear below: # - INSERT INTO test2.t2 VALUES (1), (2), (3) @@ -8,58 +40,50 @@ set global binlog_checksum=NONE; # - INSERT INTO test2.t2 VALUES (1), (2), (3) # - DELETE xtest1.xt1, test2.t2 FROM <...> ##################################################################################### -show binlog events in 'master-bin.000001' from ; +include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000001 # Gtid_list 1 # [] -master-bin.000001 # Binlog_checkpoint 1 # master-bin.000001 -master-bin.000001 # Gtid 1 # GTID 0-1-1 -master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test1 -master-bin.000001 # Gtid 1 # GTID 0-1-2 -master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test2 -master-bin.000001 # Gtid 1 # GTID 0-1-3 -master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test3 -master-bin.000001 # Gtid 1 # GTID 0-1-4 -master-bin.000001 # Query 1 # CREATE DATABASE test1 -master-bin.000001 # Gtid 1 # GTID 0-1-5 -master-bin.000001 # Query 1 # CREATE DATABASE test2 -master-bin.000001 # Gtid 1 # GTID 0-1-6 -master-bin.000001 # Query 1 # CREATE DATABASE test3 -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-7 -master-bin.000001 # Table_map 1 # table_id: # (test1.t1) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-8 -master-bin.000001 # Annotate_rows 1 # INSERT INTO test2.t2 VALUES (1), (2), (3) -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-9 -master-bin.000001 # Annotate_rows 1 # INSERT INTO test3.t3 VALUES (1), (2), (3) -master-bin.000001 # Table_map 1 # table_id: # (test3.t3) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-10 -master-bin.000001 # Annotate_rows 1 # DELETE test1.t1, test2.t2 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # CREATE DATABASE test1 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # CREATE DATABASE test2 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # CREATE DATABASE test3 +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Table_map # # table_id: # (test1.t1) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # INSERT INTO test2.t2 VALUES (1), (2), (3) +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # INSERT INTO test3.t3 VALUES (1), (2), (3) +master-bin.000001 # Table_map # # table_id: # (test3.t3) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # DELETE test1.t1, test2.t2 FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a -master-bin.000001 # Table_map 1 # table_id: # (test1.t1) -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Delete_rows_v1 1 # table_id: # -master-bin.000001 # Delete_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-11 -master-bin.000001 # Annotate_rows 1 # INSERT INTO test2.v2 VALUES (1), (2), (3) -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-12 -master-bin.000001 # Annotate_rows 1 # DELETE xtest1.xt1, test2.t2 +master-bin.000001 # Table_map # # table_id: # (test1.t1) +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Delete_rows_v1 # # table_id: # +master-bin.000001 # Delete_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # INSERT INTO test2.v2 VALUES (1), (2), (3) +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # DELETE xtest1.xt1, test2.t2 FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Delete_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Rotate 1 # master-bin.000002;pos=4 +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Delete_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Rotate # # master-bin.000002;pos=POS # ##################################################################################### # mysqlbinlog @@ -98,7 +122,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -106,7 +130,7 @@ DROP DATABASE IF EXISTS test1 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test2 +CREATE DATABASE test2 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl @@ -114,35 +138,11 @@ DROP DATABASE IF EXISTS test2 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test3 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl -/*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test2 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; CREATE DATABASE test3 /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; # at # @@ -164,8 +164,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -190,8 +190,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -216,8 +216,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -257,8 +257,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -283,8 +283,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -351,7 +351,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -362,24 +362,8 @@ DROP DATABASE IF EXISTS test1 /*!100001 SET @@session.gtid_seq_no=3*//*!*/; # at # # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 /*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -401,8 +385,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -414,8 +398,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -427,8 +411,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -458,8 +442,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -471,8 +455,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -523,7 +507,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -531,7 +515,7 @@ DROP DATABASE IF EXISTS test1 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test2 +CREATE DATABASE test2 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl @@ -539,35 +523,11 @@ DROP DATABASE IF EXISTS test2 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test3 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl -/*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test2 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; CREATE DATABASE test3 /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; # at # @@ -589,8 +549,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -613,8 +573,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -637,8 +597,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -674,8 +634,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -698,8 +658,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -766,7 +726,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -774,7 +734,7 @@ DROP DATABASE IF EXISTS test1 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test2 +CREATE DATABASE test2 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl @@ -782,35 +742,11 @@ DROP DATABASE IF EXISTS test2 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test3 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl -/*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test2 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; CREATE DATABASE test3 /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; # at # @@ -832,8 +768,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -858,8 +794,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -884,8 +820,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -925,8 +861,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -951,8 +887,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -1019,7 +955,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -1030,24 +966,8 @@ DROP DATABASE IF EXISTS test1 /*!100001 SET @@session.gtid_seq_no=3*//*!*/; # at # # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 /*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -1069,8 +989,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -1082,8 +1002,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -1095,8 +1015,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -1126,8 +1046,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -1139,8 +1059,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -1191,7 +1111,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -1199,7 +1119,7 @@ DROP DATABASE IF EXISTS test1 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test2 +CREATE DATABASE test2 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl @@ -1207,35 +1127,11 @@ DROP DATABASE IF EXISTS test2 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test3 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl -/*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test2 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; CREATE DATABASE test3 /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; # at # @@ -1257,8 +1153,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -1280,8 +1176,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -1303,8 +1199,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -1339,8 +1235,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -1362,8 +1258,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -1391,3 +1287,9 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; +set global binlog_checksum=@old_binlog_checksum; +DROP DATABASE test1; +DROP DATABASE test2; +DROP DATABASE test3; +DROP DATABASE xtest1; +DROP DATABASE xtest2; diff --git a/mysql-test/suite/binlog_encryption/binlog_row_annotate.result b/mysql-test/suite/binlog_encryption/binlog_row_annotate.result index d53849b11fd6f..f4cfc86619d7e 100644 --- a/mysql-test/suite/binlog_encryption/binlog_row_annotate.result +++ b/mysql-test/suite/binlog_encryption/binlog_row_annotate.result @@ -1,5 +1,37 @@ set @old_binlog_checksum=@@binlog_checksum; set global binlog_checksum=NONE; +set sql_mode=""; +SET timestamp=1000000000; +RESET MASTER; +CREATE DATABASE test1; +CREATE TABLE test1.t1(a int); +CREATE DATABASE test2; +CREATE TABLE test2.t2(a int); +CREATE VIEW test2.v2 AS SELECT * FROM test2.t2; +CREATE DATABASE test3; +CREATE TABLE test3.t3(a int); +CREATE DATABASE xtest1; +CREATE TABLE xtest1.xt1(a int); +CREATE DATABASE xtest2; +CREATE TABLE xtest2.xt2(a int); +INSERT INTO test1.t1 VALUES (1), (2), (3); +SET SESSION binlog_annotate_row_events = ON; +INSERT INTO test2.t2 VALUES (1), (2), (3); +INSERT INTO test3.t3 VALUES (1), (2), (3); +DELETE test1.t1, test2.t2 +FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 +WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a; +INSERT INTO xtest1.xt1 VALUES (1), (2), (3); +INSERT INTO test2.v2 VALUES (1), (2), (3); +DELETE xtest1.xt1, test2.t2 +FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 +WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a; +INSERT INTO xtest1.xt1 VALUES (1), (2), (3); +INSERT INTO xtest2.xt2 VALUES (1), (2), (3); +DELETE xtest1.xt1, xtest2.xt2 +FROM xtest1.xt1 INNER JOIN xtest2.xt2 INNER JOIN test3.t3 +WHERE xtest1.xt1.a=xtest2.xt2.a AND xtest2.xt2.a=test3.t3.a; +FLUSH LOGS; ##################################################################################### # The following Annotate_rows events should appear below: # - INSERT INTO test2.t2 VALUES (1), (2), (3) @@ -8,58 +40,51 @@ set global binlog_checksum=NONE; # - INSERT INTO test2.t2 VALUES (1), (2), (3) # - DELETE xtest1.xt1, test2.t2 FROM <...> ##################################################################################### -show binlog events in 'master-bin.000001' from ; +include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000001 # Gtid_list 1 # [] -master-bin.000001 # Binlog_checkpoint 1 # master-bin.000001 -master-bin.000001 # Gtid 1 # GTID 0-1-1 -master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test1 -master-bin.000001 # Gtid 1 # GTID 0-1-2 -master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test2 -master-bin.000001 # Gtid 1 # GTID 0-1-3 -master-bin.000001 # Query 1 # DROP DATABASE IF EXISTS test3 -master-bin.000001 # Gtid 1 # GTID 0-1-4 -master-bin.000001 # Query 1 # CREATE DATABASE test1 -master-bin.000001 # Gtid 1 # GTID 0-1-5 -master-bin.000001 # Query 1 # CREATE DATABASE test2 -master-bin.000001 # Gtid 1 # GTID 0-1-6 -master-bin.000001 # Query 1 # CREATE DATABASE test3 -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-7 -master-bin.000001 # Table_map 1 # table_id: # (test1.t1) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-8 -master-bin.000001 # Annotate_rows 1 # INSERT INTO test2.t2 VALUES (1), (2), (3) -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-9 -master-bin.000001 # Annotate_rows 1 # INSERT INTO test3.t3 VALUES (1), (2), (3) -master-bin.000001 # Table_map 1 # table_id: # (test3.t3) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-10 -master-bin.000001 # Annotate_rows 1 # DELETE test1.t1, test2.t2 +master-bin.000001 # Binlog_checkpoint # # master-bin.000001 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # CREATE DATABASE test1 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # CREATE DATABASE test2 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # CREATE DATABASE test3 +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Table_map # # table_id: # (test1.t1) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # INSERT INTO test2.t2 VALUES (1), (2), (3) +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # INSERT INTO test3.t3 VALUES (1), (2), (3) +master-bin.000001 # Table_map # # table_id: # (test3.t3) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # DELETE test1.t1, test2.t2 FROM test1.t1 INNER JOIN test2.t2 INNER JOIN test3.t3 WHERE test1.t1.a=test2.t2.a AND test2.t2.a=test3.t3.a -master-bin.000001 # Table_map 1 # table_id: # (test1.t1) -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Delete_rows_v1 1 # table_id: # -master-bin.000001 # Delete_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-11 -master-bin.000001 # Annotate_rows 1 # INSERT INTO test2.v2 VALUES (1), (2), (3) -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Write_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Gtid 1 # BEGIN GTID 0-1-12 -master-bin.000001 # Annotate_rows 1 # DELETE xtest1.xt1, test2.t2 +master-bin.000001 # Table_map # # table_id: # (test1.t1) +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Delete_rows_v1 # # table_id: # +master-bin.000001 # Delete_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # INSERT INTO test2.v2 VALUES (1), (2), (3) +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Annotate_rows # # DELETE xtest1.xt1, test2.t2 FROM xtest1.xt1 INNER JOIN test2.t2 INNER JOIN test3.t3 WHERE xtest1.xt1.a=test2.t2.a AND test2.t2.a=test3.t3.a -master-bin.000001 # Table_map 1 # table_id: # (test2.t2) -master-bin.000001 # Delete_rows_v1 1 # table_id: # flags: STMT_END_F -master-bin.000001 # Query 1 # COMMIT -master-bin.000001 # Rotate 1 # master-bin.000002;pos=4 +master-bin.000001 # Table_map # # table_id: # (test2.t2) +master-bin.000001 # Delete_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +master-bin.000001 # Rotate # # master-bin.000002;pos=POS # ##################################################################################### # mysqlbinlog --read-from-remote-server @@ -98,7 +123,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -106,7 +131,7 @@ DROP DATABASE IF EXISTS test1 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test2 +CREATE DATABASE test2 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl @@ -114,35 +139,11 @@ DROP DATABASE IF EXISTS test2 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test3 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl -/*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test2 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; CREATE DATABASE test3 /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; # at # @@ -164,8 +165,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -190,8 +191,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -216,8 +217,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -257,8 +258,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -283,8 +284,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -351,7 +352,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -362,24 +363,8 @@ DROP DATABASE IF EXISTS test1 /*!100001 SET @@session.gtid_seq_no=3*//*!*/; # at # # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 /*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -401,8 +386,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -414,8 +399,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -427,8 +412,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -458,8 +443,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -471,8 +456,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -523,7 +508,7 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP DATABASE IF EXISTS test1 +CREATE DATABASE test1 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-2 ddl @@ -531,7 +516,7 @@ DROP DATABASE IF EXISTS test1 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test2 +CREATE DATABASE test2 /*!*/; # at # #010909 4:46:40 server id # end_log_pos # GTID 0-1-3 ddl @@ -539,35 +524,11 @@ DROP DATABASE IF EXISTS test2 # at # #010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 SET TIMESTAMP=1000000000/*!*/; -DROP DATABASE IF EXISTS test3 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 ddl -/*!100001 SET @@session.gtid_seq_no=4*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test1 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 ddl -/*!100001 SET @@session.gtid_seq_no=5*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; -CREATE DATABASE test2 -/*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 ddl -/*!100001 SET @@session.gtid_seq_no=6*//*!*/; -# at # -#010909 4:46:40 server id # end_log_pos # Query thread_id=# exec_time=# error_code=0 -SET TIMESTAMP=1000000000/*!*/; CREATE DATABASE test3 /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 -/*!100001 SET @@session.gtid_seq_no=7*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-4 +/*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; # at # @@ -589,8 +550,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 -/*!100001 SET @@session.gtid_seq_no=8*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-5 +/*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; # at # @@ -612,8 +573,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 -/*!100001 SET @@session.gtid_seq_no=9*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-6 +/*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; # at # @@ -635,8 +596,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-10 -/*!100001 SET @@session.gtid_seq_no=10*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-7 +/*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; # at # @@ -671,8 +632,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-11 -/*!100001 SET @@session.gtid_seq_no=11*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-8 +/*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; # at # @@ -694,8 +655,8 @@ SET TIMESTAMP=1000000000/*!*/; COMMIT /*!*/; # at # -#010909 4:46:40 server id # end_log_pos # GTID 0-1-12 -/*!100001 SET @@session.gtid_seq_no=12*//*!*/; +#010909 4:46:40 server id # end_log_pos # GTID 0-1-9 +/*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; # at # @@ -723,3 +684,9 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; +set global binlog_checksum=@old_binlog_checksum; +DROP DATABASE test1; +DROP DATABASE test2; +DROP DATABASE test3; +DROP DATABASE xtest1; +DROP DATABASE xtest2; From b3d6cbc25a14cc73c5844b051d76f8967c653121 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Dec 2016 18:13:33 +0100 Subject: [PATCH 293/295] cleanup: binlog.binlog_killed_simulate --- .../binlog/r/binlog_killed_simulate.result | 23 ++++---------- .../binlog/t/binlog_killed_simulate.test | 30 ++++++------------- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/mysql-test/suite/binlog/r/binlog_killed_simulate.result b/mysql-test/suite/binlog/r/binlog_killed_simulate.result index b2c9a6f212f66..25b00bf87618d 100644 --- a/mysql-test/suite/binlog/r/binlog_killed_simulate.result +++ b/mysql-test/suite/binlog/r/binlog_killed_simulate.result @@ -1,16 +1,10 @@ -drop table if exists t1,t2; create table t1 (a int) engine=MyISAM; insert into t1 set a=1; reset master; update t1 set a=2 /* will be "killed" after work has been done */; -select -(@a:=load_file("MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog")) -is not null; -(@a:=load_file("MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog")) -is not null -1 -select 1 /* must return 1 as query completed before got killed*/; -1 +set @a:=load_file("MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog"); +select @a like '%#%error_code=0%' /* must return 1 as query completed before got killed*/; +@a like '%#%error_code=0%' 1 create table t2 (a int, b int) ENGINE=MyISAM; reset master; @@ -22,14 +16,9 @@ master-bin.000001 # Gtid # # BEGIN GTID #-#-# master-bin.000001 # Begin_load_query # # ;file_id=#;block_len=# master-bin.000001 # Execute_load_query # # use `test`; LOAD DATA INFILE '../../std_data/rpl_loaddata.dat' INTO TABLE `t2` FIELDS TERMINATED BY '\t' ENCLOSED BY '' ESCAPED BY '\\' LINES TERMINATED BY '\n' (`a`, `b`) ;file_id=# master-bin.000001 # Query # # COMMIT -select -(@a:=load_file("MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog")) -is not null; -(@a:=load_file("MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog")) -is not null -1 -select 0 /* must return 0 to mean the killed query is in */; -0 +set @a:=load_file("MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog"); +select @a like '%#%error_code=0%' /* must return 0 to mean the killed query is in */; +@a like '%#%error_code=0%' 0 drop table t1,t2; end of the tests diff --git a/mysql-test/suite/binlog/t/binlog_killed_simulate.test b/mysql-test/suite/binlog/t/binlog_killed_simulate.test index e1a1c9de3b2ef..20f517210b788 100644 --- a/mysql-test/suite/binlog/t/binlog_killed_simulate.test +++ b/mysql-test/suite/binlog/t/binlog_killed_simulate.test @@ -6,10 +6,6 @@ # Query_log_event::error_code # ---disable_warnings -drop table if exists t1,t2; ---enable_warnings - # # Checking that killing upon successful row-loop does not affect binlogging # @@ -21,20 +17,16 @@ reset master; update t1 set a=2 /* will be "killed" after work has been done */; # a proof the query is binlogged with no error -let $MYSQLD_DATADIR= `select @@datadir`; ---exec $MYSQL_BINLOG --force-if-open --start-position=$binlog_start_pos $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog +let datadir= `select @@datadir`; +--exec $MYSQL_BINLOG --force-if-open --start-position=$binlog_start_pos $datadir/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR -eval select -(@a:=load_file("$MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog")) -is not null; +eval set @a:=load_file("$MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog"); --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR -let $error_code= `select @a like "%#%error_code=0%" /* must return 1 */`; -eval select $error_code /* must return 1 as query completed before got killed*/; +select @a like '%#%error_code=0%' /* must return 1 as query completed before got killed*/; # cleanup for the sub-case remove_file $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog; - # # Checking that killing inside of row-loop for LOAD DATA into # non-transactional table affects binlogging @@ -46,21 +38,17 @@ reset master; load data infile '../../std_data/rpl_loaddata.dat' into table t2 /* will be "killed" in the middle */; # a proof the query is binlogged with an error ---let $binlog_load_data= query_get_value(SHOW BINLOG EVENTS, Pos, 5) ---let $binlog_end= query_get_value(SHOW BINLOG EVENTS, Pos, 6) +--let binlog_load_data= query_get_value(SHOW BINLOG EVENTS, Pos, 5) +--let binlog_end= query_get_value(SHOW BINLOG EVENTS, Pos, 6) source include/show_binlog_events.inc; - --mkdir $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571 ---exec $MYSQL_BINLOG --local-load=$MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571 --force-if-open --start-position=$binlog_load_data --stop-position=$binlog_end $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog +--exec $MYSQL_BINLOG --local-load=$MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571 --force-if-open --start-position=$binlog_load_data --stop-position=$binlog_end $datadir/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR -eval select -(@a:=load_file("$MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog")) -is not null; +eval set @a:=load_file("$MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog"); --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR -let $error_code= `select @a like "%#%error_code=0%" /* must return 0*/`; -eval select $error_code /* must return 0 to mean the killed query is in */; +select @a like '%#%error_code=0%' /* must return 0 to mean the killed query is in */; # cleanup for the sub-case remove_file $MYSQLTEST_VARDIR/tmp/binlog_killed_bug27571.binlog; From ab89359dde2f8fce2f1b7d233587804f459b9348 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 27 Dec 2016 19:40:07 +0100 Subject: [PATCH 294/295] enable tests that were skipped because of have_xtradb * some of these tests run just fine with InnoDB: -> s/have_xtradb/have_innodb/ * sys_var tests did basic tests for xtradb only variables -> remove them, they're useless anyway (sysvar_innodb does it better) * multi_update had innodb specific tests -> move to multi_update_innodb.test --- mysql-test/include/have_xtradb.combinations | 10 +- mysql-test/r/index_merge_innodb.result | 8 +- mysql-test/r/information_schema.result | 13 +- .../r/information_schema_all_engines.result | 44 ++++++- mysql-test/r/multi_update.result | 94 ------------- mysql-test/r/multi_update_innodb.result | 84 ++++++++++++ mysql-test/r/xtradb_mrr.result | 2 +- .../suite/encryption/t/filekeys_badtest.inc | 2 +- .../suite/encryption/t/filekeys_goodtest.inc | 2 +- mysql-test/suite/encryption/t/tempfiles.test | 4 +- .../suite/funcs_1/r/is_columns_is.result | 8 +- mysql-test/suite/funcs_1/t/is_columns_is.test | 2 +- ...nnodb_cleaner_eviction_factor_basic.result | 31 ----- ...nodb_cleaner_flush_chunk_size_basic.result | 31 ----- .../innodb_cleaner_free_list_lwm_basic.result | 35 ----- ...innodb_cleaner_lru_chunk_size_basic.result | 31 ----- ...innodb_cleaner_lsn_age_factor_basic.result | 21 --- ...innodb_cleaner_max_flush_time_basic.result | 25 ---- .../innodb_cleaner_max_lru_time_basic.result | 25 ---- ...odb_empty_free_list_algorithm_basic.result | 23 ---- .../r/innodb_foreground_preflush_basic.result | 23 ---- .../r/innodb_log_arch_dir_basic.result | 38 ------ .../r/innodb_log_arch_expire_sec_basic.result | 38 ------ .../r/innodb_log_archive_basic.result | 38 ------ ...innodb_log_checksum_algorithm_basic.result | 47 ------- .../r/innodb_priority_cleaner_basic.result | 31 ----- .../r/innodb_priority_io_basic.result | 31 ----- .../r/innodb_priority_master_basic.result | 31 ----- .../r/innodb_priority_purge_basic.result | 31 ----- .../innodb_cleaner_eviction_factor_basic.test | 35 ----- ...innodb_cleaner_flush_chunk_size_basic.test | 33 ----- .../t/innodb_cleaner_free_list_lwm_basic.test | 35 ----- .../innodb_cleaner_lru_chunk_size_basic.test | 33 ----- .../innodb_cleaner_lsn_age_factor_basic.test | 28 ---- .../innodb_cleaner_max_flush_time_basic.test | 31 ----- .../t/innodb_cleaner_max_lru_time_basic.test | 31 ----- ...innodb_empty_free_list_algorithm_basic.opt | 1 - ...nnodb_empty_free_list_algorithm_basic.test | 30 ----- .../t/innodb_foreground_preflush_basic.test | 30 ----- .../sys_vars/t/innodb_log_arch_dir_basic.test | 68 ---------- .../t/innodb_log_arch_expire_sec_basic.test | 60 --------- .../sys_vars/t/innodb_log_archive_basic.test | 61 --------- .../innodb_log_checksum_algorithm_basic.test | 38 ------ .../t/innodb_priority_cleaner_basic.test | 36 ----- .../sys_vars/t/innodb_priority_io_basic.test | 36 ----- .../t/innodb_priority_master_basic.test | 36 ----- .../t/innodb_priority_purge_basic.test | 36 ----- mysql-test/t/create_or_replace2.test | 2 +- mysql-test/t/index_merge_innodb.test | 2 +- mysql-test/t/information_schema.test | 3 +- .../t/information_schema_all_engines.test | 2 +- mysql-test/t/multi_update.test | 124 ------------------ mysql-test/t/multi_update_innodb.test | 98 ++++++++++++++ mysql-test/t/progress_976225.test | 2 +- mysql-test/t/rowid_order_innodb.test | 2 +- mysql-test/t/xtradb_mrr.test | 2 +- .../tokudb_mariadb/t/xa-recovery-9214.test | 2 +- 57 files changed, 251 insertions(+), 1449 deletions(-) delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_eviction_factor_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_flush_chunk_size_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_free_list_lwm_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_lru_chunk_size_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_lsn_age_factor_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_max_flush_time_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_cleaner_max_lru_time_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_empty_free_list_algorithm_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_foreground_preflush_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_log_arch_dir_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_log_arch_expire_sec_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_log_archive_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_log_checksum_algorithm_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_priority_cleaner_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_priority_io_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_priority_master_basic.result delete mode 100644 mysql-test/suite/sys_vars/r/innodb_priority_purge_basic.result delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_eviction_factor_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_flush_chunk_size_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_free_list_lwm_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_lru_chunk_size_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_lsn_age_factor_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_max_flush_time_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_cleaner_max_lru_time_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.opt delete mode 100644 mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_foreground_preflush_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_log_arch_dir_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_log_arch_expire_sec_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_log_archive_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_log_checksum_algorithm_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_priority_cleaner_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_priority_io_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_priority_master_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_priority_purge_basic.test diff --git a/mysql-test/include/have_xtradb.combinations b/mysql-test/include/have_xtradb.combinations index 3454f83cb4d65..0419dc91171c9 100644 --- a/mysql-test/include/have_xtradb.combinations +++ b/mysql-test/include/have_xtradb.combinations @@ -6,8 +6,8 @@ innodb-cmpmem innodb-trx innodb-sys-indexes -#[xtradb] -#innodb -#innodb-cmpmem -#innodb-trx -#innodb-sys-indexes +[xtradb] +innodb +innodb-cmpmem +innodb-trx +innodb-sys-indexes diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 5bf56e213abbb..0450622411ac2 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -552,7 +552,7 @@ primary key (pk1, pk2) ); explain select * from t1 where pk1 = 1 and pk2 < 80 and key1=0; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 PRIMARY 8 NULL 9 Using where +1 SIMPLE t1 range PRIMARY,key1 PRIMARY 8 NULL 10 Using where select * from t1 where pk1 = 1 and pk2 < 80 and key1=0; pk1 pk2 key1 key2 pktail1ok pktail2ok pktail3bad pktail4bad pktail5bad pk2copy badkey filler1 filler2 1 10 0 0 0 0 0 0 0 10 0 filler-data-10 filler2 @@ -597,7 +597,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index_merge key1,pktail2ok key1,pktail2ok 4,4 NULL 1 Using intersect(key1,pktail2ok); Using where explain select * from t1 where (pktail2ok=1 and pk1< 50000) or key1=10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok pktail2ok,key1 8,4 NULL 199 Using sort_union(pktail2ok,key1); Using where +1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok pktail2ok,key1 8,4 NULL 200 Using sort_union(pktail2ok,key1); Using where explain select * from t1 where pktail3bad=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref key1,pktail3bad key1 4 const 100 Using where @@ -699,8 +699,8 @@ SELECT COUNT(*) FROM (SELECT * FROM t1 FORCE INDEX(primary,idx) WHERE a BETWEEN 2 AND 7 OR pk=1000000) AS t; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY ALL NULL NULL NULL NULL 6144 -2 DERIVED t1 index_merge PRIMARY,idx idx,PRIMARY 5,4 NULL 6144 Using sort_union(idx,PRIMARY); Using where +1 PRIMARY ALL NULL NULL NULL NULL 6145 +2 DERIVED t1 index_merge PRIMARY,idx idx,PRIMARY 5,4 NULL 6145 Using sort_union(idx,PRIMARY); Using where SELECT COUNT(*) FROM (SELECT * FROM t1 FORCE INDEX(primary,idx) WHERE a BETWEEN 2 AND 7 OR pk=1000000) AS t; diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index 0c5356c1a913b..ce43e9f8a6bcd 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -455,10 +455,10 @@ Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_par select * from information_schema.views where TABLE_NAME like "v%"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME VIEW_DEFINITION CHECK_OPTION IS_UPDATABLE DEFINER SECURITY_TYPE CHARACTER_SET_CLIENT COLLATION_CONNECTION ALGORITHM def test v0 select `information_schema`.`schemata`.`SCHEMA_NAME` AS `c` from `information_schema`.`schemata` NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED -def test v1 select `information_schema`.`tables`.`TABLE_NAME` AS `c` from `information_schema`.`tables` where (`information_schema`.`tables`.`TABLE_NAME` = 'v1') NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED -def test v2 select `information_schema`.`columns`.`COLUMN_NAME` AS `c` from `information_schema`.`columns` where (`information_schema`.`columns`.`TABLE_NAME` = 'v2') NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED -def test v3 select `information_schema`.`character_sets`.`CHARACTER_SET_NAME` AS `c` from `information_schema`.`character_sets` where (`information_schema`.`character_sets`.`CHARACTER_SET_NAME` like 'latin1%') NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED -def test v4 select `information_schema`.`collations`.`COLLATION_NAME` AS `c` from `information_schema`.`collations` where (`information_schema`.`collations`.`COLLATION_NAME` like 'latin1%') NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED +def test v1 select `information_schema`.`tables`.`TABLE_NAME` AS `c` from `information_schema`.`tables` where `information_schema`.`tables`.`TABLE_NAME` = 'v1' NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED +def test v2 select `information_schema`.`columns`.`COLUMN_NAME` AS `c` from `information_schema`.`columns` where `information_schema`.`columns`.`TABLE_NAME` = 'v2' NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED +def test v3 select `information_schema`.`character_sets`.`CHARACTER_SET_NAME` AS `c` from `information_schema`.`character_sets` where `information_schema`.`character_sets`.`CHARACTER_SET_NAME` like 'latin1%' NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED +def test v4 select `information_schema`.`collations`.`COLLATION_NAME` AS `c` from `information_schema`.`collations` where `information_schema`.`collations`.`COLLATION_NAME` like 'latin1%' NONE NO root@localhost DEFINER latin1 latin1_swedish_ci UNDEFINED drop view v0, v1, v2, v3, v4; create table t1 (a int); grant select,update,insert on t1 to mysqltest_1@localhost; @@ -1247,7 +1247,6 @@ table_schema='information_schema' and or column_type = 'varchar(27)') group by column_type order by num; column_type group_concat(table_schema, '.', table_name) num -varchar(27) information_schema.COLUMNS 1 varchar(7) information_schema.ROUTINES,information_schema.VIEWS 2 varchar(20) information_schema.ALL_PLUGINS,information_schema.ALL_PLUGINS,information_schema.ALL_PLUGINS,information_schema.FILES,information_schema.FILES,information_schema.PLUGINS,information_schema.PLUGINS,information_schema.PLUGINS,information_schema.PROFILING 9 create table t1(f1 char(1) not null, f2 char(9) not null) @@ -2062,7 +2061,7 @@ where (table_schema = "osm") and (table_name = "test"); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE columns ALL NULL TABLE_SCHEMA,TABLE_NAME NULL NULL NULL NULL Using where; Open_frm_only; Scanned 0 databases Warnings: -Note 1003 select `information_schema`.`columns`.`COLUMN_NAME` AS `column_name` from `information_schema`.`columns` where ((`information_schema`.`columns`.`TABLE_SCHEMA` = 'osm') and (`information_schema`.`columns`.`TABLE_NAME` = 'test')) +Note 1003 select `information_schema`.`columns`.`COLUMN_NAME` AS `column_name` from `information_schema`.`columns` where `information_schema`.`columns`.`TABLE_SCHEMA` = 'osm' and `information_schema`.`columns`.`TABLE_NAME` = 'test' explain extended select information_schema.columns.column_name as column_name from information_schema.columns @@ -2070,7 +2069,7 @@ where (information_schema.columns.table_schema = 'osm') and (information_schema. id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE columns ALL NULL TABLE_SCHEMA,TABLE_NAME NULL NULL NULL NULL Using where; Open_frm_only; Scanned 0 databases Warnings: -Note 1003 select `information_schema`.`columns`.`COLUMN_NAME` AS `column_name` from `information_schema`.`columns` where ((`information_schema`.`columns`.`TABLE_SCHEMA` = 'osm') and (`information_schema`.`columns`.`TABLE_NAME` = 'test')) +Note 1003 select `information_schema`.`columns`.`COLUMN_NAME` AS `column_name` from `information_schema`.`columns` where `information_schema`.`columns`.`TABLE_SCHEMA` = 'osm' and `information_schema`.`columns`.`TABLE_NAME` = 'test' drop view v1; # # Clean-up. diff --git a/mysql-test/r/information_schema_all_engines.result b/mysql-test/r/information_schema_all_engines.result index 7f79dd883e22e..126a6f4bccd41 100644 --- a/mysql-test/r/information_schema_all_engines.result +++ b/mysql-test/r/information_schema_all_engines.result @@ -17,13 +17,17 @@ GEOMETRY_COLUMNS GLOBAL_STATUS GLOBAL_VARIABLES INDEX_STATISTICS -INNODB_CHANGED_PAGES +INNODB_BUFFER_PAGE +INNODB_BUFFER_PAGE_LRU +INNODB_BUFFER_POOL_STATS INNODB_CMP INNODB_CMPMEM INNODB_CMPMEM_RESET INNODB_CMP_PER_INDEX INNODB_CMP_RESET +INNODB_LOCKS INNODB_LOCK_WAITS +INNODB_METRICS INNODB_MUTEXES INNODB_SYS_COLUMNS INNODB_SYS_FIELDS @@ -32,6 +36,7 @@ INNODB_SYS_FOREIGN_COLS INNODB_SYS_INDEXES INNODB_SYS_TABLES INNODB_SYS_TABLESTATS +INNODB_SYS_VIRTUAL INNODB_TABLESPACES_ENCRYPTION INNODB_TABLESPACES_SCRUBBING INNODB_TRX @@ -91,13 +96,17 @@ GEOMETRY_COLUMNS F_TABLE_SCHEMA GLOBAL_STATUS VARIABLE_NAME GLOBAL_VARIABLES VARIABLE_NAME INDEX_STATISTICS TABLE_SCHEMA -INNODB_CHANGED_PAGES space_id +INNODB_BUFFER_PAGE POOL_ID +INNODB_BUFFER_PAGE_LRU POOL_ID +INNODB_BUFFER_POOL_STATS POOL_ID INNODB_CMP page_size INNODB_CMPMEM page_size INNODB_CMPMEM_RESET page_size INNODB_CMP_PER_INDEX database_name INNODB_CMP_RESET page_size +INNODB_LOCKS lock_id INNODB_LOCK_WAITS requesting_trx_id +INNODB_METRICS NAME INNODB_MUTEXES NAME INNODB_SYS_COLUMNS TABLE_ID INNODB_SYS_FIELDS INDEX_ID @@ -106,6 +115,7 @@ INNODB_SYS_FOREIGN_COLS ID INNODB_SYS_INDEXES INDEX_ID INNODB_SYS_TABLES TABLE_ID INNODB_SYS_TABLESTATS TABLE_ID +INNODB_SYS_VIRTUAL TABLE_ID INNODB_TABLESPACES_ENCRYPTION SPACE INNODB_TABLESPACES_SCRUBBING SPACE INNODB_TRX trx_id @@ -165,13 +175,17 @@ GEOMETRY_COLUMNS F_TABLE_SCHEMA GLOBAL_STATUS VARIABLE_NAME GLOBAL_VARIABLES VARIABLE_NAME INDEX_STATISTICS TABLE_SCHEMA -INNODB_CHANGED_PAGES space_id +INNODB_BUFFER_PAGE POOL_ID +INNODB_BUFFER_PAGE_LRU POOL_ID +INNODB_BUFFER_POOL_STATS POOL_ID INNODB_CMP page_size INNODB_CMPMEM page_size INNODB_CMPMEM_RESET page_size INNODB_CMP_PER_INDEX database_name INNODB_CMP_RESET page_size +INNODB_LOCKS lock_id INNODB_LOCK_WAITS requesting_trx_id +INNODB_METRICS NAME INNODB_MUTEXES NAME INNODB_SYS_COLUMNS TABLE_ID INNODB_SYS_FIELDS INDEX_ID @@ -180,6 +194,7 @@ INNODB_SYS_FOREIGN_COLS ID INNODB_SYS_INDEXES INDEX_ID INNODB_SYS_TABLES TABLE_ID INNODB_SYS_TABLESTATS TABLE_ID +INNODB_SYS_VIRTUAL TABLE_ID INNODB_TABLESPACES_ENCRYPTION SPACE INNODB_TABLESPACES_SCRUBBING SPACE INNODB_TRX trx_id @@ -244,13 +259,17 @@ GEOMETRY_COLUMNS information_schema.GEOMETRY_COLUMNS 1 GLOBAL_STATUS information_schema.GLOBAL_STATUS 1 GLOBAL_VARIABLES information_schema.GLOBAL_VARIABLES 1 INDEX_STATISTICS information_schema.INDEX_STATISTICS 1 -INNODB_CHANGED_PAGES information_schema.INNODB_CHANGED_PAGES 1 +INNODB_BUFFER_PAGE information_schema.INNODB_BUFFER_PAGE 1 +INNODB_BUFFER_PAGE_LRU information_schema.INNODB_BUFFER_PAGE_LRU 1 +INNODB_BUFFER_POOL_STATS information_schema.INNODB_BUFFER_POOL_STATS 1 INNODB_CMP information_schema.INNODB_CMP 1 INNODB_CMPMEM information_schema.INNODB_CMPMEM 1 INNODB_CMPMEM_RESET information_schema.INNODB_CMPMEM_RESET 1 INNODB_CMP_PER_INDEX information_schema.INNODB_CMP_PER_INDEX 1 INNODB_CMP_RESET information_schema.INNODB_CMP_RESET 1 +INNODB_LOCKS information_schema.INNODB_LOCKS 1 INNODB_LOCK_WAITS information_schema.INNODB_LOCK_WAITS 1 +INNODB_METRICS information_schema.INNODB_METRICS 1 INNODB_MUTEXES information_schema.INNODB_MUTEXES 1 INNODB_SYS_COLUMNS information_schema.INNODB_SYS_COLUMNS 1 INNODB_SYS_FIELDS information_schema.INNODB_SYS_FIELDS 1 @@ -259,6 +278,7 @@ INNODB_SYS_FOREIGN_COLS information_schema.INNODB_SYS_FOREIGN_COLS 1 INNODB_SYS_INDEXES information_schema.INNODB_SYS_INDEXES 1 INNODB_SYS_TABLES information_schema.INNODB_SYS_TABLES 1 INNODB_SYS_TABLESTATS information_schema.INNODB_SYS_TABLESTATS 1 +INNODB_SYS_VIRTUAL information_schema.INNODB_SYS_VIRTUAL 1 INNODB_TABLESPACES_ENCRYPTION information_schema.INNODB_TABLESPACES_ENCRYPTION 1 INNODB_TABLESPACES_SCRUBBING information_schema.INNODB_TABLESPACES_SCRUBBING 1 INNODB_TRX information_schema.INNODB_TRX 1 @@ -308,13 +328,17 @@ Database: information_schema | GLOBAL_STATUS | | GLOBAL_VARIABLES | | INDEX_STATISTICS | -| INNODB_CHANGED_PAGES | +| INNODB_BUFFER_PAGE | +| INNODB_BUFFER_PAGE_LRU | +| INNODB_BUFFER_POOL_STATS | | INNODB_CMP | | INNODB_CMPMEM | | INNODB_CMPMEM_RESET | | INNODB_CMP_PER_INDEX | | INNODB_CMP_RESET | +| INNODB_LOCKS | | INNODB_LOCK_WAITS | +| INNODB_METRICS | | INNODB_MUTEXES | | INNODB_SYS_COLUMNS | | INNODB_SYS_FIELDS | @@ -323,6 +347,7 @@ Database: information_schema | INNODB_SYS_INDEXES | | INNODB_SYS_TABLES | | INNODB_SYS_TABLESTATS | +| INNODB_SYS_VIRTUAL | | INNODB_TABLESPACES_ENCRYPTION | | INNODB_TABLESPACES_SCRUBBING | | INNODB_TRX | @@ -372,13 +397,17 @@ Database: INFORMATION_SCHEMA | GLOBAL_STATUS | | GLOBAL_VARIABLES | | INDEX_STATISTICS | -| INNODB_CHANGED_PAGES | +| INNODB_BUFFER_PAGE | +| INNODB_BUFFER_PAGE_LRU | +| INNODB_BUFFER_POOL_STATS | | INNODB_CMP | | INNODB_CMPMEM | | INNODB_CMPMEM_RESET | | INNODB_CMP_PER_INDEX | | INNODB_CMP_RESET | +| INNODB_LOCKS | | INNODB_LOCK_WAITS | +| INNODB_METRICS | | INNODB_MUTEXES | | INNODB_SYS_COLUMNS | | INNODB_SYS_FIELDS | @@ -387,6 +416,7 @@ Database: INFORMATION_SCHEMA | INNODB_SYS_INDEXES | | INNODB_SYS_TABLES | | INNODB_SYS_TABLESTATS | +| INNODB_SYS_VIRTUAL | | INNODB_TABLESPACES_ENCRYPTION | | INNODB_TABLESPACES_SCRUBBING | | INNODB_TRX | @@ -423,5 +453,5 @@ Wildcard: inf_rmation_schema | information_schema | SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') GROUP BY TABLE_SCHEMA; table_schema count(*) -information_schema 59 +information_schema 64 mysql 30 diff --git a/mysql-test/r/multi_update.result b/mysql-test/r/multi_update.result index b9c12be8d0d96..1ae05a8cff9c9 100644 --- a/mysql-test/r/multi_update.result +++ b/mysql-test/r/multi_update.result @@ -1,10 +1,4 @@ CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); -drop table if exists t1,t2,t3; -drop database if exists mysqltest; -drop view if exists v1; -revoke all privileges on mysqltest.t1 from mysqltest_1@localhost; -revoke all privileges on mysqltest.* from mysqltest_1@localhost; -delete from mysql.user where user=_binary'mysqltest_1'; create table t1(id1 int not null auto_increment primary key, t char(12)); create table t2(id2 int not null, t char(12)); create table t3(id3 int not null, t char(12), index(id3)); @@ -452,44 +446,11 @@ ERROR HY000: Table 't1' is specified twice, both as a target for 'UPDATE' and as delete t1 from t1,t2 where t1.col1 < (select max(col1) from t1) and t1.col1 = t2.col1; ERROR HY000: Table 't1' is specified twice, both as a target for 'DELETE' and as a separate source for data drop table t1,t2; -create table t1 ( -aclid bigint not null primary key, -status tinyint(1) not null -) engine = innodb; -create table t2 ( -refid bigint not null primary key, -aclid bigint, index idx_acl(aclid) -) engine = innodb; -insert into t2 values(1,null); -delete t2, t1 from t2 left join t1 on (t2.aclid=t1.aclid) where t2.refid='1'; -drop table t1, t2; create table t1(a int); create table t2(a int); delete from t1,t2 using t1,t2 where t1.a=(select a from t1); ERROR HY000: Table 't1' is specified twice, both as a target for 'DELETE' and as a separate source for data drop table t1, t2; -create table t1 ( c char(8) not null ) engine=innodb; -insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); -insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); -alter table t1 add b char(8) not null; -alter table t1 add a char(8) not null; -alter table t1 add primary key (a,b,c); -update t1 set a=c, b=c; -create table t2 like t1; -insert into t2 select * from t1; -delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; -drop table t1,t2; -create table t1 ( c char(8) not null ) engine=innodb; -insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); -insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); -alter table t1 add b char(8) not null; -alter table t1 add a char(8) not null; -alter table t1 add primary key (a,b,c); -update t1 set a=c, b=c; -create table t2 like t1; -insert into t2 select * from t1; -delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; -drop table t1,t2; create table t1 (a int, b int); insert into t1 values (1, 2), (2, 3), (3, 4); create table t2 (a int); @@ -646,7 +607,6 @@ master-bin.000001 # Update_rows_v1 # # table_id: # flags: STMT_END_F master-bin.000001 # Query # # COMMIT drop table t1, t2; set @@session.binlog_format= @sav_binlog_format; -drop table if exists t1, t2, t3; CREATE TABLE t1 (a int, PRIMARY KEY (a)); CREATE TABLE t2 (a int, PRIMARY KEY (a)); CREATE TABLE t3 (a int, PRIMARY KEY (a)) ENGINE=MyISAM; @@ -1040,57 +1000,3 @@ deallocate prepare stmt1; drop view v3,v2,v1; drop table t1,t2,t3; end of 5.5 tests - -# Bug mdev-5970 -# Bug#13256831 - ERROR 1032 (HY000): CAN'T FIND RECORD - -CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; -CREATE TABLE t2 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; -INSERT INTO t1 VALUES (5, 7); -INSERT INTO t2 VALUES (6, 97); -CREATE ALGORITHM = MERGE VIEW v1 AS -SELECT a2.f1 AS f1, a2.f2 AS f2 -FROM t1 AS a1 JOIN t2 AS a2 ON a1.f2 > a2.f1 -WITH LOCAL CHECK OPTION; -SELECT * FROM v1; -f1 f2 -6 97 -UPDATE v1 SET f1 = 1; -SELECT * FROM v1; -f1 f2 -1 97 -DROP TABLE t1, t2; -DROP VIEW v1; -# -# MDEV-5973: MySQL Bug#11757486:49539: NON-DESCRIPTIVE ERR (ERROR 0 -# FROM STORAGE ENGINE) WITH MULTI-TABLE UPDATE -# -CREATE TABLE table_11757486 (field1 tinyint) ENGINE=INNODB; -INSERT INTO table_11757486 VALUES (0),(0); -SET SESSION SQL_MODE='STRICT_ALL_TABLES'; -UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; -Warnings: -Warning 1264 Out of range value for column 'field1' at row 1 -Warning 1264 Out of range value for column 'field1' at row 2 -UPDATE IGNORE table_11757486 SET field1=128; -Warnings: -Warning 1264 Out of range value for column 'field1' at row 1 -Warning 1264 Out of range value for column 'field1' at row 2 -Note 1592 Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. UPDATE IGNORE is unsafe because the order in which rows are updated determines which (if any) rows are ignored. This order cannot be predicted and may differ on master and the slave. -UPDATE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; -ERROR 22003: Out of range value for column 'field1' at row 1 -UPDATE table_11757486 SET field1=128; -ERROR 22003: Out of range value for column 'field1' at row 1 -SET SESSION SQL_MODE=''; -UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; -Warnings: -Warning 1264 Out of range value for column 'field1' at row 1 -Warning 1264 Out of range value for column 'field1' at row 2 -UPDATE IGNORE table_11757486 SET field1=128; -Warnings: -Warning 1264 Out of range value for column 'field1' at row 1 -Warning 1264 Out of range value for column 'field1' at row 2 -Note 1592 Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. UPDATE IGNORE is unsafe because the order in which rows are updated determines which (if any) rows are ignored. This order cannot be predicted and may differ on master and the slave. -DROP TABLE table_11757486; -SET SESSION SQL_MODE=default; -end of 10.0 tests diff --git a/mysql-test/r/multi_update_innodb.result b/mysql-test/r/multi_update_innodb.result index 4c4aa9f4684c3..5890fd24f5f8d 100644 --- a/mysql-test/r/multi_update_innodb.result +++ b/mysql-test/r/multi_update_innodb.result @@ -67,3 +67,87 @@ SELECT * FROM t2; col_int_key pk_1 pk_2 col_int 1 2 3 4 DROP TABLE t1,t2; + +# Bug mdev-5970 +# Bug#13256831 - ERROR 1032 (HY000): CAN'T FIND RECORD + +CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +CREATE TABLE t2 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (5, 7); +INSERT INTO t2 VALUES (6, 97); +CREATE ALGORITHM = MERGE VIEW v1 AS +SELECT a2.f1 AS f1, a2.f2 AS f2 +FROM t1 AS a1 JOIN t2 AS a2 ON a1.f2 > a2.f1 +WITH LOCAL CHECK OPTION; +SELECT * FROM v1; +f1 f2 +6 97 +UPDATE v1 SET f1 = 1; +SELECT * FROM v1; +f1 f2 +1 97 +DROP TABLE t1, t2; +DROP VIEW v1; +# +# MDEV-5973: MySQL Bug#11757486:49539: NON-DESCRIPTIVE ERR (ERROR 0 +# FROM STORAGE ENGINE) WITH MULTI-TABLE UPDATE +# +CREATE TABLE table_11757486 (field1 tinyint) ENGINE=INNODB; +INSERT INTO table_11757486 VALUES (0),(0); +SET SESSION SQL_MODE='STRICT_ALL_TABLES'; +UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; +Warnings: +Warning 1264 Out of range value for column 'field1' at row 1 +Warning 1264 Out of range value for column 'field1' at row 2 +UPDATE IGNORE table_11757486 SET field1=128; +Warnings: +Warning 1264 Out of range value for column 'field1' at row 1 +Warning 1264 Out of range value for column 'field1' at row 2 +UPDATE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; +ERROR 22003: Out of range value for column 'field1' at row 1 +UPDATE table_11757486 SET field1=128; +ERROR 22003: Out of range value for column 'field1' at row 1 +SET SESSION SQL_MODE=''; +UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; +Warnings: +Warning 1264 Out of range value for column 'field1' at row 1 +Warning 1264 Out of range value for column 'field1' at row 2 +UPDATE IGNORE table_11757486 SET field1=128; +Warnings: +Warning 1264 Out of range value for column 'field1' at row 1 +Warning 1264 Out of range value for column 'field1' at row 2 +DROP TABLE table_11757486; +SET SESSION SQL_MODE=default; +create table t1 ( +aclid bigint not null primary key, +status tinyint(1) not null +) engine = innodb; +create table t2 ( +refid bigint not null primary key, +aclid bigint, index idx_acl(aclid) +) engine = innodb; +insert into t2 values(1,null); +delete t2, t1 from t2 left join t1 on (t2.aclid=t1.aclid) where t2.refid='1'; +drop table t1, t2; +create table t1 ( c char(8) not null ) engine=innodb; +insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); +insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); +alter table t1 add b char(8) not null; +alter table t1 add a char(8) not null; +alter table t1 add primary key (a,b,c); +update t1 set a=c, b=c; +create table t2 like t1; +insert into t2 select * from t1; +delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; +drop table t1,t2; +create table t1 ( c char(8) not null ) engine=innodb; +insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); +insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); +alter table t1 add b char(8) not null; +alter table t1 add a char(8) not null; +alter table t1 add primary key (a,b,c); +update t1 set a=c, b=c; +create table t2 like t1; +insert into t2 select * from t1; +delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; +drop table t1,t2; diff --git a/mysql-test/r/xtradb_mrr.result b/mysql-test/r/xtradb_mrr.result index c238d0530af61..db90ada80534e 100644 --- a/mysql-test/r/xtradb_mrr.result +++ b/mysql-test/r/xtradb_mrr.result @@ -313,7 +313,7 @@ from t1 A, t1 B, t1 C; explain select count(length(a) + length(filler)) from t2 force index (a) where a>='a-1000-a' and a <'a-1001-a'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range a a 9 NULL 99 Using index condition; Rowid-ordered scan +1 SIMPLE t2 range a a 9 NULL 100 Using index condition; Rowid-ordered scan select count(length(a) + length(filler)) from t2 force index (a) where a>='a-1000-a' and a <'a-1001-a'; count(length(a) + length(filler)) 100 diff --git a/mysql-test/suite/encryption/t/filekeys_badtest.inc b/mysql-test/suite/encryption/t/filekeys_badtest.inc index 7c0858af20a4a..1cdea0e1a5376 100644 --- a/mysql-test/suite/encryption/t/filekeys_badtest.inc +++ b/mysql-test/suite/encryption/t/filekeys_badtest.inc @@ -1,5 +1,5 @@ -- source include/not_embedded.inc --- source include/have_xtradb.inc +-- source include/have_innodb.inc -- source filekeys_plugin.inc --eval call mtr.add_suppression("$SEARCH_PATTERN") diff --git a/mysql-test/suite/encryption/t/filekeys_goodtest.inc b/mysql-test/suite/encryption/t/filekeys_goodtest.inc index 146a570412c40..12a79c13d2a76 100644 --- a/mysql-test/suite/encryption/t/filekeys_goodtest.inc +++ b/mysql-test/suite/encryption/t/filekeys_goodtest.inc @@ -1,4 +1,4 @@ --- source include/have_xtradb.inc +-- source include/have_innodb.inc -- source filekeys_plugin.inc create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; diff --git a/mysql-test/suite/encryption/t/tempfiles.test b/mysql-test/suite/encryption/t/tempfiles.test index 34dcbdf5963cd..065d775c18261 100644 --- a/mysql-test/suite/encryption/t/tempfiles.test +++ b/mysql-test/suite/encryption/t/tempfiles.test @@ -7,9 +7,7 @@ source include/have_sequence.inc; # Row binlog format to fill binlog cache faster source include/have_binlog_format_row.inc; -# Nothing XtraDB specific in this test, it just needs *some* transactional -# engine. But there's no need to run it twice for InnoDB and XtraDB. -source include/have_xtradb.inc; +source include/have_innodb.inc; # # MyISAM messing around with IO_CACHE::file diff --git a/mysql-test/suite/funcs_1/r/is_columns_is.result b/mysql-test/suite/funcs_1/r/is_columns_is.result index 0f35e66d6a6db..04619a3e3c88c 100644 --- a/mysql-test/suite/funcs_1/r/is_columns_is.result +++ b/mysql-test/suite/funcs_1/r/is_columns_is.result @@ -68,7 +68,7 @@ def information_schema COLUMNS COLUMN_NAME 4 NO varchar 64 192 NULL NULL NULL u def information_schema COLUMNS COLUMN_TYPE 16 NO longtext 4294967295 4294967295 NULL NULL NULL utf8 utf8_general_ci longtext select def information_schema COLUMNS DATA_TYPE 8 NO varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select def information_schema COLUMNS DATETIME_PRECISION 13 NULL YES bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned select -def information_schema COLUMNS EXTRA 18 NO varchar 27 81 NULL NULL NULL utf8 utf8_general_ci varchar(27) select +def information_schema COLUMNS EXTRA 18 NO varchar 30 90 NULL NULL NULL utf8 utf8_general_ci varchar(30) select def information_schema COLUMNS IS_NULLABLE 7 NO varchar 3 9 NULL NULL NULL utf8 utf8_general_ci varchar(3) select def information_schema COLUMNS NUMERIC_PRECISION 11 NULL YES bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned select def information_schema COLUMNS NUMERIC_SCALE 12 NULL YES bigint NULL NULL 20 0 NULL NULL NULL bigint(21) unsigned select @@ -128,7 +128,7 @@ def information_schema FILES ENGINE 10 NO varchar 64 192 NULL NULL NULL utf8 ut def information_schema FILES EXTENT_SIZE 16 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(4) select def information_schema FILES EXTRA 38 NULL YES varchar 255 765 NULL NULL NULL utf8 utf8_general_ci varchar(255) select def information_schema FILES FILE_ID 1 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(4) select -def information_schema FILES FILE_NAME 2 NULL YES varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select +def information_schema FILES FILE_NAME 2 NULL YES varchar 512 1536 NULL NULL NULL utf8 utf8_general_ci varchar(512) select def information_schema FILES FILE_TYPE 3 NO varchar 20 60 NULL NULL NULL utf8 utf8_general_ci varchar(20) select def information_schema FILES FREE_EXTENTS 14 NULL YES bigint NULL NULL 19 0 NULL NULL NULL bigint(4) select def information_schema FILES FULLTEXT_KEYS 11 NULL YES varchar 64 192 NULL NULL NULL utf8 utf8_general_ci varchar(64) select @@ -602,7 +602,7 @@ NULL information_schema COLUMNS DATETIME_PRECISION bigint NULL NULL NULL NULL bi 3.0000 information_schema COLUMNS COLLATION_NAME varchar 32 96 utf8 utf8_general_ci varchar(32) 1.0000 information_schema COLUMNS COLUMN_TYPE longtext 4294967295 4294967295 utf8 utf8_general_ci longtext 3.0000 information_schema COLUMNS COLUMN_KEY varchar 3 9 utf8 utf8_general_ci varchar(3) -3.0000 information_schema COLUMNS EXTRA varchar 27 81 utf8 utf8_general_ci varchar(27) +3.0000 information_schema COLUMNS EXTRA varchar 30 90 utf8 utf8_general_ci varchar(30) 3.0000 information_schema COLUMNS PRIVILEGES varchar 80 240 utf8 utf8_general_ci varchar(80) 3.0000 information_schema COLUMNS COLUMN_COMMENT varchar 1024 3072 utf8 utf8_general_ci varchar(1024) 3.0000 information_schema COLUMN_PRIVILEGES GRANTEE varchar 190 570 utf8 utf8_general_ci varchar(190) @@ -644,7 +644,7 @@ NULL information_schema EVENTS ORIGINATOR bigint NULL NULL NULL NULL bigint(10) 3.0000 information_schema EVENTS COLLATION_CONNECTION varchar 32 96 utf8 utf8_general_ci varchar(32) 3.0000 information_schema EVENTS DATABASE_COLLATION varchar 32 96 utf8 utf8_general_ci varchar(32) NULL information_schema FILES FILE_ID bigint NULL NULL NULL NULL bigint(4) -3.0000 information_schema FILES FILE_NAME varchar 64 192 utf8 utf8_general_ci varchar(64) +3.0000 information_schema FILES FILE_NAME varchar 512 1536 utf8 utf8_general_ci varchar(512) 3.0000 information_schema FILES FILE_TYPE varchar 20 60 utf8 utf8_general_ci varchar(20) 3.0000 information_schema FILES TABLESPACE_NAME varchar 64 192 utf8 utf8_general_ci varchar(64) 3.0000 information_schema FILES TABLE_CATALOG varchar 64 192 utf8 utf8_general_ci varchar(64) diff --git a/mysql-test/suite/funcs_1/t/is_columns_is.test b/mysql-test/suite/funcs_1/t/is_columns_is.test index 29d387857c939..e1771dfa41ef3 100644 --- a/mysql-test/suite/funcs_1/t/is_columns_is.test +++ b/mysql-test/suite/funcs_1/t/is_columns_is.test @@ -16,7 +16,7 @@ # --source include/not_embedded.inc ---source include/have_xtradb.inc +--source include/have_innodb.inc let $my_where = WHERE table_schema = 'information_schema' AND table_name <> 'profiling' AND table_name not like 'innodb_%'; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_eviction_factor_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_eviction_factor_basic.result deleted file mode 100644 index 8f017ea40ecb9..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_eviction_factor_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_eviction_factor; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -@@GLOBAL.innodb_cleaner_eviction_factor -0 -SELECT @@SESSION.innodb_cleaner_eviction_factor; -ERROR HY000: Variable 'innodb_cleaner_eviction_factor' is a GLOBAL variable -SET GLOBAL innodb_cleaner_eviction_factor='OFF'; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -@@GLOBAL.innodb_cleaner_eviction_factor -0 -SET GLOBAL innodb_cleaner_eviction_factor='ON'; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -@@GLOBAL.innodb_cleaner_eviction_factor -1 -SET GLOBAL innodb_cleaner_eviction_factor=0; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -@@GLOBAL.innodb_cleaner_eviction_factor -0 -SET GLOBAL innodb_cleaner_eviction_factor=1; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -@@GLOBAL.innodb_cleaner_eviction_factor -1 -SET GLOBAL innodb_cleaner_eviction_factor=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_eviction_factor' -SET GLOBAL innodb_cleaner_eviction_factor=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_eviction_factor' -SET GLOBAL innodb_cleaner_eviction_factor=2; -ERROR 42000: Variable 'innodb_cleaner_eviction_factor' can't be set to the value of '2' -SET GLOBAL innodb_cleaner_eviction_factor='foo'; -ERROR 42000: Variable 'innodb_cleaner_eviction_factor' can't be set to the value of 'foo' -SET GLOBAL innodb_cleaner_eviction_factor = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_flush_chunk_size_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_flush_chunk_size_basic.result deleted file mode 100644 index 651023d7a387e..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_flush_chunk_size_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_flush_chunk_size; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -@@GLOBAL.innodb_cleaner_flush_chunk_size -100 -SELECT @@SESSION.innodb_cleaner_flush_chunk_size; -ERROR HY000: Variable 'innodb_cleaner_flush_chunk_size' is a GLOBAL variable -SET GLOBAL innodb_cleaner_flush_chunk_size=1; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -@@GLOBAL.innodb_cleaner_flush_chunk_size -1 -SET GLOBAL innodb_cleaner_flush_chunk_size=1000; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -@@GLOBAL.innodb_cleaner_flush_chunk_size -1000 -SET GLOBAL innodb_cleaner_flush_chunk_size=4294967295; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -@@GLOBAL.innodb_cleaner_flush_chunk_size -4294967295 -SET GLOBAL innodb_cleaner_flush_chunk_size=0; -Warnings: -Warning 1292 Truncated incorrect innodb_cleaner_flush_chunk_size value: '0' -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -@@GLOBAL.innodb_cleaner_flush_chunk_size -1 -SET GLOBAL innodb_cleaner_flush_chunk_size=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_flush_chunk_size' -SET GLOBAL innodb_cleaner_flush_chunk_size=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_flush_chunk_size' -SET GLOBAL innodb_cleaner_flush_chunk_size='foo'; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_flush_chunk_size' -SET GLOBAL innodb_cleaner_flush_chunk_size = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_free_list_lwm_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_free_list_lwm_basic.result deleted file mode 100644 index 2d7883b7d83da..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_free_list_lwm_basic.result +++ /dev/null @@ -1,35 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_free_list_lwm; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -@@GLOBAL.innodb_cleaner_free_list_lwm -10 -SELECT @@SESSION.innodb_cleaner_free_list_lwm; -ERROR HY000: Variable 'innodb_cleaner_free_list_lwm' is a GLOBAL variable -SET GLOBAL innodb_cleaner_free_list_lwm=0; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -@@GLOBAL.innodb_cleaner_free_list_lwm -0 -SET GLOBAL innodb_cleaner_free_list_lwm=1; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -@@GLOBAL.innodb_cleaner_free_list_lwm -1 -SET GLOBAL innodb_cleaner_free_list_lwm=99; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -@@GLOBAL.innodb_cleaner_free_list_lwm -99 -SET GLOBAL innodb_cleaner_free_list_lwm=100; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -@@GLOBAL.innodb_cleaner_free_list_lwm -100 -SET GLOBAL innodb_cleaner_free_list_lwm=101; -Warnings: -Warning 1292 Truncated incorrect innodb_cleaner_free_list_lwm value: '101' -SELECT @@innodb_cleaner_free_list_lwm; -@@innodb_cleaner_free_list_lwm -100 -SET GLOBAL innodb_cleaner_free_list_lwm=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_free_list_lwm' -SET GLOBAL innodb_cleaner_free_list_lwm=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_free_list_lwm' -SET GLOBAL innodb_cleaner_free_list_lwm='foo'; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_free_list_lwm' -SET GLOBAL innodb_cleaner_free_list_lwm = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_lru_chunk_size_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_lru_chunk_size_basic.result deleted file mode 100644 index 5dfc6738e11e2..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_lru_chunk_size_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_lru_chunk_size; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -@@GLOBAL.innodb_cleaner_lru_chunk_size -100 -SELECT @@SESSION.innodb_cleaner_lru_chunk_size; -ERROR HY000: Variable 'innodb_cleaner_lru_chunk_size' is a GLOBAL variable -SET GLOBAL innodb_cleaner_lru_chunk_size=1; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -@@GLOBAL.innodb_cleaner_lru_chunk_size -1 -SET GLOBAL innodb_cleaner_lru_chunk_size=1000; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -@@GLOBAL.innodb_cleaner_lru_chunk_size -1000 -SET GLOBAL innodb_cleaner_lru_chunk_size=4294967295; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -@@GLOBAL.innodb_cleaner_lru_chunk_size -4294967295 -SET GLOBAL innodb_cleaner_lru_chunk_size=0; -Warnings: -Warning 1292 Truncated incorrect innodb_cleaner_lru_chunk_size value: '0' -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -@@GLOBAL.innodb_cleaner_lru_chunk_size -1 -SET GLOBAL innodb_cleaner_lru_chunk_size=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_lru_chunk_size' -SET GLOBAL innodb_cleaner_lru_chunk_size=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_lru_chunk_size' -SET GLOBAL innodb_cleaner_lru_chunk_size='foo'; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_lru_chunk_size' -SET GLOBAL innodb_cleaner_lru_chunk_size = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_lsn_age_factor_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_lsn_age_factor_basic.result deleted file mode 100644 index 65046d93344d8..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_lsn_age_factor_basic.result +++ /dev/null @@ -1,21 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_lsn_age_factor; -SELECT @@GLOBAL.innodb_cleaner_lsn_age_factor; -@@GLOBAL.innodb_cleaner_lsn_age_factor -HIGH_CHECKPOINT -SELECT @@SESSION.innodb_cleaner_lsn_age_factor; -ERROR HY000: Variable 'innodb_cleaner_lsn_age_factor' is a GLOBAL variable -SET GLOBAL innodb_cleaner_lsn_age_factor='legacy'; -SELECT @@GLOBAL.innodb_cleaner_lsn_age_factor; -@@GLOBAL.innodb_cleaner_lsn_age_factor -LEGACY -SET GLOBAL innodb_cleaner_lsn_age_factor='high_checkpoint'; -SELECT @@GLOBAL.innodb_cleaner_lsn_age_factor; -@@GLOBAL.innodb_cleaner_lsn_age_factor -HIGH_CHECKPOINT -SET GLOBAL innodb_cleaner_lsn_age_factor=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_lsn_age_factor' -SET GLOBAL innodb_cleaner_lsn_age_factor=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_lsn_age_factor' -SET GLOBAL innodb_cleaner_lsn_age_factor='foo'; -ERROR 42000: Variable 'innodb_cleaner_lsn_age_factor' can't be set to the value of 'foo' -SET GLOBAL innodb_cleaner_lsn_age_factor = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_max_flush_time_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_max_flush_time_basic.result deleted file mode 100644 index e4a3fa26e73e1..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_max_flush_time_basic.result +++ /dev/null @@ -1,25 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_max_flush_time; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; -@@GLOBAL.innodb_cleaner_max_flush_time -1000 -SELECT @@SESSION.innodb_cleaner_max_flush_time; -ERROR HY000: Variable 'innodb_cleaner_max_flush_time' is a GLOBAL variable -SET GLOBAL innodb_cleaner_max_flush_time=0; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; -@@GLOBAL.innodb_cleaner_max_flush_time -0 -SET GLOBAL innodb_cleaner_max_flush_time=1000; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; -@@GLOBAL.innodb_cleaner_max_flush_time -1000 -SET GLOBAL innodb_cleaner_max_flush_time=4294967295; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; -@@GLOBAL.innodb_cleaner_max_flush_time -4294967295 -SET GLOBAL innodb_cleaner_max_flush_time=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_max_flush_time' -SET GLOBAL innodb_cleaner_max_flush_time=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_max_flush_time' -SET GLOBAL innodb_cleaner_max_flush_time='foo'; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_max_flush_time' -SET GLOBAL innodb_cleaner_max_flush_time = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_cleaner_max_lru_time_basic.result b/mysql-test/suite/sys_vars/r/innodb_cleaner_max_lru_time_basic.result deleted file mode 100644 index f7bacbbd62e1b..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_cleaner_max_lru_time_basic.result +++ /dev/null @@ -1,25 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_cleaner_max_lru_time; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; -@@GLOBAL.innodb_cleaner_max_lru_time -1000 -SELECT @@SESSION.innodb_cleaner_max_lru_time; -ERROR HY000: Variable 'innodb_cleaner_max_lru_time' is a GLOBAL variable -SET GLOBAL innodb_cleaner_max_lru_time=0; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; -@@GLOBAL.innodb_cleaner_max_lru_time -0 -SET GLOBAL innodb_cleaner_max_lru_time=1000; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; -@@GLOBAL.innodb_cleaner_max_lru_time -1000 -SET GLOBAL innodb_cleaner_max_lru_time=4294967295; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; -@@GLOBAL.innodb_cleaner_max_lru_time -4294967295 -SET GLOBAL innodb_cleaner_max_lru_time=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_max_lru_time' -SET GLOBAL innodb_cleaner_max_lru_time=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_max_lru_time' -SET GLOBAL innodb_cleaner_max_lru_time='foo'; -ERROR 42000: Incorrect argument type to variable 'innodb_cleaner_max_lru_time' -SET GLOBAL innodb_cleaner_max_lru_time = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_empty_free_list_algorithm_basic.result b/mysql-test/suite/sys_vars/r/innodb_empty_free_list_algorithm_basic.result deleted file mode 100644 index 0730055d1041e..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_empty_free_list_algorithm_basic.result +++ /dev/null @@ -1,23 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_empty_free_list_algorithm; -SELECT @@GLOBAL.innodb_empty_free_list_algorithm; -@@GLOBAL.innodb_empty_free_list_algorithm -BACKOFF -SELECT @@SESSION.innodb_empty_free_list_algorithm; -ERROR HY000: Variable 'innodb_empty_free_list_algorithm' is a GLOBAL variable -SET GLOBAL innodb_empty_free_list_algorithm='legacy'; -SELECT @@GLOBAL.innodb_empty_free_list_algorithm; -@@GLOBAL.innodb_empty_free_list_algorithm -LEGACY -SET GLOBAL innodb_empty_free_list_algorithm='backoff'; -SELECT @@GLOBAL.innodb_empty_free_list_algorithm; -@@GLOBAL.innodb_empty_free_list_algorithm -BACKOFF -SET GLOBAL innodb_empty_free_list_algorithm=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_empty_free_list_algorithm' -SET GLOBAL innodb_empty_free_list_algorithm=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_empty_free_list_algorithm' -SET GLOBAL innodb_empty_free_list_algorithm=2; -ERROR 42000: Variable 'innodb_empty_free_list_algorithm' can't be set to the value of '2' -SET GLOBAL innodb_empty_free_list_algorithm='foo'; -ERROR 42000: Variable 'innodb_empty_free_list_algorithm' can't be set to the value of 'foo' -SET GLOBAL innodb_empty_free_list_algorithm = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_foreground_preflush_basic.result b/mysql-test/suite/sys_vars/r/innodb_foreground_preflush_basic.result deleted file mode 100644 index 17e67d2b8618b..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_foreground_preflush_basic.result +++ /dev/null @@ -1,23 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_foreground_preflush; -SELECT @@GLOBAL.innodb_foreground_preflush; -@@GLOBAL.innodb_foreground_preflush -EXPONENTIAL_BACKOFF -SELECT @@SESSION.innodb_foreground_preflush; -ERROR HY000: Variable 'innodb_foreground_preflush' is a GLOBAL variable -SET GLOBAL innodb_foreground_preflush='sync_preflush'; -SELECT @@GLOBAL.innodb_foreground_preflush; -@@GLOBAL.innodb_foreground_preflush -SYNC_PREFLUSH -SET GLOBAL innodb_foreground_preflush='exponential_backoff'; -SELECT @@GLOBAL.innodb_foreground_preflush; -@@GLOBAL.innodb_foreground_preflush -EXPONENTIAL_BACKOFF -SET GLOBAL innodb_foreground_preflush=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_foreground_preflush' -SET GLOBAL innodb_foreground_preflush=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_foreground_preflush' -SET GLOBAL innodb_foreground_preflush=2; -ERROR 42000: Variable 'innodb_foreground_preflush' can't be set to the value of '2' -SET GLOBAL innodb_foreground_preflush='foo'; -ERROR 42000: Variable 'innodb_foreground_preflush' can't be set to the value of 'foo' -SET GLOBAL innodb_foreground_preflush = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_log_arch_dir_basic.result b/mysql-test/suite/sys_vars/r/innodb_log_arch_dir_basic.result deleted file mode 100644 index bead03035209b..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_log_arch_dir_basic.result +++ /dev/null @@ -1,38 +0,0 @@ -SELECT @@GLOBAL.innodb_log_arch_dir; -@@GLOBAL.innodb_log_arch_dir -./ -NULL Expected -SET @@GLOBAL.innodb_log_arch_dir=1; -ERROR HY000: Variable 'innodb_log_arch_dir' is a read only variable -Expected error 'Read only variable' -SELECT @@GLOBAL.innodb_log_arch_dir; -@@GLOBAL.innodb_log_arch_dir -./ -NULL Expected -SELECT VARIABLE_VALUE -FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES -WHERE VARIABLE_NAME='innodb_log_arch_dir'; -VARIABLE_VALUE -./ -empty string Expected -SELECT @@innodb_log_arch_dir; -@@innodb_log_arch_dir -./ -NULL Expected -SELECT @@innodb_log_arch_dir; -@@innodb_log_arch_dir -./ -NULL Expected -SELECT @@local.innodb_log_arch_dir; -ERROR HY000: Variable 'innodb_log_arch_dir' is a GLOBAL variable -Expected error 'Variable is a GLOBAL variable' -SELECT @@SESSION.innodb_log_arch_dir; -ERROR HY000: Variable 'innodb_log_arch_dir' is a GLOBAL variable -Expected error 'Variable is a GLOBAL variable' -SELECT @@GLOBAL.innodb_log_arch_dir; -@@GLOBAL.innodb_log_arch_dir -./ -NULL Expected -SELECT innodb_log_arch_dir = @@SESSION.innodb_log_arch_dir; -ERROR 42S22: Unknown column 'innodb_log_arch_dir' in 'field list' -Expected error Unknown column 'innodb_log_arch_dir' in 'field list' diff --git a/mysql-test/suite/sys_vars/r/innodb_log_arch_expire_sec_basic.result b/mysql-test/suite/sys_vars/r/innodb_log_arch_expire_sec_basic.result deleted file mode 100644 index 97bff0972528b..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_log_arch_expire_sec_basic.result +++ /dev/null @@ -1,38 +0,0 @@ -SELECT @@GLOBAL.innodb_log_arch_expire_sec INTO @save; -SELECT @@GLOBAL.innodb_log_arch_expire_sec; -@@GLOBAL.innodb_log_arch_expire_sec -0 -0 Expected -SET @@GLOBAL.innodb_log_arch_expire_sec=1; -SELECT @@GLOBAL.innodb_log_arch_expire_sec; -@@GLOBAL.innodb_log_arch_expire_sec -1 -1 Expected -SELECT VARIABLE_VALUE -FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES -WHERE VARIABLE_NAME='innodb_log_arch_expire_sec'; -VARIABLE_VALUE -1 -1 Expected -SELECT @@innodb_log_arch_expire_sec; -@@innodb_log_arch_expire_sec -1 -1 Expected -SELECT @@innodb_log_arch_expire_sec; -@@innodb_log_arch_expire_sec -1 -1 Expected -SELECT @@local.innodb_log_arch_expire_sec; -ERROR HY000: Variable 'innodb_log_arch_expire_sec' is a GLOBAL variable -Expected error 'Variable is a GLOBAL variable' -SELECT @@SESSION.innodb_log_arch_expire_sec; -ERROR HY000: Variable 'innodb_log_arch_expire_sec' is a GLOBAL variable -Expected error 'Variable is a GLOBAL variable' -SELECT @@GLOBAL.innodb_log_arch_expire_sec; -@@GLOBAL.innodb_log_arch_expire_sec -1 -1 Expected -SELECT innodb_log_arch_expire_sec = @@SESSION.innodb_log_arch_expire_sec; -ERROR 42S22: Unknown column 'innodb_log_arch_expire_sec' in 'field list' -Expected error Unknown column 'innodb_log_arch_expire_sec' in 'field list' -SET @@GLOBAL.innodb_log_arch_expire_sec = @save; diff --git a/mysql-test/suite/sys_vars/r/innodb_log_archive_basic.result b/mysql-test/suite/sys_vars/r/innodb_log_archive_basic.result deleted file mode 100644 index bb9b53482fa17..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_log_archive_basic.result +++ /dev/null @@ -1,38 +0,0 @@ -SELECT @@GLOBAL.innodb_log_archive; -@@GLOBAL.innodb_log_archive -0 -0 Expected -SET @save_innodb_log_archive = @@GLOBAL.innodb_log_archive; -SET @@GLOBAL.innodb_log_archive=1; -SELECT @@GLOBAL.innodb_log_archive; -@@GLOBAL.innodb_log_archive -1 -1 Expected -SELECT VARIABLE_VALUE -FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES -WHERE VARIABLE_NAME='innodb_log_archive'; -VARIABLE_VALUE -ON -ON Expected -SET @@GLOBAL.innodb_log_archive = @save_innodb_log_archive; -SELECT @@innodb_log_archive; -@@innodb_log_archive -0 -0 Expected -SELECT @@innodb_log_archive; -@@innodb_log_archive -0 -0 Expected -SELECT @@local.innodb_log_archive; -ERROR HY000: Variable 'innodb_log_archive' is a GLOBAL variable -Expected error 'Variable is a GLOBAL variable' -SELECT @@SESSION.innodb_log_archive; -ERROR HY000: Variable 'innodb_log_archive' is a GLOBAL variable -Expected error 'Variable is a GLOBAL variable' -SELECT @@GLOBAL.innodb_log_archive; -@@GLOBAL.innodb_log_archive -0 -0 Expected -SELECT innodb_log_archive = @@SESSION.innodb_log_archive; -ERROR 42S22: Unknown column 'innodb_log_archive' in 'field list' -Expected error Unknown column 'innodb_log_archive' in 'field list' diff --git a/mysql-test/suite/sys_vars/r/innodb_log_checksum_algorithm_basic.result b/mysql-test/suite/sys_vars/r/innodb_log_checksum_algorithm_basic.result deleted file mode 100644 index 4497e8f208aa0..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_log_checksum_algorithm_basic.result +++ /dev/null @@ -1,47 +0,0 @@ -SET @orig = @@global.innodb_log_checksum_algorithm; -SELECT @orig; -@orig -INNODB -SET GLOBAL innodb_log_checksum_algorithm = 'crc32'; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -CRC32 -SET GLOBAL innodb_log_checksum_algorithm = 'strict_crc32'; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -STRICT_CRC32 -SET GLOBAL innodb_log_checksum_algorithm = 'innodb'; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -INNODB -SET GLOBAL innodb_log_checksum_algorithm = 'strict_innodb'; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -STRICT_INNODB -SET GLOBAL innodb_log_checksum_algorithm = 'none'; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -NONE -SET GLOBAL innodb_log_checksum_algorithm = 'strict_none'; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -STRICT_NONE -SET GLOBAL innodb_log_checksum_algorithm = ''; -ERROR 42000: Variable 'innodb_log_checksum_algorithm' can't be set to the value of '' -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -STRICT_NONE -SET GLOBAL innodb_log_checksum_algorithm = 'foobar'; -ERROR 42000: Variable 'innodb_log_checksum_algorithm' can't be set to the value of 'foobar' -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -STRICT_NONE -SET GLOBAL innodb_log_checksum_algorithm = 123; -ERROR 42000: Variable 'innodb_log_checksum_algorithm' can't be set to the value of '123' -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -STRICT_NONE -SET GLOBAL innodb_log_checksum_algorithm = @orig; -SELECT @@global.innodb_log_checksum_algorithm; -@@global.innodb_log_checksum_algorithm -INNODB diff --git a/mysql-test/suite/sys_vars/r/innodb_priority_cleaner_basic.result b/mysql-test/suite/sys_vars/r/innodb_priority_cleaner_basic.result deleted file mode 100644 index ae5e12ee64a82..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_priority_cleaner_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_priority_cleaner; -SELECT @@GLOBAL.innodb_priority_cleaner; -@@GLOBAL.innodb_priority_cleaner -0 -SELECT @@SESSION.innodb_priority_cleaner; -ERROR HY000: Variable 'innodb_priority_cleaner' is a GLOBAL variable -SET GLOBAL innodb_priority_cleaner='OFF'; -SELECT @@GLOBAL.innodb_priority_cleaner; -@@GLOBAL.innodb_priority_cleaner -0 -SET GLOBAL innodb_priority_cleaner='ON'; -SELECT @@GLOBAL.innodb_priority_cleaner; -@@GLOBAL.innodb_priority_cleaner -1 -SET GLOBAL innodb_priority_cleaner=0; -SELECT @@GLOBAL.innodb_priority_cleaner; -@@GLOBAL.innodb_priority_cleaner -0 -SET GLOBAL innodb_priority_cleaner=1; -SELECT @@GLOBAL.innodb_priority_cleaner; -@@GLOBAL.innodb_priority_cleaner -1 -SET GLOBAL innodb_priority_cleaner=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_cleaner' -SET GLOBAL innodb_priority_cleaner=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_cleaner' -SET GLOBAL innodb_priority_cleaner=2; -ERROR 42000: Variable 'innodb_priority_cleaner' can't be set to the value of '2' -SET GLOBAL innodb_priority_cleaner='foo'; -ERROR 42000: Variable 'innodb_priority_cleaner' can't be set to the value of 'foo' -SET GLOBAL innodb_priority_cleaner = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_priority_io_basic.result b/mysql-test/suite/sys_vars/r/innodb_priority_io_basic.result deleted file mode 100644 index 70ccb5e4cf4a4..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_priority_io_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_priority_io; -SELECT @@GLOBAL.innodb_priority_io; -@@GLOBAL.innodb_priority_io -0 -SELECT @@SESSION.innodb_priority_io; -ERROR HY000: Variable 'innodb_priority_io' is a GLOBAL variable -SET GLOBAL innodb_priority_io='OFF'; -SELECT @@GLOBAL.innodb_priority_io; -@@GLOBAL.innodb_priority_io -0 -SET GLOBAL innodb_priority_io='ON'; -SELECT @@GLOBAL.innodb_priority_io; -@@GLOBAL.innodb_priority_io -1 -SET GLOBAL innodb_priority_io=0; -SELECT @@GLOBAL.innodb_priority_io; -@@GLOBAL.innodb_priority_io -0 -SET GLOBAL innodb_priority_io=1; -SELECT @@GLOBAL.innodb_priority_io; -@@GLOBAL.innodb_priority_io -1 -SET GLOBAL innodb_priority_io=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_io' -SET GLOBAL innodb_priority_io=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_io' -SET GLOBAL innodb_priority_io=2; -ERROR 42000: Variable 'innodb_priority_io' can't be set to the value of '2' -SET GLOBAL innodb_priority_io='foo'; -ERROR 42000: Variable 'innodb_priority_io' can't be set to the value of 'foo' -SET GLOBAL innodb_priority_io = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_priority_master_basic.result b/mysql-test/suite/sys_vars/r/innodb_priority_master_basic.result deleted file mode 100644 index d26ead2ff7ed9..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_priority_master_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_priority_master; -SELECT @@GLOBAL.innodb_priority_master; -@@GLOBAL.innodb_priority_master -0 -SELECT @@SESSION.innodb_priority_master; -ERROR HY000: Variable 'innodb_priority_master' is a GLOBAL variable -SET GLOBAL innodb_priority_master='OFF'; -SELECT @@GLOBAL.innodb_priority_master; -@@GLOBAL.innodb_priority_master -0 -SET GLOBAL innodb_priority_master='ON'; -SELECT @@GLOBAL.innodb_priority_master; -@@GLOBAL.innodb_priority_master -1 -SET GLOBAL innodb_priority_master=0; -SELECT @@GLOBAL.innodb_priority_master; -@@GLOBAL.innodb_priority_master -0 -SET GLOBAL innodb_priority_master=1; -SELECT @@GLOBAL.innodb_priority_master; -@@GLOBAL.innodb_priority_master -1 -SET GLOBAL innodb_priority_master=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_master' -SET GLOBAL innodb_priority_master=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_master' -SET GLOBAL innodb_priority_master=2; -ERROR 42000: Variable 'innodb_priority_master' can't be set to the value of '2' -SET GLOBAL innodb_priority_master='foo'; -ERROR 42000: Variable 'innodb_priority_master' can't be set to the value of 'foo' -SET GLOBAL innodb_priority_master = @start_value; diff --git a/mysql-test/suite/sys_vars/r/innodb_priority_purge_basic.result b/mysql-test/suite/sys_vars/r/innodb_priority_purge_basic.result deleted file mode 100644 index 57153ebf82ad5..0000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_priority_purge_basic.result +++ /dev/null @@ -1,31 +0,0 @@ -SET @start_value = @@GLOBAL.innodb_priority_purge; -SELECT @@GLOBAL.innodb_priority_purge; -@@GLOBAL.innodb_priority_purge -0 -SELECT @@SESSION.innodb_priority_purge; -ERROR HY000: Variable 'innodb_priority_purge' is a GLOBAL variable -SET GLOBAL innodb_priority_purge='OFF'; -SELECT @@GLOBAL.innodb_priority_purge; -@@GLOBAL.innodb_priority_purge -0 -SET GLOBAL innodb_priority_purge='ON'; -SELECT @@GLOBAL.innodb_priority_purge; -@@GLOBAL.innodb_priority_purge -1 -SET GLOBAL innodb_priority_purge=0; -SELECT @@GLOBAL.innodb_priority_purge; -@@GLOBAL.innodb_priority_purge -0 -SET GLOBAL innodb_priority_purge=1; -SELECT @@GLOBAL.innodb_priority_purge; -@@GLOBAL.innodb_priority_purge -1 -SET GLOBAL innodb_priority_purge=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_purge' -SET GLOBAL innodb_priority_purge=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_priority_purge' -SET GLOBAL innodb_priority_purge=2; -ERROR 42000: Variable 'innodb_priority_purge' can't be set to the value of '2' -SET GLOBAL innodb_priority_purge='foo'; -ERROR 42000: Variable 'innodb_priority_purge' can't be set to the value of 'foo' -SET GLOBAL innodb_priority_purge = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_eviction_factor_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_eviction_factor_basic.test deleted file mode 100644 index 8e0af20a47ef9..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_eviction_factor_basic.test +++ /dev/null @@ -1,35 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_eviction_factor; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_eviction_factor; - -# Correct values -SET GLOBAL innodb_cleaner_eviction_factor='OFF'; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -SET GLOBAL innodb_cleaner_eviction_factor='ON'; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -SET GLOBAL innodb_cleaner_eviction_factor=0; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; -SET GLOBAL innodb_cleaner_eviction_factor=1; -SELECT @@GLOBAL.innodb_cleaner_eviction_factor; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_eviction_factor=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_eviction_factor=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_cleaner_eviction_factor=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_cleaner_eviction_factor='foo'; - -SET GLOBAL innodb_cleaner_eviction_factor = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_flush_chunk_size_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_flush_chunk_size_basic.test deleted file mode 100644 index c65fc63c20f9c..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_flush_chunk_size_basic.test +++ /dev/null @@ -1,33 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_flush_chunk_size; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_flush_chunk_size; - -# Correct values -SET GLOBAL innodb_cleaner_flush_chunk_size=1; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -SET GLOBAL innodb_cleaner_flush_chunk_size=1000; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; -SET GLOBAL innodb_cleaner_flush_chunk_size=4294967295; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; - -# Incorrect values -SET GLOBAL innodb_cleaner_flush_chunk_size=0; -SELECT @@GLOBAL.innodb_cleaner_flush_chunk_size; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_flush_chunk_size=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_flush_chunk_size=1e1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_flush_chunk_size='foo'; - -SET GLOBAL innodb_cleaner_flush_chunk_size = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_free_list_lwm_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_free_list_lwm_basic.test deleted file mode 100644 index fa9d1e9f57432..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_free_list_lwm_basic.test +++ /dev/null @@ -1,35 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_free_list_lwm; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_free_list_lwm; - -# Correct values -SET GLOBAL innodb_cleaner_free_list_lwm=0; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -SET GLOBAL innodb_cleaner_free_list_lwm=1; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -SET GLOBAL innodb_cleaner_free_list_lwm=99; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; -SET GLOBAL innodb_cleaner_free_list_lwm=100; -SELECT @@GLOBAL.innodb_cleaner_free_list_lwm; - -# Incorrect values -SET GLOBAL innodb_cleaner_free_list_lwm=101; -SELECT @@innodb_cleaner_free_list_lwm; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_free_list_lwm=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_free_list_lwm=1e1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_free_list_lwm='foo'; - -SET GLOBAL innodb_cleaner_free_list_lwm = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_lru_chunk_size_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_lru_chunk_size_basic.test deleted file mode 100644 index 12da590446c9b..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_lru_chunk_size_basic.test +++ /dev/null @@ -1,33 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_lru_chunk_size; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_lru_chunk_size; - -# Correct values -SET GLOBAL innodb_cleaner_lru_chunk_size=1; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -SET GLOBAL innodb_cleaner_lru_chunk_size=1000; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; -SET GLOBAL innodb_cleaner_lru_chunk_size=4294967295; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; - -# Incorrect values -SET GLOBAL innodb_cleaner_lru_chunk_size=0; -SELECT @@GLOBAL.innodb_cleaner_lru_chunk_size; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_lru_chunk_size=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_lru_chunk_size=1e1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_lru_chunk_size='foo'; - -SET GLOBAL innodb_cleaner_lru_chunk_size = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_lsn_age_factor_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_lsn_age_factor_basic.test deleted file mode 100644 index b34fcc94494cf..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_lsn_age_factor_basic.test +++ /dev/null @@ -1,28 +0,0 @@ ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_lsn_age_factor; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_lsn_age_factor; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_lsn_age_factor; - -# Correct values -SET GLOBAL innodb_cleaner_lsn_age_factor='legacy'; -SELECT @@GLOBAL.innodb_cleaner_lsn_age_factor; -SET GLOBAL innodb_cleaner_lsn_age_factor='high_checkpoint'; -SELECT @@GLOBAL.innodb_cleaner_lsn_age_factor; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_lsn_age_factor=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_lsn_age_factor=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_cleaner_lsn_age_factor='foo'; - -SET GLOBAL innodb_cleaner_lsn_age_factor = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_max_flush_time_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_max_flush_time_basic.test deleted file mode 100644 index 283c651d0c5a1..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_max_flush_time_basic.test +++ /dev/null @@ -1,31 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_max_flush_time; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_max_flush_time; - -# Correct values -SET GLOBAL innodb_cleaner_max_flush_time=0; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; -SET GLOBAL innodb_cleaner_max_flush_time=1000; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; -SET GLOBAL innodb_cleaner_max_flush_time=4294967295; -SELECT @@GLOBAL.innodb_cleaner_max_flush_time; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_max_flush_time=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_max_flush_time=1e1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_max_flush_time='foo'; - -SET GLOBAL innodb_cleaner_max_flush_time = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_cleaner_max_lru_time_basic.test b/mysql-test/suite/sys_vars/t/innodb_cleaner_max_lru_time_basic.test deleted file mode 100644 index d0621e77df378..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_cleaner_max_lru_time_basic.test +++ /dev/null @@ -1,31 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_cleaner_max_lru_time; - -# Default value -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_cleaner_max_lru_time; - -# Correct values -SET GLOBAL innodb_cleaner_max_lru_time=0; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; -SET GLOBAL innodb_cleaner_max_lru_time=1000; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; -SET GLOBAL innodb_cleaner_max_lru_time=4294967295; -SELECT @@GLOBAL.innodb_cleaner_max_lru_time; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_max_lru_time=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_max_lru_time=1e1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_cleaner_max_lru_time='foo'; - -SET GLOBAL innodb_cleaner_max_lru_time = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.opt b/mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.opt deleted file mode 100644 index c788dc76ac753..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.opt +++ /dev/null @@ -1 +0,0 @@ ---loose-innodb-buffer-pool-size=20M diff --git a/mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.test b/mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.test deleted file mode 100644 index 6bb34f36a4f8f..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_empty_free_list_algorithm_basic.test +++ /dev/null @@ -1,30 +0,0 @@ ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_empty_free_list_algorithm; - -# Default value -SELECT @@GLOBAL.innodb_empty_free_list_algorithm; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_empty_free_list_algorithm; - -# Correct values -SET GLOBAL innodb_empty_free_list_algorithm='legacy'; -SELECT @@GLOBAL.innodb_empty_free_list_algorithm; -SET GLOBAL innodb_empty_free_list_algorithm='backoff'; -SELECT @@GLOBAL.innodb_empty_free_list_algorithm; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_empty_free_list_algorithm=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_empty_free_list_algorithm=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_empty_free_list_algorithm=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_empty_free_list_algorithm='foo'; - -SET GLOBAL innodb_empty_free_list_algorithm = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_foreground_preflush_basic.test b/mysql-test/suite/sys_vars/t/innodb_foreground_preflush_basic.test deleted file mode 100644 index f388b392f9ba1..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_foreground_preflush_basic.test +++ /dev/null @@ -1,30 +0,0 @@ ---source include/have_xtradb.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_foreground_preflush; - -# Default value -SELECT @@GLOBAL.innodb_foreground_preflush; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_foreground_preflush; - -# Correct values -SET GLOBAL innodb_foreground_preflush='sync_preflush'; -SELECT @@GLOBAL.innodb_foreground_preflush; -SET GLOBAL innodb_foreground_preflush='exponential_backoff'; -SELECT @@GLOBAL.innodb_foreground_preflush; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_foreground_preflush=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_foreground_preflush=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_foreground_preflush=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_foreground_preflush='foo'; - -SET GLOBAL innodb_foreground_preflush = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_log_arch_dir_basic.test b/mysql-test/suite/sys_vars/t/innodb_log_arch_dir_basic.test deleted file mode 100644 index 084d97fa46065..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_log_arch_dir_basic.test +++ /dev/null @@ -1,68 +0,0 @@ -####################################################### -# Basic test for innodb_log_arch_dir variable # -####################################################### - ---source include/have_xtradb.inc - -let $datadir= `select @@datadir`; - -#################################################################### -# Displaying default value # -#################################################################### ---replace_result $datadir ./ -SELECT @@GLOBAL.innodb_log_arch_dir; ---echo NULL Expected - - -#################################################################### -# Check if Value can set # -#################################################################### - ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SET @@GLOBAL.innodb_log_arch_dir=1; ---echo Expected error 'Read only variable' - ---replace_result $datadir ./ -SELECT @@GLOBAL.innodb_log_arch_dir; ---echo NULL Expected - ---replace_result $datadir ./ -SELECT VARIABLE_VALUE -FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES -WHERE VARIABLE_NAME='innodb_log_arch_dir'; ---echo empty string Expected - -############################################ -# Check accessing variable without GLOBAL # -############################################ ---replace_result $datadir ./ -SELECT @@innodb_log_arch_dir; ---echo NULL Expected - - - -########################################################################## -# Check if innodb_log_arch_dir can be accessed without @@ sign # -########################################################################## - ---replace_result $datadir ./ -SELECT @@innodb_log_arch_dir; ---echo NULL Expected - ---Error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@local.innodb_log_arch_dir; ---echo Expected error 'Variable is a GLOBAL variable' - ---Error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_log_arch_dir; ---echo Expected error 'Variable is a GLOBAL variable' - ---replace_result $datadir ./ -SELECT @@GLOBAL.innodb_log_arch_dir; ---echo NULL Expected - ---Error ER_BAD_FIELD_ERROR -SELECT innodb_log_arch_dir = @@SESSION.innodb_log_arch_dir; ---echo Expected error Unknown column 'innodb_log_arch_dir' in 'field list' - - diff --git a/mysql-test/suite/sys_vars/t/innodb_log_arch_expire_sec_basic.test b/mysql-test/suite/sys_vars/t/innodb_log_arch_expire_sec_basic.test deleted file mode 100644 index 87c374ea8860a..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_log_arch_expire_sec_basic.test +++ /dev/null @@ -1,60 +0,0 @@ -############################################################### -# Basic test for innodb_log_arch_expire_sec variable # -############################################################### - ---source include/have_xtradb.inc - -SELECT @@GLOBAL.innodb_log_arch_expire_sec INTO @save; - -#################################################################### -# Displaying default value # -#################################################################### -SELECT @@GLOBAL.innodb_log_arch_expire_sec; ---echo 0 Expected - - -#################################################################### -# Check if Value can set # -#################################################################### - -SET @@GLOBAL.innodb_log_arch_expire_sec=1; - -SELECT @@GLOBAL.innodb_log_arch_expire_sec; ---echo 1 Expected - -SELECT VARIABLE_VALUE -FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES -WHERE VARIABLE_NAME='innodb_log_arch_expire_sec'; ---echo 1 Expected - -############################################ -# Check accessing variable without GLOBAL # -############################################ -SELECT @@innodb_log_arch_expire_sec; ---echo 1 Expected - - - -################################################################################## -# Check if innodb_log_arch_expire_sec can be accessed without @@ sign # -################################################################################## - -SELECT @@innodb_log_arch_expire_sec; ---echo 1 Expected - ---Error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@local.innodb_log_arch_expire_sec; ---echo Expected error 'Variable is a GLOBAL variable' - ---Error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_log_arch_expire_sec; ---echo Expected error 'Variable is a GLOBAL variable' - -SELECT @@GLOBAL.innodb_log_arch_expire_sec; ---echo 1 Expected - ---Error ER_BAD_FIELD_ERROR -SELECT innodb_log_arch_expire_sec = @@SESSION.innodb_log_arch_expire_sec; ---echo Expected error Unknown column 'innodb_log_arch_expire_sec' in 'field list' - -SET @@GLOBAL.innodb_log_arch_expire_sec = @save; diff --git a/mysql-test/suite/sys_vars/t/innodb_log_archive_basic.test b/mysql-test/suite/sys_vars/t/innodb_log_archive_basic.test deleted file mode 100644 index cbc885123cea6..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_log_archive_basic.test +++ /dev/null @@ -1,61 +0,0 @@ -################################################### -# Basic test for innodb_log_archive variable # -################################################### - ---source include/have_xtradb.inc - -#################################################################### -# Displaying default value # -#################################################################### -SELECT @@GLOBAL.innodb_log_archive; ---echo 0 Expected - - -#################################################################### -# Check if Value can set # -#################################################################### - -SET @save_innodb_log_archive = @@GLOBAL.innodb_log_archive; -SET @@GLOBAL.innodb_log_archive=1; - -SELECT @@GLOBAL.innodb_log_archive; ---echo 1 Expected - -SELECT VARIABLE_VALUE -FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES -WHERE VARIABLE_NAME='innodb_log_archive'; ---echo ON Expected - -SET @@GLOBAL.innodb_log_archive = @save_innodb_log_archive; - -############################################ -# Check accessing variable without GLOBAL # -############################################ -SELECT @@innodb_log_archive; ---echo 0 Expected - - - -########################################################################## -# Check if innodb_log_archive can be accessed without @@ sign # -########################################################################## - -SELECT @@innodb_log_archive; ---echo 0 Expected - ---Error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@local.innodb_log_archive; ---echo Expected error 'Variable is a GLOBAL variable' - ---Error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_log_archive; ---echo Expected error 'Variable is a GLOBAL variable' - -SELECT @@GLOBAL.innodb_log_archive; ---echo 0 Expected - ---Error ER_BAD_FIELD_ERROR -SELECT innodb_log_archive = @@SESSION.innodb_log_archive; ---echo Expected error Unknown column 'innodb_log_archive' in 'field list' - - diff --git a/mysql-test/suite/sys_vars/t/innodb_log_checksum_algorithm_basic.test b/mysql-test/suite/sys_vars/t/innodb_log_checksum_algorithm_basic.test deleted file mode 100644 index 1a83d4f2602a8..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_log_checksum_algorithm_basic.test +++ /dev/null @@ -1,38 +0,0 @@ ---source include/have_xtradb.inc - -# Check the default value -SET @orig = @@global.innodb_log_checksum_algorithm; -SELECT @orig; - -SET GLOBAL innodb_log_checksum_algorithm = 'crc32'; -SELECT @@global.innodb_log_checksum_algorithm; - -SET GLOBAL innodb_log_checksum_algorithm = 'strict_crc32'; -SELECT @@global.innodb_log_checksum_algorithm; - -SET GLOBAL innodb_log_checksum_algorithm = 'innodb'; -SELECT @@global.innodb_log_checksum_algorithm; - -SET GLOBAL innodb_log_checksum_algorithm = 'strict_innodb'; -SELECT @@global.innodb_log_checksum_algorithm; - -SET GLOBAL innodb_log_checksum_algorithm = 'none'; -SELECT @@global.innodb_log_checksum_algorithm; - -SET GLOBAL innodb_log_checksum_algorithm = 'strict_none'; -SELECT @@global.innodb_log_checksum_algorithm; - --- error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_log_checksum_algorithm = ''; -SELECT @@global.innodb_log_checksum_algorithm; - --- error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_log_checksum_algorithm = 'foobar'; -SELECT @@global.innodb_log_checksum_algorithm; - --- error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_log_checksum_algorithm = 123; -SELECT @@global.innodb_log_checksum_algorithm; - -SET GLOBAL innodb_log_checksum_algorithm = @orig; -SELECT @@global.innodb_log_checksum_algorithm; diff --git a/mysql-test/suite/sys_vars/t/innodb_priority_cleaner_basic.test b/mysql-test/suite/sys_vars/t/innodb_priority_cleaner_basic.test deleted file mode 100644 index a305978a280fe..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_priority_cleaner_basic.test +++ /dev/null @@ -1,36 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc ---source include/linux.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_priority_cleaner; - -# Default value -SELECT @@GLOBAL.innodb_priority_cleaner; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_priority_cleaner; - -# Correct values -SET GLOBAL innodb_priority_cleaner='OFF'; -SELECT @@GLOBAL.innodb_priority_cleaner; -SET GLOBAL innodb_priority_cleaner='ON'; -SELECT @@GLOBAL.innodb_priority_cleaner; -SET GLOBAL innodb_priority_cleaner=0; -SELECT @@GLOBAL.innodb_priority_cleaner; -SET GLOBAL innodb_priority_cleaner=1; -SELECT @@GLOBAL.innodb_priority_cleaner; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_cleaner=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_cleaner=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_cleaner=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_cleaner='foo'; - -SET GLOBAL innodb_priority_cleaner = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_priority_io_basic.test b/mysql-test/suite/sys_vars/t/innodb_priority_io_basic.test deleted file mode 100644 index d8a04cccf1ecb..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_priority_io_basic.test +++ /dev/null @@ -1,36 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc ---source include/linux.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_priority_io; - -# Default value -SELECT @@GLOBAL.innodb_priority_io; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_priority_io; - -# Correct values -SET GLOBAL innodb_priority_io='OFF'; -SELECT @@GLOBAL.innodb_priority_io; -SET GLOBAL innodb_priority_io='ON'; -SELECT @@GLOBAL.innodb_priority_io; -SET GLOBAL innodb_priority_io=0; -SELECT @@GLOBAL.innodb_priority_io; -SET GLOBAL innodb_priority_io=1; -SELECT @@GLOBAL.innodb_priority_io; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_io=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_io=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_io=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_io='foo'; - -SET GLOBAL innodb_priority_io = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_priority_master_basic.test b/mysql-test/suite/sys_vars/t/innodb_priority_master_basic.test deleted file mode 100644 index f202738f4e14d..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_priority_master_basic.test +++ /dev/null @@ -1,36 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc ---source include/linux.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_priority_master; - -# Default value -SELECT @@GLOBAL.innodb_priority_master; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_priority_master; - -# Correct values -SET GLOBAL innodb_priority_master='OFF'; -SELECT @@GLOBAL.innodb_priority_master; -SET GLOBAL innodb_priority_master='ON'; -SELECT @@GLOBAL.innodb_priority_master; -SET GLOBAL innodb_priority_master=0; -SELECT @@GLOBAL.innodb_priority_master; -SET GLOBAL innodb_priority_master=1; -SELECT @@GLOBAL.innodb_priority_master; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_master=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_master=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_master=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_master='foo'; - -SET GLOBAL innodb_priority_master = @start_value; diff --git a/mysql-test/suite/sys_vars/t/innodb_priority_purge_basic.test b/mysql-test/suite/sys_vars/t/innodb_priority_purge_basic.test deleted file mode 100644 index b17a97838a516..0000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_priority_purge_basic.test +++ /dev/null @@ -1,36 +0,0 @@ ---source include/have_debug.inc ---source include/have_xtradb.inc ---source include/linux.inc - -# A dynamic, global variable - -SET @start_value = @@GLOBAL.innodb_priority_purge; - -# Default value -SELECT @@GLOBAL.innodb_priority_purge; - -# Global only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -SELECT @@SESSION.innodb_priority_purge; - -# Correct values -SET GLOBAL innodb_priority_purge='OFF'; -SELECT @@GLOBAL.innodb_priority_purge; -SET GLOBAL innodb_priority_purge='ON'; -SELECT @@GLOBAL.innodb_priority_purge; -SET GLOBAL innodb_priority_purge=0; -SELECT @@GLOBAL.innodb_priority_purge; -SET GLOBAL innodb_priority_purge=1; -SELECT @@GLOBAL.innodb_priority_purge; - -# Incorrect values ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_purge=1.1; ---error ER_WRONG_TYPE_FOR_VAR -SET GLOBAL innodb_priority_purge=1e1; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_purge=2; ---error ER_WRONG_VALUE_FOR_VAR -SET GLOBAL innodb_priority_purge='foo'; - -SET GLOBAL innodb_priority_purge = @start_value; diff --git a/mysql-test/t/create_or_replace2.test b/mysql-test/t/create_or_replace2.test index be1bd9a3d817b..6cee7fac2e947 100644 --- a/mysql-test/t/create_or_replace2.test +++ b/mysql-test/t/create_or_replace2.test @@ -5,7 +5,7 @@ --source include/have_debug.inc --source include/master-slave.inc --source include/have_binlog_format_row.inc ---source include/have_xtradb.inc +--source include/have_innodb.inc --disable_warnings drop table if exists t1; diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test index 6a1cb53dc404e..3a2601342ba23 100644 --- a/mysql-test/t/index_merge_innodb.test +++ b/mysql-test/t/index_merge_innodb.test @@ -12,7 +12,7 @@ # Slow test, don't run during staging part --source include/not_staging.inc ---source include/have_xtradb.inc +--source include/have_innodb.inc let $engine_type= InnoDB; # InnoDB does not support Merge tables (affects include/index_merge1.inc) diff --git a/mysql-test/t/information_schema.test b/mysql-test/t/information_schema.test index a4bc1cb299121..157b4c6950805 100644 --- a/mysql-test/t/information_schema.test +++ b/mysql-test/t/information_schema.test @@ -8,8 +8,7 @@ # on the presence of the log tables (which are CSV-based). --source include/have_csv.inc -# Check that XtraDB is enabled as result depends on it --- source include/have_xtradb.inc +-- source include/have_innodb.inc # Save the initial number of concurrent sessions --source include/count_sessions.inc diff --git a/mysql-test/t/information_schema_all_engines.test b/mysql-test/t/information_schema_all_engines.test index 8c898909c1e36..f8d685d25605c 100644 --- a/mysql-test/t/information_schema_all_engines.test +++ b/mysql-test/t/information_schema_all_engines.test @@ -3,7 +3,7 @@ # available (since these engines inject tables into INFORMATION_SCHEMA). --source include/not_embedded.inc ---source include/have_xtradb.inc +--source include/have_innodb.inc --source include/have_perfschema.inc --source include/not_staging.inc diff --git a/mysql-test/t/multi_update.test b/mysql-test/t/multi_update.test index 0297280340db5..4e09b2370f508 100644 --- a/mysql-test/t/multi_update.test +++ b/mysql-test/t/multi_update.test @@ -8,17 +8,6 @@ source include/have_log_bin.inc; CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); ---disable_warnings -drop table if exists t1,t2,t3; -drop database if exists mysqltest; -drop view if exists v1; ---error 0,ER_NONEXISTING_GRANT,ER_NONEXISTING_TABLE_GRANT -revoke all privileges on mysqltest.t1 from mysqltest_1@localhost; ---error 0,ER_NONEXISTING_GRANT,ER_NONEXISTING_TABLE_GRANT -revoke all privileges on mysqltest.* from mysqltest_1@localhost; -delete from mysql.user where user=_binary'mysqltest_1'; ---enable_warnings - create table t1(id1 int not null auto_increment primary key, t char(12)); create table t2(id2 int not null, t char(12)); create table t3(id3 int not null, t char(12), index(id3)); @@ -372,9 +361,7 @@ drop table t1, t2; connect (root,localhost,root,,test,$MASTER_MYPORT,$MASTER_MYSOCK); connection root; ---disable_warnings create database mysqltest; ---enable_warnings create table mysqltest.t1 (a int, b int, primary key (a)); create table mysqltest.t2 (a int, b int, primary key (a)); create table mysqltest.t3 (a int, b int, primary key (a)); @@ -415,22 +402,6 @@ update t1,t2 set t1.col1 = (select max(col1) from t1) where t1.col1 = t2.col1; delete t1 from t1,t2 where t1.col1 < (select max(col1) from t1) and t1.col1 = t2.col1; drop table t1,t2; -# Test for Bug#5837 delete with outer join and const tables ---disable_warnings -create table t1 ( - aclid bigint not null primary key, - status tinyint(1) not null -) engine = innodb; - -create table t2 ( - refid bigint not null primary key, - aclid bigint, index idx_acl(aclid) -) engine = innodb; ---enable_warnings -insert into t2 values(1,null); -delete t2, t1 from t2 left join t1 on (t2.aclid=t1.aclid) where t2.refid='1'; -drop table t1, t2; - # # Bug#19225 unchecked error leads to server crash # @@ -441,47 +412,6 @@ delete from t1,t2 using t1,t2 where t1.a=(select a from t1); drop table t1, t2; # End of 4.1 tests -# -# Test for Bug#1980. -# ---disable_warnings -create table t1 ( c char(8) not null ) engine=innodb; ---enable_warnings - -insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); -insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); - -alter table t1 add b char(8) not null; -alter table t1 add a char(8) not null; -alter table t1 add primary key (a,b,c); -update t1 set a=c, b=c; - -create table t2 like t1; -insert into t2 select * from t1; - -delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; - -drop table t1,t2; - ---disable_warnings -create table t1 ( c char(8) not null ) engine=innodb; ---enable_warnings - -insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); -insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); - -alter table t1 add b char(8) not null; -alter table t1 add a char(8) not null; -alter table t1 add primary key (a,b,c); -update t1 set a=c, b=c; - -create table t2 like t1; -insert into t2 select * from t1; - -delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; - -drop table t1,t2; - # # Test alter table and a concurrent multi update # (Before we have introduced data-lock-aware metadata locks @@ -618,9 +548,6 @@ set @@session.binlog_format= @sav_binlog_format; # # prepare ---disable_warnings -drop table if exists t1, t2, t3; ---enable_warnings CREATE TABLE t1 (a int, PRIMARY KEY (a)); CREATE TABLE t2 (a int, PRIMARY KEY (a)); CREATE TABLE t3 (a int, PRIMARY KEY (a)) ENGINE=MyISAM; @@ -1057,54 +984,3 @@ deallocate prepare stmt1; drop view v3,v2,v1; drop table t1,t2,t3; --echo end of 5.5 tests - - ---source include/have_xtradb.inc - ---echo ---echo # Bug mdev-5970 ---echo # Bug#13256831 - ERROR 1032 (HY000): CAN'T FIND RECORD ---echo - -CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; -CREATE TABLE t2 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; -INSERT INTO t1 VALUES (5, 7); -INSERT INTO t2 VALUES (6, 97); - -CREATE ALGORITHM = MERGE VIEW v1 AS -SELECT a2.f1 AS f1, a2.f2 AS f2 -FROM t1 AS a1 JOIN t2 AS a2 ON a1.f2 > a2.f1 -WITH LOCAL CHECK OPTION; - -SELECT * FROM v1; -UPDATE v1 SET f1 = 1; -SELECT * FROM v1; - -DROP TABLE t1, t2; -DROP VIEW v1; - ---echo # ---echo # MDEV-5973: MySQL Bug#11757486:49539: NON-DESCRIPTIVE ERR (ERROR 0 ---echo # FROM STORAGE ENGINE) WITH MULTI-TABLE UPDATE ---echo # - -CREATE TABLE table_11757486 (field1 tinyint) ENGINE=INNODB; -INSERT INTO table_11757486 VALUES (0),(0); -SET SESSION SQL_MODE='STRICT_ALL_TABLES'; -UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; -UPDATE IGNORE table_11757486 SET field1=128; - ---error ER_WARN_DATA_OUT_OF_RANGE -UPDATE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; ---error ER_WARN_DATA_OUT_OF_RANGE -UPDATE table_11757486 SET field1=128; - -SET SESSION SQL_MODE=''; -UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; -UPDATE IGNORE table_11757486 SET field1=128; - -DROP TABLE table_11757486; - -SET SESSION SQL_MODE=default; - ---echo end of 10.0 tests diff --git a/mysql-test/t/multi_update_innodb.test b/mysql-test/t/multi_update_innodb.test index 51757c29553b3..f5f8f91edb8f4 100644 --- a/mysql-test/t/multi_update_innodb.test +++ b/mysql-test/t/multi_update_innodb.test @@ -75,3 +75,101 @@ UPDATE t2 AS A NATURAL JOIN t2 B SET A.pk_2=10,B.pk_2=11; SELECT * FROM t2; DROP TABLE t1,t2; + +--echo +--echo # Bug mdev-5970 +--echo # Bug#13256831 - ERROR 1032 (HY000): CAN'T FIND RECORD +--echo + +CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +CREATE TABLE t2 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (5, 7); +INSERT INTO t2 VALUES (6, 97); + +CREATE ALGORITHM = MERGE VIEW v1 AS +SELECT a2.f1 AS f1, a2.f2 AS f2 +FROM t1 AS a1 JOIN t2 AS a2 ON a1.f2 > a2.f1 +WITH LOCAL CHECK OPTION; + +SELECT * FROM v1; +UPDATE v1 SET f1 = 1; +SELECT * FROM v1; + +DROP TABLE t1, t2; +DROP VIEW v1; + +--echo # +--echo # MDEV-5973: MySQL Bug#11757486:49539: NON-DESCRIPTIVE ERR (ERROR 0 +--echo # FROM STORAGE ENGINE) WITH MULTI-TABLE UPDATE +--echo # + +CREATE TABLE table_11757486 (field1 tinyint) ENGINE=INNODB; +INSERT INTO table_11757486 VALUES (0),(0); +SET SESSION SQL_MODE='STRICT_ALL_TABLES'; +UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; +UPDATE IGNORE table_11757486 SET field1=128; + +--error ER_WARN_DATA_OUT_OF_RANGE +UPDATE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; +--error ER_WARN_DATA_OUT_OF_RANGE +UPDATE table_11757486 SET field1=128; + +SET SESSION SQL_MODE=''; +UPDATE IGNORE (SELECT 128 as col1) x, table_11757486 SET field1=x.col1; +UPDATE IGNORE table_11757486 SET field1=128; + +DROP TABLE table_11757486; + +SET SESSION SQL_MODE=default; + +# Test for Bug#5837 delete with outer join and const tables +create table t1 ( + aclid bigint not null primary key, + status tinyint(1) not null +) engine = innodb; + +create table t2 ( + refid bigint not null primary key, + aclid bigint, index idx_acl(aclid) +) engine = innodb; +insert into t2 values(1,null); +delete t2, t1 from t2 left join t1 on (t2.aclid=t1.aclid) where t2.refid='1'; +drop table t1, t2; + +# +# Test for Bug#1980. +# +create table t1 ( c char(8) not null ) engine=innodb; + +insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); +insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); + +alter table t1 add b char(8) not null; +alter table t1 add a char(8) not null; +alter table t1 add primary key (a,b,c); +update t1 set a=c, b=c; + +create table t2 like t1; +insert into t2 select * from t1; + +delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; + +drop table t1,t2; + +create table t1 ( c char(8) not null ) engine=innodb; + +insert into t1 values ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9'); +insert into t1 values ('A'),('B'),('C'),('D'),('E'),('F'); + +alter table t1 add b char(8) not null; +alter table t1 add a char(8) not null; +alter table t1 add primary key (a,b,c); +update t1 set a=c, b=c; + +create table t2 like t1; +insert into t2 select * from t1; + +delete t1,t2 from t2,t1 where t1.a<'B' and t2.b=t1.b; + +drop table t1,t2; + diff --git a/mysql-test/t/progress_976225.test b/mysql-test/t/progress_976225.test index 32ded8b0ee027..885f9d69de2eb 100644 --- a/mysql-test/t/progress_976225.test +++ b/mysql-test/t/progress_976225.test @@ -1,4 +1,4 @@ ---source include/have_xtradb.inc +--source include/have_innodb.inc CREATE TABLE t1 (a INT) ENGINE=InnoDB; CREATE TABLE t2 (b INT) ENGINE=InnoDB; diff --git a/mysql-test/t/rowid_order_innodb.test b/mysql-test/t/rowid_order_innodb.test index 1053b8bd7c299..152eb28d38870 100644 --- a/mysql-test/t/rowid_order_innodb.test +++ b/mysql-test/t/rowid_order_innodb.test @@ -8,7 +8,7 @@ # main code t/rowid_order_innodb.test -> include/rowid_order.inc # ---source include/have_xtradb.inc +--source include/have_innodb.inc let $engine_type= InnoDB; --source include/rowid_order.inc diff --git a/mysql-test/t/xtradb_mrr.test b/mysql-test/t/xtradb_mrr.test index d994c182cccdb..b56cbb0459fb4 100644 --- a/mysql-test/t/xtradb_mrr.test +++ b/mysql-test/t/xtradb_mrr.test @@ -1,4 +1,4 @@ --- source include/have_xtradb.inc +-- source include/have_innodb.inc --disable_warnings drop table if exists t1,t2,t3,t4; diff --git a/storage/tokudb/mysql-test/tokudb_mariadb/t/xa-recovery-9214.test b/storage/tokudb/mysql-test/tokudb_mariadb/t/xa-recovery-9214.test index 3854fb99a3c32..7d396eb84f41b 100644 --- a/storage/tokudb/mysql-test/tokudb_mariadb/t/xa-recovery-9214.test +++ b/storage/tokudb/mysql-test/tokudb_mariadb/t/xa-recovery-9214.test @@ -1,7 +1,7 @@ # # MDEV-9214 Server miscalculates the number of XA-capable engines # ---source include/have_xtradb.inc +--source include/have_innodb.inc select 1; From f2fe65106f731df48072a16eea3cda0a903d7fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Dec 2016 17:13:14 +0200 Subject: [PATCH 295/295] MDEV-11679 Remove redundant function fsp_header_get_crypt_offset() fsp_header_get_crypt_offset(): Remove. xdes_arr_size(): Remove. fsp_header_get_encryption_offset(): Make this an inline function. The correctness of this change was ensured with the following patch that ensures that the two functions returned the same value, only differing by FSP_HEADER_OFFSET (38 bytes): diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index f2a4c6bf218..e96c788b7df 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -850,6 +850,7 @@ fsp_parse_init_file_page( return(ptr); } +static ulint fsp_header_get_encryption_offset(const page_size_t&); /**********************************************************************//** Initializes the fsp system. */ void @@ -868,6 +869,31 @@ fsp_init(void) #endif /* Does nothing at the moment */ + + for (ulint sz = 4096; sz <= 65536; sz *= 2) { + ulint m; + if (sz <= 16384) { + for (ulint ph = 1024; ph <= sz; ph *= 2) { + const page_size_t ps(ph, sz, true); + ulint maria = fsp_header_get_crypt_offset(ps, &m), + oracle = fsp_header_get_encryption_offset(ps); + if (maria != oracle + 38) { + ib::error() << "zip size mismatch: " + << maria << "!=" << oracle + << "(" << ph <<","<page0_offset = - fsp_header_get_crypt_offset(page_size, &maxsize); + crypt_data->page0_offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size); + const ulint maxsize = page_size.logical() + - crypt_data->page0_offset - FIL_PAGE_DATA_END; /* 4 - write crypt data to page 0 */ fil_space_write_crypt_data_low(crypt_data, @@ -2165,10 +2166,9 @@ fil_crypt_flush_space( if (block && err == DB_SUCCESS) { byte* frame = buf_block_get_frame(block); - ulint maxsize=0; - crypt_data->page0_offset = - fsp_header_get_crypt_offset(page_size, &maxsize); + crypt_data->page0_offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size); fil_space_write_crypt_data(space, frame, crypt_data->page0_offset, diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 3bd3237d1aa78..e1e97fc6e6492 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6661,12 +6661,11 @@ fil_tablespace_iterate( iter.n_io_buffers = n_io_buffers; iter.page_size = callback.get_page_size().physical(); - ulint crypt_data_offset = fsp_header_get_crypt_offset( - callback.get_page_size(), 0); - /* read (optional) crypt data */ iter.crypt_data = fil_space_read_crypt_data( - 0, page, crypt_data_offset); + 0, page, FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset( + callback.get_page_size())); #ifdef MYSQL_ENCRYPTION /* Set encryption info. */ @@ -7612,8 +7611,9 @@ fil_space_get_crypt_data( fil_read(page_id_t(space_id, 0), univ_page_size, 0, univ_page_size.physical(), page); ulint flags = fsp_header_get_flags(page); - ulint offset = fsp_header_get_crypt_offset( - page_size_t(flags), NULL); + ulint offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset( + page_size_t(flags)); space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 3c12bde0acf1f..4a7df5b33d3fb 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -368,9 +368,10 @@ Datafile::read_first_page(bool read_only_mode) m_space_id = fsp_header_get_space_id(m_first_page); } - const page_size_t page_sz = fsp_header_get_page_size(m_first_page); - ulint offset = fsp_header_get_crypt_offset(page_sz, NULL); - m_crypt_info = fil_space_read_crypt_data(m_space_id, m_first_page, offset); + m_crypt_info = fil_space_read_crypt_data( + m_space_id, m_first_page, + FSP_HEADER_OFFSET + fsp_header_get_encryption_offset( + fsp_header_get_page_size(m_first_page))); return(err); } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index f2a4c6bf2188b..2ccad2801d16b 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -890,29 +890,6 @@ fsp_header_init_fields( } #ifndef UNIV_HOTBACKUP -/** Get the offset of encrytion information in page 0. -@param[in] page_size page size. -@return offset on success, otherwise 0. */ -static -ulint -fsp_header_get_encryption_offset( - const page_size_t& page_size) -{ - ulint offset; -#ifdef UNIV_DEBUG - ulint left_size; -#endif - - offset = XDES_ARR_OFFSET + XDES_SIZE * xdes_arr_size(page_size); -#ifdef UNIV_DEBUG - left_size = page_size.physical() - FSP_HEADER_OFFSET - offset - - FIL_PAGE_DATA_END; - - ut_ad(left_size >= ENCRYPTION_INFO_SIZE_V2); -#endif - - return offset; -} #if 0 /* MySQL 5.7 Encryption */ /** Fill the encryption info. @@ -1180,9 +1157,11 @@ fsp_header_init( } } - ulint maxsize = 0; - ulint offset = fsp_header_get_crypt_offset(page_size, &maxsize); - fil_space_write_crypt_data(space_id, page, offset, maxsize, mtr); + ulint offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size); + fil_space_write_crypt_data(space_id, page, offset, + page_size.logical() + - offset - FIL_PAGE_DATA_END, mtr); return(true); } @@ -4342,35 +4321,3 @@ fsp_page_is_free_func( return xdes_mtr_get_bit( descr, XDES_FREE_BIT, page_no % FSP_EXTENT_SIZE, mtr); } - -/**********************************************************************//** -Compute offset after xdes where crypt data can be stored -@return offset */ -ulint -fsp_header_get_crypt_offset( -/*========================*/ - const page_size_t& page_size,/*!< in: page size */ - ulint* max_size) /*!< out: free space available for crypt data */ -{ - ulint pageno = 0; - /* compute first page_no that will have xdes stored on page != 0*/ - - for (ulint i = 0; - (pageno = xdes_calc_descriptor_page(page_size, i)) == 0; ) - i++; - - /* use pageno prior to this...i.e last page on page 0 */ - ut_ad(pageno > 0); - pageno--; - - ulint iv_offset = XDES_ARR_OFFSET + - XDES_SIZE * (1 + xdes_calc_descriptor_index(page_size, pageno)); - - if (max_size != NULL) { - /* return how much free space there is available on page */ - *max_size = (page_size.logical() ? page_size.logical() : UNIV_PAGE_SIZE) - - (FSP_HEADER_OFFSET + iv_offset + FIL_PAGE_DATA_END); - } - - return FSP_HEADER_OFFSET + iv_offset; -} diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 17ccb113e9151..f503580a1a72c 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -361,6 +361,16 @@ fsp_header_get_encryption_key( byte* iv, page_t* page); +/** Get the byte offset of encryption information in page 0. +@param[in] ps page size +@return byte offset relative to FSP_HEADER_OFFSET */ +inline MY_ATTRIBUTE((pure, warn_unused_result)) +ulint +fsp_header_get_encryption_offset(const page_size_t& ps) +{ + return XDES_ARR_OFFSET + XDES_SIZE * ps.physical() / FSP_EXTENT_SIZE; +} + /** Check the encryption key from the first page of a tablespace. @param[in] fsp_flags tablespace flags @param[in] page first page of a tablespace @@ -770,20 +780,6 @@ xdes_calc_descriptor_page( const page_size_t& page_size, ulint offset); -#endif /* !UNIV_INNOCHECKSUM */ - -/*********************************************************************//** -@return offset into fsp header where crypt data is stored */ -UNIV_INTERN -ulint -fsp_header_get_crypt_offset( -/*========================*/ - const page_size_t& page_size,/*!< in: page size */ - ulint* max_size); /*!< out: free space after offset */ - - -#ifndef UNIV_INNOCHECKSUM - /**********************************************************************//** Checks if a single page is free. @return true if free */ diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 475dd2387287b..c675c6302a6e1 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -337,15 +337,4 @@ xdes_calc_descriptor_page( return(ut_2pow_round(offset, page_size.physical())); } - -/** Calculates the descriptor array size. -@param[in] page_size page size -@return size of descriptor array */ -UNIV_INLINE -ulint -xdes_arr_size( - const page_size_t& page_size) -{ - return(page_size.physical()/FSP_EXTENT_SIZE); -} #endif /* !UNIV_INNOCHECKSUM */