Skip to content

Commit ca15214

Browse files
author
Pavan Naik
committed
WL#12113: MTR SUPPORT FOR AUTOMATIC LOADING OF DATA TO SECONDARY
 ENGINE AFTER A DML STATEMENT Descrption: ----------- This WL will enhance --change-propagation option to take an optional argument (i.e either 0 or 1). Setting this option value to 1 is same as specifying the option without an argument. When the value is set to 0, MTR will automatically unload and then load the data back to secondary engine after a DML statement. The scope of this WL is limited to following DML statements 1. DELETE 2. INSERT 3. REPLACE 4. UPDATE Change-Id: I28a189fe869d3216d246d59fed6d19f7b5d3db72
1 parent df6c2f1 commit ca15214

8 files changed

+369
-246
lines changed

‎client/mysqltest.cc

+4-3
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
#include <functional>
3737
#include <limits>
3838
#include <new>
39-
#include <string>
4039
#ifdef _WIN32
4140
#include <thread> // std::thread
4241
#endif
@@ -7768,7 +7767,8 @@ static void run_query_normal(struct st_connection *cn,
77687767
std::vector<unsigned int> ignore_errors = expected_errors->errors();
77697768
// Run secondary engine load statements.
77707769
if (run_secondary_engine_load_statements(opt_secondary_engine, query, mysql,
7771-
ignore_errors))
7770+
ignore_errors,
7771+
opt_change_propagation))
77727772
die("Original query '%s'.", query);
77737773
}
77747774

@@ -7997,7 +7997,8 @@ static void run_query_stmt(MYSQL *mysql, struct st_command *command,
79977997
std::vector<unsigned int> ignore_errors = expected_errors->errors();
79987998
// Run secondary engine load statements.
79997999
if (run_secondary_engine_load_statements(opt_secondary_engine, query, mysql,
8000-
ignore_errors))
8000+
ignore_errors,
8001+
opt_change_propagation))
80018002
die("Original query '%s'.", query);
80028003
}
80038004

‎client/mysqltest/secondary_engine.cc

+97-11
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <iostream>
2727
#include <map>
2828
#include <regex>
29+
#include <sstream>
2930
#include <string>
3031

3132
#include "client/mysqltest/error_names.h"
@@ -112,10 +113,18 @@ static bool run_secondary_load_statement(MYSQL *mysql, std::string table_name) {
112113
ignore_errors.push_back(get_errcode_from_name("ER_CHECK_NOT_IMPLEMENTED"));
113114
ignore_errors.push_back(get_errcode_from_name("ER_RAPID_PLUGIN"));
114115

115-
// ALTER TABLE statement
116-
std::string alter_stmt = "ALTER TABLE " + table_name + " SECONDARY_LOAD;";
116+
std::stringstream table_names(table_name);
117+
std::string table;
117118

118-
return run_alter_statement(alter_stmt, mysql, ignore_errors);
119+
// DML Statements like DELETE or UPDATE may contain multiple table
120+
// names separated by comma.
121+
while (std::getline(table_names, table, ',')) {
122+
// ALTER TABLE statement to load the data to secondary engine.
123+
std::string alter_stmt = "ALTER TABLE " + table + " SECONDARY_LOAD;";
124+
if (run_alter_statement(alter_stmt, mysql, ignore_errors)) return true;
125+
}
126+
127+
return false;
119128
}
120129

121130
/// Run ALTER TABLE table_name SECONDARY_UNLOAD statement to unload
@@ -139,10 +148,18 @@ static bool run_secondary_unload_statement(
139148
ignore_errors.push_back(get_errcode_from_name("ER_DBACCESS_DENIED_ERROR"));
140149
ignore_errors.push_back(get_errcode_from_name("ER_SECONDARY_ENGINE"));
141150

142-
// ALTER TABLE statement
143-
std::string alter_stmt = "ALTER TABLE " + table_name + " SECONDARY_UNLOAD;";
151+
std::stringstream table_names(table_name);
152+
std::string table;
144153

145-
return run_alter_statement(alter_stmt, mysql, ignore_errors);
154+
// DML Statements like DELETE or UPDATE may contain multiple table
155+
// names separated by comma.
156+
while (std::getline(table_names, table, ',')) {
157+
// ALTER TABLE statement to unload the data from secondary engine.
158+
std::string alter_stmt = "ALTER TABLE " + table + " SECONDARY_UNLOAD;";
159+
if (run_alter_statement(alter_stmt, mysql, ignore_errors)) return true;
160+
}
161+
162+
return false;
146163
}
147164

148165
/// Match a statement string against the pattern. If match found,
@@ -226,20 +243,72 @@ static bool match_ddl_statement(std::string statement,
226243
return false;
227244
}
228245

246+
/// Check if a statement is a DML statement. If yes, parse the statement
247+
/// to extract the table name and store it in a string object.
248+
///
249+
/// @param statement Original statement
250+
/// @param table_name String object to store the table name
251+
///
252+
/// @retval True if match or table name found, false otherwise.
253+
static bool match_dml_statement(std::string statement,
254+
std::string *table_name) {
255+
std::map<std::string, std::uint16_t> dml_pattern_strings = {
256+
{"^\\s*delete\\s+((\\/\\*\\+.*\\*\\/"
257+
"\\s+)?)((LOW_PRIORITY\\s+)?)((QUICK\\s+)?)((IGNORE\\s+)?)"
258+
"FROM\\s+(.*?)((\\s+.*)?)",
259+
9},
260+
{"^\\s*insert\\s+((\\/\\*\\+.*\\*\\/"
261+
"\\s+)?)((LOW_PRIORITY\\s+|DELAYED\\s+|HIGH_PRIORITY\\s+)?)"
262+
"((IGNORE\\s+)?)((INTO\\s+)?)(.*?)(\\s*\\(|\\s+).*",
263+
9},
264+
{"^\\s*replace\\s+((\\/\\*\\+.*\\*\\/"
265+
"\\s+)?)((LOW_PRIORITY\\s+|DELAYED\\s+)?)((INTO\\s+)?)(.*?)"
266+
"(\\s*\\(|\\s+).*",
267+
7},
268+
{"^\\s*update\\s+((\\/\\*\\+.*\\*\\/"
269+
"\\s+)?)((LOW_PRIORITY\\s+)?)((IGNORE\\s+)?)(.*?)\\s+set.*",
270+
7}};
271+
272+
for (std::pair<std::string, std::uint16_t> pattern : dml_pattern_strings) {
273+
*table_name =
274+
regex_match_statement(statement, pattern.first, pattern.second);
275+
if (table_name->length()) return true;
276+
}
277+
278+
return false;
279+
}
280+
281+
/// Check if a statement is a TRUNCATE statement. If yes, parse the
282+
/// statement to extract the table name and store it in a string object.
283+
///
284+
/// @param statement Original statement
285+
/// @param table_name String object to store the table name
286+
///
287+
/// @retval True if the match or table name found, false otherwise.
288+
static bool match_truncate_statement(std::string statement,
289+
std::string *table_name) {
290+
std::string pattern_str = "^\\s*truncate\\s+(table\\s+)?(.*?)(\\s+.*|\\s*)";
291+
*table_name = regex_match_statement(statement, pattern_str, 2);
292+
if (table_name->length()) return true;
293+
return false;
294+
}
295+
229296
/// Check if the statement is a CREATE TABLE statement or a DDL
230297
/// statement. If yes, run the ALTER TABLE statements needed to change
231298
/// the secondary engine and to load the data from primary engine to
232299
/// secondary engine.
233300
///
234-
/// @param secondary_engine Secondary engine name
235-
/// @param statement Original statement
236-
/// @param mysql mysql handle
237-
/// @param expected_errors List of expected errors
301+
/// @param secondary_engine Secondary engine name
302+
/// @param statement Original statement
303+
/// @param mysql mysql handle
304+
/// @param expected_errors List of expected errors
305+
/// @param opt_change_propagation Boolean flag indicating whether change
306+
/// propagation is enabled or not.
238307
///
239308
/// @retval True if load operation fails, false otherwise.
240309
bool run_secondary_engine_load_statements(
241310
const char *secondary_engine, char *statement, MYSQL *mysql,
242-
std::vector<unsigned int> expected_errors) {
311+
std::vector<unsigned int> expected_errors, bool opt_change_propagation) {
243312
std::string table_name("");
244313

245314
// Check if the statement is a CREATE TABLE statement or a DDL statement
@@ -262,6 +331,23 @@ bool run_secondary_engine_load_statements(
262331
// Load the data from primary engine to secondary engine.
263332
if (run_secondary_load_statement(mysql, table_name)) return true;
264333
}
334+
} else if (expected_errors.size() == 0) {
335+
if (match_truncate_statement(statement, &table_name) ||
336+
(!opt_change_propagation &&
337+
match_dml_statement(statement, &table_name))) {
338+
DBUG_ASSERT(table_name.length());
339+
340+
// Unload the data from secondary engine.
341+
if (run_secondary_unload_statement(mysql, table_name, expected_errors))
342+
return true;
343+
344+
// Skip running ALTER TABLE statement to load the data from primary
345+
// engine to secondary engine if the previous ALTER TABLE statement
346+
// failed with an error.
347+
if (mysql_errno(mysql) == 0)
348+
// Load the data from primary engine to secondary engine.
349+
if (run_secondary_load_statement(mysql, table_name)) return true;
350+
}
265351
}
266352

267353
return false;

‎client/mysqltest/secondary_engine.h

+7-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
#ifndef SECONDARY_ENGINE_INCLUDED
32
#define SECONDARY_ENGINE_INCLUDED
43
// Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
@@ -32,15 +31,17 @@
3231
/// the secondary engine and to load the data from primary engine to
3332
/// secondary engine.
3433
///
35-
/// @param secondary_engine Secondary engine name
36-
/// @param statement Original statement
37-
/// @param mysql mysql handle
38-
/// @param expected_errors List of expected errors
34+
/// @param secondary_engine Secondary engine name
35+
/// @param statement Original statement
36+
/// @param mysql mysql handle
37+
/// @param expected_errors List of expected errors
38+
/// @param opt_change_propagation Boolean flag indicating whether change
39+
/// propagation is enabled or not.
3940
///
4041
/// @retval True if load operation fails, false otherwise.
4142
bool run_secondary_engine_load_statements(
4243
const char *secondary_engine, char *statement, MYSQL *mysql,
43-
std::vector<unsigned int> expected_errors);
44+
std::vector<unsigned int> expected_errors, bool opt_change_propagation);
4445

4546
/// Check if the statement is a DDL statement. If yes, run the ALTER
4647
/// TABLE statements needed to change the secondary engine to NULL and

‎mysql-test/lib/My/ConfigFactory.pm

+28-29
Original file line numberDiff line numberDiff line change
@@ -331,23 +331,24 @@ sub fix_rapid_load_pool {
331331
}
332332

333333
# Rules to run for each mysqld server in the config if
334-
# secondary engine rapid is enabled.
334+
# secondary engine is enabled.
335335
# - will be run in order listed here
336-
my @rapid_mysqld_rules = ({ 'binlog-format' => "ROW" },
337-
{ 'binlog-row-image' => "FULL" },
338-
{ 'binlog-checksum' => "NONE" },
339-
{ 'gtid-mode' => "OFF" },
340-
{ 'log-bin' => "binlog" },
341-
{ 'loose-rapid_load_pool' => \&fix_rapid_load_pool },
342-
{ 'loose-rapid_use_rowsets' => "OFF" },
343-
{ 'innodb_buffer_pool_size' => 268435456 },
344-
{ 'innodb_read_io_threads' => 32 },
345-
{ 'innodb_thread_concurrency' => 32 },
346-
{ 'innodb_use_native_aio' => "ON" },);
347-
348-
# Rules to run for each rapid server in the config
336+
my @secondary_engine_mysqld_rules = (
337+
{ 'binlog-format' => "ROW" },
338+
{ 'binlog-row-image' => "FULL" },
339+
{ 'binlog-checksum' => "NONE" },
340+
{ 'gtid-mode' => "OFF" },
341+
{ 'log-bin' => "binlog" },
342+
{ 'loose-rapid_load_pool' => \&fix_rapid_load_pool },
343+
{ 'loose-rapid_use_rowsets' => "OFF" },
344+
{ 'innodb_buffer_pool_size' => 268435456 },
345+
{ 'innodb_read_io_threads' => 32 },
346+
{ 'innodb_thread_concurrency' => 32 },
347+
{ 'innodb_use_native_aio' => "ON" },);
348+
349+
# Rules to run for each secondary engine server in the config
349350
# - will be run in order listed here
350-
my @rapid_rules = (
351+
my @secondary_engine_rules = (
351352
{ 'load_multi_buffering' => 2 },
352353
{ 'memory_heap_size' => \&fix_memory_heap_size },
353354
{ 'orma_heartbeat_check_interval' => 60 },
@@ -360,7 +361,7 @@ my @rapid_rules = (
360361
{ 'trans_iplist' => "127.0.0.1" },
361362
{ 'trans_spareiplist' => "" },
362363

363-
{ 'trans_ports' => sub { return $::rapid_port }
364+
{ 'trans_ports' => sub { return $::secondary_engine_port }
364365
},);
365366

366367
sub fix_ndb_mgmd_port {
@@ -426,12 +427,11 @@ my @mysqlbinlog_rules = (
426427
# - will be run in order listed here
427428
my @mysql_upgrade_rules = ();
428429

429-
# Generate a [rapid] group to be used for connecting
430-
# to a rapid server.
431-
sub post_check_rapid_group {
430+
# Generate a group to be used for connecting to a secondary engine server.
431+
sub post_check_secondary_engine_group {
432432
my ($self, $config) = @_;
433433

434-
foreach my $hash (@rapid_rules) {
434+
foreach my $hash (@secondary_engine_rules) {
435435
while (my ($option, $rule) = each(%{$hash})) {
436436
my $value;
437437
if (ref $rule eq "CODE") {
@@ -445,12 +445,11 @@ sub post_check_rapid_group {
445445
}
446446
}
447447

448-
# Insert mysqld server options required when secondary engine
449-
# rapid is enabled.
450-
sub post_check_rapid_mysqld_group {
448+
# Insert mysqld server options required when secondary engine is enabled.
449+
sub post_check_secondary_engine_mysqld_group {
451450
my ($self, $config) = @_;
452451

453-
foreach my $hash (@rapid_mysqld_rules) {
452+
foreach my $hash (@secondary_engine_mysqld_rules) {
454453
while (my ($option, $rule) = each(%{$hash})) {
455454
my $value;
456455
if (ref $rule eq "CODE") {
@@ -721,11 +720,11 @@ sub new_config {
721720
$self->run_rules_for_group($config, $config->insert('mysqltest'),
722721
@mysqltest_rules);
723722

724-
if ($::secondary_engine_rapid) {
725-
# Additional rules required for [rapid]
726-
push(@post_rules, \&post_check_rapid_group);
727-
# Additional rules required for [mysqld] when rapid is enabled
728-
push(@post_rules, \&post_check_rapid_mysqld_group);
723+
if ($::opt_secondary_engine) {
724+
# Additional rules required for secondary engine
725+
push(@post_rules, \&post_check_secondary_engine_group);
726+
# Additional rules required for [mysqld] when secondary engine is enabled
727+
push(@post_rules, \&post_check_secondary_engine_mysqld_group);
729728
}
730729

731730
# Run post rules

‎mysql-test/lib/mtr_cases.pm

+11-9
Original file line numberDiff line numberDiff line change
@@ -770,12 +770,12 @@ sub optimize_cases {
770770
# Skip processing if already marked as skipped
771771
next if $tinfo->{skip};
772772

773-
# Binlog must be enabled for rapid
774-
if ($::secondary_engine_rapid) {
773+
# Binlog must be enabled for secondary engine
774+
if ($::opt_secondary_engine) {
775775
my $skip_log_bin_pattern = "^--(loose[-_])?skip[-_]log[-_]bin";
776776
if (grep(/$skip_log_bin_pattern/, @{ $tinfo->{'master_opt'} }) ||
777777
grep(/$skip_log_bin_pattern/, @{ $tinfo->{'slave_opt'} })) {
778-
skip_test($tinfo, "Binlog must be enabled for RAPID.");
778+
skip_test($tinfo, "Binlog must be enabled.");
779779
next;
780780
}
781781
}
@@ -796,9 +796,10 @@ sub optimize_cases {
796796
}
797797
}
798798

799-
# Tests with binlog_format other than ROW can't be run with rapid.
800-
if ($::secondary_engine_rapid and "row" ne lc $binlog_format) {
801-
skip_test($tinfo, "Binlog format must be ROW for RAPID.");
799+
# Tests with binlog_format other than ROW can't be run with
800+
# secondary engine.
801+
if ($::opt_secondary_engine and "row" ne lc $binlog_format) {
802+
skip_test($tinfo, "Binlog format must be ROW.");
802803
next;
803804
}
804805
} else {
@@ -827,10 +828,11 @@ sub optimize_cases {
827828
}
828829
}
829830

830-
# Tests with binlog_format other than ROW can't be run with rapid.
831+
# Tests with binlog_format other than ROW can't be run with
832+
# secondary engine.
831833
if (defined $test_binlog_format and !$tinfo->{skip}) {
832-
if ($::secondary_engine_rapid and "row" ne lc $test_binlog_format) {
833-
skip_test($tinfo, "Binlog format must be ROW for RAPID.");
834+
if ($::opt_secondary_engine and "row" ne lc $test_binlog_format) {
835+
skip_test($tinfo, "Binlog format must be ROW.");
834836
next;
835837
}
836838
}

0 commit comments

Comments
 (0)