WL#7491: GTID-based replication applier recovery and positioning

Step 3: Add the syntax for CHANGE REPLICATION SOURCE

ReviewBoard: 26588

Author: pedrolgomes

mysql-test/suite/rpl_gtid/r/rpl_async_conn_failover_checks.result

@@ -12,10 +12,10 @@ CHANGE REPLICATION SOURCE TO SOURCE_AUTO_POSITION=1;
 #
 # 2. Verify disabling AUTO_POSITION while enabling Asynchronous
 #    Replication Connection Failover results in
-#    ER_DISABLE_AUTO_POSITION_REQUIRES_ASYNC_RECONNECT_OFF error.
+#    ER_RPL_ASYNC_RECONNECT_AUTO_POSITION_OFF
 #
 CHANGE REPLICATION SOURCE TO SOURCE_AUTO_POSITION=0, SOURCE_CONNECTION_AUTO_FAILOVER=1;
-ERROR HY000: CHANGE MASTER TO MASTER_AUTO_POSITION = 0 cannot be executed because Asynchronous Replication Connection Failover is enabled i.e. CHANGE MASTER TO SOURCE_CONNECTION_AUTO_FAILOVER = 1.
+ERROR HY000: Failed to enable Asynchronous Replication Connection Failover feature. The CHANGE MASTER TO SOURCE_CONNECTION_AUTO_FAILOVER = 1 can only be set when MASTER_AUTO_POSITION option of CHANGE MASTER TO is enabled.
 #
 # 3. Verify Asynchronous Replication Connection Failover is disabled and
 #    Auto_position is enabled.

mysql-test/suite/rpl_gtid/t/rpl_async_conn_failover_checks.test

@@ -46,13 +46,12 @@ CHANGE REPLICATION SOURCE TO SOURCE_AUTO_POSITION=1;
 --echo #
 --echo # 2. Verify disabling AUTO_POSITION while enabling Asynchronous
 --echo #    Replication Connection Failover results in
---echo #    ER_DISABLE_AUTO_POSITION_REQUIRES_ASYNC_RECONNECT_OFF error.
+--echo #    ER_RPL_ASYNC_RECONNECT_AUTO_POSITION_OFF
 --echo #
 
---error ER_DISABLE_AUTO_POSITION_REQUIRES_ASYNC_RECONNECT_OFF
+--error ER_RPL_ASYNC_RECONNECT_AUTO_POSITION_OFF
 CHANGE REPLICATION SOURCE TO SOURCE_AUTO_POSITION=0, SOURCE_CONNECTION_AUTO_FAILOVER=1;
 
-
 --echo #
 --echo # 3. Verify Asynchronous Replication Connection Failover is disabled and
 --echo #    Auto_position is enabled.

mysql-test/suite/rpl_nogtid/r/rpl_assign_gtids_to_anonymous_transactions.result

@@ -55,7 +55,7 @@ include/sync_slave_sql_with_master.inc
 include/gtid_step_assert.inc [count=2, only_count=0]
 include/stop_slave.inc
 SET GLOBAL GTID_MODE=ON_PERMISSIVE;
-ERROR HY000: SET @@GLOBAL.GTID_MODE = OFF is not allowed because replication channel '' is configured with ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS set to LOCAL or to a UUID. Execute CHANGE MASTER TO ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS = OFF FOR CHANNEL '' before you set @@GLOBAL.GTID_MODE = 'ON_PERMISSIVE'..
+ERROR HY000: SET @@GLOBAL.GTID_MODE = ON_PERMISSIVE is not allowed because replication channel '' is configured with ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS set to LOCAL or to a UUID. Execute CHANGE MASTER TO ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS = OFF FOR CHANNEL '' before you set @@GLOBAL.GTID_MODE = 'ON_PERMISSIVE'.
 CHANGE REPLICATION SOURCE TO ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS = LOCAL, SOURCE_AUTO_POSITION = 1;
 ERROR HY000: The options ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS = LOCAL|<UUID> and MASTER_AUTO_POSITION = 1 cannot be used together.
 include/start_slave_io.inc

share/messages_to_clients.txt

@@ -9441,7 +9441,7 @@ ER_DA_ERROR_LOG_MULTIPLE_FILTERS
 ER_DA_CANT_OPEN_ERROR_LOG
   eng "Could not open file '%s' for error logging%s%s"
 
-ER_USER_REFERENCED_AS_DEFINER  
+ER_USER_REFERENCED_AS_DEFINER
   eng "User %.256s is referenced as a definer account in %s."
 
 ER_CANNOT_USER_REFERENCED_AS_DEFINER
@@ -9625,6 +9625,21 @@ ER_MFA_METHODS_IDENTICAL
 ER_INVALID_MFA_OPERATIONS_FOR_PASSWORDLESS_USER
   eng "The operation \"%s\" cannot be performed for user '%s'@'%s' configured to connect without a password."
 
+ER_CHANGE_REPLICATION_SOURCE_NO_OPTIONS_FOR_GTID_ONLY
+  eng "GTID_ONLY cannot be enabled for replication channel '%.192s'. You need GTID_MODE = ON, SOURCE_AUTO_POSITION = 1 and REQUIRE_ROW_FORMAT = 1."
+
+ER_CHANGE_REP_SOURCE_CANT_DISABLE_REQ_ROW_FORMAT_WITH_GTID_ONLY
+  eng "REQUIRE_ROW_FORMAT cannot be disabled for replication channel '%.192s' when GTID_ONLY=1."
+
+ER_CHANGE_REP_SOURCE_CANT_DISABLE_AUTO_POS_FORMAT_WITH_GTID_ONLY
+  eng "SOURCE_AUTO_POSITION cannot be disabled for replication channel '%.192s' when GTID_ONLY=1."
+
+ER_CHANGE_REP_SOURCE_CANT_DISABLE_GTID_ONLY_WITHOUT_POSITIONS
+  eng "When disabling GTID_ONLY and SOURCE_AUTO_POSITION FOR CHANNEL '%.192s' you must provide SOURCE_LOG_FILE and SOURCE_LOG_POS."
+
+ER_CHANGE_REP_SOURCE_CANT_DISABLE_AUTO_POS_WITHOUT_POSITIONS
+  eng "When disabling SOURCE_AUTO_POSITION FOR CHANNEL '%.192s' you must provide SOURCE_LOG_FILE and SOURCE_LOG_POS as source positions are invalid"
+
 #
 #  End of 8.0 error messages (server-to-client).
 #  Do NOT add messages intended for the error log above!

sql/CMakeLists.txt

@@ -975,6 +975,7 @@ TARGET_LINK_LIBRARIES(master binlog sql_main)
 SET (SLAVE_SOURCE
   dynamic_ids.cc
   changestreams/apply/commit_order_queue.cc
+  changestreams/apply/replication_thread_status.cc
   rpl_applier_reader.cc
   rpl_async_conn_failover.cc
   rpl_async_conn_failover_table_operations.cc

sql/changestreams/apply/constants.h

@@ -0,0 +1,33 @@
+/*
+   Copyright (c) 2021, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef CHANGE_STREAM_APPLY_CONSTANTS_H_
+#define CHANGE_STREAM_APPLY_CONSTANTS_H_
+
+/* masks for start/stop operations on io and sql slave threads */
+static constexpr unsigned long SLAVE_IO = 1;
+static constexpr unsigned long SLAVE_SQL = 2;
+// We also have SLAVE_FORCE_ALL 4
+static constexpr unsigned long SLAVE_MONITOR = 8;
+
+#endif /* CHANGE_STREAM_APPLY_CONSTANTS_H_ */

sql/changestreams/apply/replication_thread_status.cc

@@ -0,0 +1,70 @@
+/*
+   Copyright (c) 2021, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include "sql/changestreams/apply/replication_thread_status.h"
+#include "sql/changestreams/apply/constants.h"
+#include "sql/rpl_io_monitor.h"
+
+void lock_slave_threads(Master_info *mi) {
+  DBUG_TRACE;
+
+  // protection against mixed locking order (see rpl_slave.cc header)
+  mi->channel_assert_some_wrlock();
+
+  // TODO: see if we can do this without dual mutex
+  mysql_mutex_lock(&mi->run_lock);
+  mysql_mutex_lock(&mi->rli->run_lock);
+}
+
+void unlock_slave_threads(Master_info *mi) {
+  DBUG_TRACE;
+
+  // TODO: see if we can do this without dual mutex
+  mysql_mutex_unlock(&mi->rli->run_lock);
+  mysql_mutex_unlock(&mi->run_lock);
+}
+
+void init_thread_mask(int *mask, Master_info *mi, bool inverse,
+                      bool ignore_monitor_thread) {
+  bool set_io = mi->slave_running, set_sql = mi->rli->slave_running;
+  bool set_monitor{
+      Source_IO_monitor::get_instance()->is_monitoring_process_running()};
+  int tmp_mask{0};
+  DBUG_TRACE;
+
+  if (set_io) tmp_mask |= SLAVE_IO;
+  if (set_sql) tmp_mask |= SLAVE_SQL;
+  if (!ignore_monitor_thread && set_monitor &&
+      mi->is_source_connection_auto_failover()) {
+    tmp_mask |= SLAVE_MONITOR;
+  }
+
+  if (inverse) {
+    tmp_mask ^= (SLAVE_IO | SLAVE_SQL);
+    if (!ignore_monitor_thread && mi->is_source_connection_auto_failover()) {
+      tmp_mask ^= SLAVE_MONITOR;
+    }
+  }
+
+  *mask = tmp_mask;
+}

sql/changestreams/apply/replication_thread_status.h

@@ -0,0 +1,67 @@
+/*
+   Copyright (c) 2021, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef CHANGE_STREAM_REPLICATION_THREAD_STATUS_H_
+#define CHANGE_STREAM_REPLICATION_THREAD_STATUS_H_
+
+#include "sql/rpl_mi.h"
+
+/**
+  This method locks both (in this order)
+    mi->run_lock
+    rli->run_lock
+
+  @param mi The associated master info object
+
+  @note this method shall be invoked while locking mi->m_channel_lock
+  for writes. This is due to the mixed order in which these locks are released
+  and acquired in such method as the slave threads start and stop methods.
+*/
+void lock_slave_threads(Master_info *mi);
+
+/**
+  Unlock replica master and relay log info locks.
+
+  @param mi the repo info object that contains the lock
+*/
+void unlock_slave_threads(Master_info *mi);
+
+/**
+  Find out which replications threads are running
+
+  @param mask                   Return value here
+  @param mi                     master_info for slave
+  @param inverse                If set, returns which threads are not running
+  @param ignore_monitor_thread  If set, ignores monitor io thread
+
+  IMPLEMENTATION
+    Get a bit mask for which threads are running.
+
+  @return
+            If inverse == 0, running threads
+            If inverse == 1, stopped threads
+*/
+void init_thread_mask(int *mask, Master_info *mi, bool inverse,
+                      bool ignore_monitor_thread = false);
+
+#endif /* CHANGE_STREAM_REPLICATION_THREAD_STATUS_H_ */

sql/lex.h

@@ -297,6 +297,7 @@ static const SYMBOL symbols[] = {
     {SYM("GROUP", GROUP_SYM)},
     {SYM("GROUPING", GROUPING_SYM)},
     {SYM("GROUPS", GROUPS_SYM)},
+    {SYM("GTID_ONLY", GTID_ONLY_SYM)},
     {SYM("HANDLER", HANDLER_SYM)},
     {SYM("HASH", HASH_SYM)},
     {SYM("HAVING", HAVING)},

sql/rpl_async_conn_failover.cc

@@ -22,6 +22,7 @@
 
 #include "mysql/components/services/log_builtins.h"
 
+#include "sql/changestreams/apply/replication_thread_status.h"
 #include "sql/current_thd.h"
 #include "sql/debug_sync.h"  // DEBUG_SYNC
 #include "sql/rpl_async_conn_failover.h"

sql/rpl_channel_service_interface.cc

@@ -47,6 +47,7 @@
 #include "mysqld_error.h"
 #include "sql/auth/sql_security_ctx.h"
 #include "sql/binlog.h"
+#include "sql/changestreams/apply/replication_thread_status.h"
 #include "sql/current_thd.h"
 #include "sql/log.h"
 #include "sql/log_event.h"

sql/rpl_filter.cc

@@ -45,6 +45,7 @@
 #include "mysqld_error.h"
 #include "sql/auth/auth_acls.h"
 #include "sql/auth/sql_security_ctx.h"
+#include "sql/changestreams/apply/replication_thread_status.h"
 #include "sql/current_thd.h"
 #include "sql/item.h"    // Item
 #include "sql/mysqld.h"  // table_alias_charset

sql/rpl_io_monitor.cc

@@ -25,6 +25,7 @@
 #include <mysql/components/services/group_replication_status_service.h>
 #include "mysql/components/services/log_builtins.h"
 
+#include "sql/changestreams/apply/replication_thread_status.h"
 #include "sql/json_dom.h"
 #include "sql/mysqld.h"
 #include "sql/mysqld_thd_manager.h"  // Global_THD_manager