BUG#34357345: Do not use SQL Service API to set/get option super_read…

…_only value

Group Replication needs to set/get some system options that are
inaccessible to plugins.
The 3 cases are:
 1) read_only
 2) super_read_only
 3) offline_mode
We also persist only the options:
 4) group_replication_single_primary_mode
 5) group_replication_enforce_update_everywhere_checks
To access those, Group Replication uses the SQL Service API which
requires to open a internal connection to the server and goes
through the complete SQL pipeline.
On 8.0 more efficient services were added to perform these
operations.

This patch does replace the use of the SQL Service API with the more
efficient services.

Change-Id: I405d2ef2bd5bcdab224e5a892e6bcf12f9ec7197

mysql-test/suite/group_replication/r/gr_check_stage_values_correctness.result

@@ -18,7 +18,8 @@ include/assert.inc [Group replication recovery thread is not present in threads
 
 include/stop_group_replication.inc
 include/assert.inc [No group replication entries are present in events_stages_current table.]
-include/assert.inc [No group replication entries are present in threads table.]
+include/assert.inc [2 group replication entries are present in threads table.]
+include/assert.inc [THD_mysql_thread_handler* are present in threads table.]
 
 # 03. Create some data on server1.
 #     Recovery channel will have some data on server2.

mysql-test/suite/group_replication/r/gr_mysql_thread_queued_task_psi_memory.result

@@ -44,8 +44,11 @@ include/assert.inc ['There should be 1 entry corresponding to the mysql_thread_q
 
 #################################################################
 # 4. Assert that the number of bytes allocated for
-#    mysql_thread_queued_task event must be 0.
-include/assert.inc ['The sum of bytes allocated must be zero after stopping GR']
+#    mysql_thread_queued_task event must be 0 when the plugin
+#    is uninstalled.
+include/uninstall_group_replication_plugin.inc
+include/assert.inc ['The sum of bytes allocated must be zero after uninstall GR']
+include/install_group_replication_plugin.inc
 
 #################################################################
 # 5. Clean up

mysql-test/suite/group_replication/r/gr_psi_keys_mysql.result

@@ -68,6 +68,8 @@ wait/synch/mutex/group_rpl/LOCK_mysql_thread_dispatcher_run	YES	YES	singleton	NU
 wait/synch/mutex/group_rpl/LOCK_connection_map	YES	YES	singleton	NULL	0	NULL
 wait/synch/mutex/group_rpl/LOCK_mysql_handler_thread_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/mutex/group_rpl/LOCK_mysql_thread_handler_dispatcher_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/mutex/group_rpl/LOCK_mysql_handler_thread_read_only_mode_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/mutex/group_rpl/LOCK_mysql_thread_handler_read_only_mode_dispatcher_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/rwlock/group_rpl/RWLOCK_certifier_stable_gtid_set	YES	YES	singleton	NULL	0	NULL
 wait/synch/rwlock/group_rpl/RWLOCK_channel_observation_list	YES	YES	singleton	NULL	0	NULL
 wait/synch/rwlock/group_rpl/RWLOCK_gcs_operations	YES	YES	singleton	NULL	0	NULL
@@ -119,6 +121,8 @@ wait/synch/cond/group_rpl/COND_mysql_thread_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/cond/group_rpl/COND_mysql_thread_dispatcher_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/cond/group_rpl/COND_mysql_thread_handler_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/cond/group_rpl/COND_mysql_thread_handler_dispatcher_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/cond/group_rpl/COND_mysql_thread_handler_read_only_mode_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/cond/group_rpl/COND_mysql_thread_handler_read_only_mode_dispatcher_run	YES	YES	singleton	NULL	0	NULL
 stage/group_rpl/Undergoing auto-rejoin procedure	YES	YES	progress	NULL	0	NULL
 stage/group_rpl/Multi-primary Switch: waiting for pending transactions to finish	YES	YES	progress	NULL	0	NULL
 stage/group_rpl/Multi-primary Switch: waiting on another member step completion	YES	YES	progress	NULL	0	NULL
@@ -181,6 +185,7 @@ thread/group_rpl/THD_Gcs_xcom_engine::m_engine_thread	BACKGROUND	YES
 thread/group_rpl/THD_message_service_handler	BACKGROUND	YES
 thread/group_rpl/THD_mysql_thread	BACKGROUND	YES
 thread/group_rpl/THD_mysql_thread_handler	BACKGROUND	YES
+thread/group_rpl/THD_mysql_thread_handler_read_only_mode	BACKGROUND	YES
 
 ############################################################
 # 5. Test end.

mysql-test/suite/group_replication/r/gr_psi_keys_xcom.result

@@ -68,6 +68,8 @@ wait/synch/mutex/group_rpl/LOCK_mysql_thread_dispatcher_run	YES	YES	singleton	NU
 wait/synch/mutex/group_rpl/LOCK_connection_map	YES	YES	singleton	NULL	0	NULL
 wait/synch/mutex/group_rpl/LOCK_mysql_handler_thread_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/mutex/group_rpl/LOCK_mysql_thread_handler_dispatcher_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/mutex/group_rpl/LOCK_mysql_handler_thread_read_only_mode_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/mutex/group_rpl/LOCK_mysql_thread_handler_read_only_mode_dispatcher_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/rwlock/group_rpl/RWLOCK_certifier_stable_gtid_set	YES	YES	singleton	NULL	0	NULL
 wait/synch/rwlock/group_rpl/RWLOCK_channel_observation_list	YES	YES	singleton	NULL	0	NULL
 wait/synch/rwlock/group_rpl/RWLOCK_gcs_operations	YES	YES	singleton	NULL	0	NULL
@@ -119,6 +121,8 @@ wait/synch/cond/group_rpl/COND_mysql_thread_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/cond/group_rpl/COND_mysql_thread_dispatcher_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/cond/group_rpl/COND_mysql_thread_handler_run	YES	YES	singleton	NULL	0	NULL
 wait/synch/cond/group_rpl/COND_mysql_thread_handler_dispatcher_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/cond/group_rpl/COND_mysql_thread_handler_read_only_mode_run	YES	YES	singleton	NULL	0	NULL
+wait/synch/cond/group_rpl/COND_mysql_thread_handler_read_only_mode_dispatcher_run	YES	YES	singleton	NULL	0	NULL
 stage/group_rpl/Undergoing auto-rejoin procedure	YES	YES	progress	NULL	0	NULL
 stage/group_rpl/Multi-primary Switch: waiting for pending transactions to finish	YES	YES	progress	NULL	0	NULL
 stage/group_rpl/Multi-primary Switch: waiting on another member step completion	YES	YES	progress	NULL	0	NULL
@@ -181,6 +185,7 @@ thread/group_rpl/THD_Gcs_xcom_engine::m_engine_thread	BACKGROUND	YES
 thread/group_rpl/THD_message_service_handler	BACKGROUND	YES
 thread/group_rpl/THD_mysql_thread	BACKGROUND	YES
 thread/group_rpl/THD_mysql_thread_handler	BACKGROUND	YES
+thread/group_rpl/THD_mysql_thread_handler_read_only_mode	BACKGROUND	YES
 
 ############################################################
 # 5. Test end.

mysql-test/suite/group_replication/t/gr_addmember_error_message.test

@@ -43,7 +43,7 @@
 --let $assert_file = $MYSQLTEST_VARDIR/tmp/gr_add_member_error_message.2.err
 --let $assert_only_after=  Plugin 'group_replication' is starting
 --let $assert_select = \[System\] \[MY-011565\] \[Repl\] Plugin group_replication reported: 'Setting super_read_only=ON\.'
---let $assert_count = 1
+--let $assert_count = 2
 --let $assert_text = Found message informing that `super_read_only` is ON.
 --source include/assert_grep.inc
 

mysql-test/suite/group_replication/t/gr_check_stage_values_correctness.test

@@ -57,8 +57,11 @@ CREATE TABLE test.t1 (a INT PRIMARY KEY);
 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.events_stages_current WHERE event_name LIKE "%/group_rpl/%"] = 0
 --source include/assert.inc
 
---let $assert_text= No group replication entries are present in threads table.
---let $assert_cond= [SELECT COUNT(*) FROM performance_schema.threads WHERE name LIKE "%/group_rpl/%"] = 0
+--let $assert_text= 2 group replication entries are present in threads table.
+--let $assert_cond= [SELECT COUNT(*) FROM performance_schema.threads WHERE name LIKE "%/group_rpl/%"] = 2
+--source include/assert.inc
+--let $assert_text= THD_mysql_thread_handler* are present in threads table.
+--let $assert_cond= [SELECT COUNT(*) FROM performance_schema.threads WHERE name LIKE "%/group_rpl/THD_mysql_thread%"] = 2
 --source include/assert.inc
 
 --echo

mysql-test/suite/group_replication/t/gr_consistent_reads_before_switch_group_mode.test

@@ -93,7 +93,7 @@ SET @@SESSION.group_replication_consistency= 'BEFORE';
 --let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist WHERE user='root' AND state='executing' AND info LIKE 'SELECT group_replication_switch_to_single_primary_mode(%'
 --source include/wait_condition.inc
 
---let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist WHERE user='mysql.session' AND state='Waiting for global read lock' AND info='PLUGIN: SET GLOBAL super_read_only= 1'
+--let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.threads WHERE NAME = 'thread/group_rpl/THD_mysql_thread_handler_read_only_mode' AND PROCESSLIST_STATE = 'Waiting for global read lock'
 --source include/wait_condition.inc
 
 

mysql-test/suite/group_replication/t/gr_lock_tables_unblock.test

@@ -47,7 +47,7 @@ LOCK TABLE t1 WRITE;
 --echo # super_read_only mode.
 --let $rpl_connection_name= server_1_1
 --source include/rpl_connection.inc
---let $wait_condition=SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'Waiting for global read lock' and Info = 'PLUGIN: SET GLOBAL super_read_only= 1'
+--let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.threads WHERE NAME = 'thread/group_rpl/THD_mysql_thread_handler_read_only_mode' AND PROCESSLIST_STATE = 'Waiting for global read lock'
 --source include/wait_condition.inc
 
 --connection server_1

mysql-test/suite/group_replication/t/gr_mysql_thread_queued_task_psi_memory.test

@@ -16,12 +16,14 @@
 # 3. Verify that the entries corresponding to the instrumented
 #    mysql_thread_queued_task remain in the tables after stopping GR.
 # 4. Assert that the number of bytes allocated for
-#    mysql_thread_queued_task event must be 0.
+#    mysql_thread_queued_task event must be 0 when the plugin
+#    is uninstalled.
 # 5. Clean up
 ###############################################################################
 
 --source include/have_debug_sync.inc
 --source include/have_group_replication_plugin.inc
+--source include/force_restart.inc
 --let $rpl_skip_group_replication_start= 1
 --let $rpl_group_replication_single_primary_mode=1
 --source include/group_replication.inc
@@ -99,10 +101,13 @@ SELECT EVENT_NAME FROM performance_schema.memory_summary_global_by_event_name WH
 --echo
 --echo #################################################################
 --echo # 4. Assert that the number of bytes allocated for
---echo #    mysql_thread_queued_task event must be 0.
---let $assert_text= 'The sum of bytes allocated must be zero after stopping GR'
+--echo #    mysql_thread_queued_task event must be 0 when the plugin
+--echo #    is uninstalled.
+--source include/uninstall_group_replication_plugin.inc
+--let $assert_text= 'The sum of bytes allocated must be zero after uninstall GR'
 --let $assert_cond= CURRENT_NUMBER_OF_BYTES_USED = 0 FROM performance_schema.memory_summary_global_by_event_name WHERE EVENT_NAME LIKE "memory/group_rpl/mysql_thread_queued_task"
 --source include/assert.inc
+--source include/install_group_replication_plugin.inc
 
 --echo
 --echo #################################################################

mysql-test/suite/group_replication/t/gr_recovery_clone_activation_threshold.test

@@ -60,7 +60,9 @@ SET SESSION sql_log_bin = 0;
 call mtr.add_suppression("Plugin group_replication reported.*");
 SET SESSION sql_log_bin = 1;
 
---let $group_replication_start_member_state= RECOVERING
+# Clone plugin is not installed, as such the member will error
+# during recovery.
+--let $group_replication_start_member_state= ERROR
 --source include/start_group_replication.inc
 
 # Assert that recovery using clone is logged to the server error log.

mysql-test/suite/group_replication/t/gr_stop_procedure_hang.test

@@ -71,7 +71,7 @@ SET @@GLOBAL.DEBUG='d,group_replication_before_commit_hook_wait';
 
 --let $rpl_connection_name= server_1_1
 --source include/rpl_connection.inc
---let $wait_condition=SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'Waiting for global read lock'
+--let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.threads WHERE NAME = 'thread/group_rpl/THD_mysql_thread_handler_read_only_mode' AND PROCESSLIST_STATE = 'Waiting for global read lock'
 --source include/wait_condition.inc
 
 SET DEBUG_SYNC='now SIGNAL continue_commit';

mysql-test/suite/x/r/notices_gr_join_leave.result

@@ -115,6 +115,8 @@ RUN STOP GROUP_REPLICATION
 RUN SET GLOBAL read_only= FALSE
 RUN DROP TABLE test.tab1
 RUN RESET MASTER
+[ASSERT view_change on 'verify_1' connection]
+[ASSERT state_change on 'verify_1' connection]
 [connection server_2]
 RUN DROP TABLE test.tab1
 RUN RESET MASTER
@@ -253,6 +255,8 @@ RUN STOP GROUP_REPLICATION
 RUN SET GLOBAL read_only= FALSE
 RUN DROP TABLE test.tab1
 RUN RESET MASTER
+[ASSERT no notices on 'verify_1' connection]
+[ASSERT no notices on 'verify_1' connection]
 [connection server_2]
 RUN DROP TABLE test.tab1
 RUN RESET MASTER

mysql-test/suite/x/t/notices_gr_join_leave.test

@@ -173,6 +173,8 @@ noquery_result;
 DROP TABLE test.tab1;
 RESET MASTER;
 query_result;
+callmacro %ASSERT_NOTICES%	view_changed	verify_1;
+callmacro %ASSERT_NOTICES%	state_changed	verify_1;
 
 callmacro Setsession	server_2;
 noquery_result;

plugin/group_replication/CMakeLists.txt

@@ -67,7 +67,6 @@ SET(GROUP_REPLICATION_SOURCES
   src/plugin_handlers/member_actions_handler.cc
   src/plugin_handlers/member_actions_handler_configuration.cc
   src/plugin_handlers/offline_mode_handler.cc
-  src/plugin_handlers/persistent_variables_handler.cc
   src/plugin_handlers/primary_election_invocation_handler.cc
   src/plugin_handlers/primary_election_primary_process.cc
   src/plugin_handlers/primary_election_secondary_process.cc
@@ -97,11 +96,13 @@ SET(GROUP_REPLICATION_SOURCES
   src/recovery_channel_state_observer.cc
   src/recovery_state_transfer.cc
   src/replication_threads_api.cc
-  src/services/get_system_variable/get_system_variable.cc
   src/services/message_service/message_service.cc
   src/services/notification/notification.cc
   src/services/status_service/status_service.cc
+  src/services/system_variable/get_system_variable.cc
+  src/services/system_variable/set_system_variable.cc
   src/services/registry.cc
+  src/services/server_services_references.cc
   src/sql_service/sql_command_test.cc
   src/sql_service/sql_resultset.cc
   src/sql_service/sql_service_command.cc

plugin/group_replication/include/delayed_plugin_initialization.h

@@ -23,7 +23,7 @@
 #ifndef DELAYED_PLUGIN_INITIALIZATION_INCLUDE
 #define DELAYED_PLUGIN_INITIALIZATION_INCLUDE
 
-#include "plugin/group_replication/include/plugin_handlers/read_mode_handler.h"
+#include "plugin/group_replication/include/plugin_utils.h"
 
 class Delayed_initialization_thread {
  public:

plugin/group_replication/include/leave_group_on_failure.h

@@ -25,8 +25,8 @@
 
 #include <bitset>
 
+#include "my_inttypes.h"
 #include "plugin/group_replication/include/services/notification/notification.h"
-#include "plugin/group_replication/include/sql_service/sql_service_command.h"
 
 /**
   Structure that holds the actions taken by the plugin when the
@@ -57,10 +57,6 @@ struct leave_group_on_failure {
                   Actions performed.
     @param[in]  error_to_log
                   Error logged into error log.
-    @param[in]  session_isolation
-                  Session creation requirements: use current thread,
-                  use thread but initialize it or create it in a
-                  dedicated thread.
     @param[in]  caller_notification_context
                   If defined the member state change notification
                   will update this notification context and the
@@ -73,7 +69,6 @@ struct leave_group_on_failure {
                   that function is called.
   */
   static void leave(const mask &actions, longlong error_to_log,
-                    enum_plugin_con_isolation session_isolation,
                     Notification_context *caller_notification_context,
                     const char *exit_state_action_abort_log_message);
 };

plugin/group_replication/include/plugin.h

@@ -49,6 +49,7 @@
 #include "plugin/group_replication/include/recovery.h"
 #include "plugin/group_replication/include/services/message_service/message_service.h"
 #include "plugin/group_replication/include/services/registry.h"
+#include "plugin/group_replication/include/services/server_services_references.h"
 #include "plugin/group_replication/libmysqlgcs/include/mysql/gcs/gcs_interface.h"
 
 // Forward declarations
@@ -116,7 +117,6 @@ struct gr_modules {
     MESSAGE_SERVICE_HANDLER,
     BINLOG_DUMP_THREAD_KILL,
     MEMBER_ACTIONS_HANDLER,
-    MYSQL_THREAD_HANDLER,
     NUM_MODULES
   };
   using mask = std::bitset<NUM_MODULES>;
@@ -173,6 +173,8 @@ extern Autorejoin_thread *autorejoin_module;
 extern Message_service_handler *message_service_handler;
 extern Member_actions_handler *member_actions_handler;
 extern Mysql_thread *mysql_thread_handler;
+extern Mysql_thread *mysql_thread_handler_read_only_mode;
+extern Server_services_references *server_services_references_module;
 
 // Auxiliary Functionality
 extern Plugin_gcs_events_handler *events_handler;
@@ -230,6 +232,7 @@ bool get_plugin_is_stopping();
 bool get_wait_on_engine_initialization();
 void enable_server_shutdown_status();
 bool get_server_shutdown_status();
+void mysql_thread_handler_finalize();
 void set_plugin_is_setting_read_mode(bool value);
 bool get_plugin_is_setting_read_mode();
 const char *get_group_name_var();

plugin/group_replication/include/plugin_handlers/offline_mode_handler.h

@@ -23,16 +23,10 @@
 #ifndef OFFLINE_MODE_HANDLER_INCLUDE
 #define OFFLINE_MODE_HANDLER_INCLUDE
 
-#include "plugin/group_replication/include/sql_service/sql_service_command.h"
-
 /**
   This method creates a server session and connects to the server
   to enable the offline mode
-
-  @param session_isolation session creation requirements: use current thread,
-                           use thread but initialize it or create it in a
-                           dedicated thread
 */
-void enable_server_offline_mode(enum_plugin_con_isolation session_isolation);
+void enable_server_offline_mode();
 
 #endif /* OFFLINE_MODE_HANDLER_INCLUDE */

plugin/group_replication/include/plugin_handlers/primary_election_secondary_process.h

@@ -85,12 +85,6 @@ class Primary_election_secondary_process : public Group_event_observer {
   void set_stop_wait_timeout(ulong timeout);
 
  private:
-  enum enum_election_read_mode_status {
-    SECONDARY_ELECTION_READ_MODE_NOT_SET = 0,    // Read only mode not set
-    SECONDARY_ELECTION_READ_MODE_BEING_SET = 1,  // Read only mode being set
-    SECONDARY_ELECTION_READ_MODE_IS_SET = 2,     // Read only mode is set
-  };
-
   // The listeners for group events
 
   int after_view_change(const std::vector<Gcs_member_identifier> &joining,
@@ -113,12 +107,6 @@ class Primary_election_secondary_process : public Group_event_observer {
   */
   bool enable_read_mode_on_server();
 
-  /**
-    Kills the current read mode query that might be stuck
-    @return false in case of success, or true otherwise
-  */
-  bool kill_read_mode_query();
-
   /**
      Signal that the read mode is ready on this member
      @returns false in case of success, or true otherwise
@@ -138,10 +126,6 @@ class Primary_election_secondary_process : public Group_event_observer {
   bool group_in_read_mode;
   /** Process is waiting on read mode - stage related var*/
   bool is_waiting_on_read_mode_group;
-  /** The session id that is set the read mode*/
-  unsigned long read_mode_session_id;
-  /** What is the status on the read only mode enabling query */
-  enum_election_read_mode_status is_read_mode_set;
 
   /** The election invocation context */
   enum_primary_election_mode election_mode;