BUG#25673788: MYSQLADMIN SHUTDOWN HANGS WHEN GR PLUGIN IS BUSY

When we try to shutdown server through mysqladmin when GR plugin is in
deleted state, shutdown hangs.

When the plugin is in use and the server tries to uninstall it, it will mark
the plugin as deleted and delay the uninstall to when execute shutdown.
When executing the shutdown the server don't execute anything on plugins
deleted.

To solve this we don't allow remove the plugin when this is in use, it need to
be executed first the command STOP GROUP_REPLICATION.

rapid/plugin/group_replication/tests/mtr/r/gr_uninstall_while_network_blocked.result

@@ -0,0 +1,47 @@
+include/group_replication.inc [rpl_server_count=2]
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection server1]
+[connection server1]
+CREATE TABLE t1 (a INT PRIMARY KEY);
+include/rpl_sync.inc
+[connection server2]
+
+# 1. Kill and restart server 2
+# Kill and restart:--group_replication_local_address=GROUP_REPLICATION_LOCAL_ADDRESS2 --group_replication_group_seeds=GROUP_REPLICATION_GROUP_SEEDS --group_replication_group_name=GROUP_REPLICATION_GROUP_NAME
+include/rpl_reconnect.inc
+[connection server1]
+
+# 2. Wait until server2 is marked as UNREACHABLE
+include/gr_wait_for_member_state.inc
+
+# 3. Execute query that will be blocked and leave the group plugin busy
+INSERT INTO test.t1 VALUES(11);
+[connection server_1]
+set session sql_log_bin=0;
+call mtr.add_suppression("Group Replication plugin is busy, it cannot be uninstalled. To force");
+call mtr.add_suppression("Timeout while waiting for the group communication engine to exit!");
+call mtr.add_suppression("The member has failed to gracefully leave the group.");
+call mtr.add_suppression("read failed");
+call mtr.add_suppression("Due to a plugin error, some transactions can't be certified and will now rollback.");
+call mtr.add_suppression("The member lost contact with a majority of the members in the group. Until the network is restored.*");
+set session sql_log_bin=1;
+
+# 4. Uninstall plugin will return error due plugin being in use
+UNINSTALL PLUGIN group_replication;
+ERROR HY000: Plugin 'group_replication' cannot be uninstalled now. Plugin is busy, it cannot be uninstalled. To force a stop run STOP GROUP_REPLICATION and then UNINSTALL PLUGIN group_replication.
+
+# 5. Call mysqladmin to shutdown server
+
+# 6. Clean-up
+[connection server1]
+include/rpl_start_server.inc [server_number=1]
+include/rpl_reconnect.inc
+# CR_SERVER_LOST, CR_SERVER_GONE_ERROR
+Got one of the listed errors
+[connection server2]
+DROP TABLE test.t1;
+[connection server_1]
+DROP TABLE test.t1;
+include/group_replication_end.inc

rapid/plugin/group_replication/tests/mtr/t/gr_uninstall_while_network_blocked.test

@@ -0,0 +1,115 @@
+################################################################################
+# When we try to shutdown server through mysqladmin and GR plugin is
+# busy (in deleted state), it returns and error that STOP GROUP_REPLICATION
+# need to be executed before that.
+# Executing mysqladmin shutdown will shutdown the server.
+#
+# Test:
+# 0. The test requires two servers
+# 1. Kill and restart server 2
+# 2. Wait until server2 is marked as UNREACHABLE
+# 3. Execute query that will be blocked and leave the group plugin busy
+# 4. Uninstall plugin will return error due plugin being in use
+# 5. Call mysqladmin to shutdown server
+# 6. Clean up
+
+# This test does crashes servers, hence we skip it on asan and valgrind.
+--source include/not_asan.inc
+--source include/not_valgrind.inc
+--source include/big_test.inc
+--source ../inc/have_group_replication_plugin.inc
+
+--let $rpl_server_count= 2
+--source ../inc/group_replication.inc
+
+--let $rpl_connection_name= server1
+--source include/rpl_connection.inc
+
+
+# Create table
+CREATE TABLE t1 (a INT PRIMARY KEY);
+--source include/rpl_sync.inc
+
+--let $rpl_connection_name= server2
+--source include/rpl_connection.inc
+
+# Save the variables, for server restart.
+--let $member2_group_replication_local_address= `SELECT @@GLOBAL.group_replication_local_address`
+--let $group_replication_group_seeds= `SELECT @@GLOBAL.group_replication_group_seeds`
+--let $member2_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
+
+--echo
+--echo # 1. Kill and restart server 2
+--let $restart_parameters=restart:--group_replication_local_address=$member2_group_replication_local_address --group_replication_group_seeds=$group_replication_group_seeds --group_replication_group_name=$group_replication_group_name
+--replace_result $member2_group_replication_local_address GROUP_REPLICATION_LOCAL_ADDRESS2 $group_replication_group_seeds GROUP_REPLICATION_GROUP_SEEDS $group_replication_group_name GROUP_REPLICATION_GROUP_NAME
+--source ../inc/kill_and_restart_mysqld.inc
+
+--let $rpl_server_number= 2
+--source include/rpl_reconnect.inc
+
+--let $rpl_connection_name= server1
+--source include/rpl_connection.inc
+
+--echo
+--echo # 2. Wait until server2 is marked as UNREACHABLE
+--let $group_replication_member_state= UNREACHABLE
+--let $group_replication_member_id= $member2_uuid
+--source ../inc/gr_wait_for_member_state.inc
+
+--echo
+--echo # 3. Execute query that will be blocked and leave the group plugin busy
+--send INSERT INTO test.t1 VALUES(11)
+
+--let $rpl_connection_name= server_1
+--source include/rpl_connection.inc
+
+set session sql_log_bin=0;
+call mtr.add_suppression("Group Replication plugin is busy, it cannot be uninstalled. To force");
+call mtr.add_suppression("Timeout while waiting for the group communication engine to exit!");
+call mtr.add_suppression("The member has failed to gracefully leave the group.");
+call mtr.add_suppression("read failed");
+call mtr.add_suppression("Due to a plugin error, some transactions can't be certified and will now rollback.");
+call mtr.add_suppression("The member lost contact with a majority of the members in the group. Until the network is restored.*");
+set session sql_log_bin=1;
+
+--echo
+--echo # 4. Uninstall plugin will return error due plugin being in use
+--error ER_PLUGIN_CANNOT_BE_UNINSTALLED
+UNINSTALL PLUGIN group_replication;
+
+--echo
+--echo # 5. Call mysqladmin to shutdown server
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--disable_result_log
+--replace_regex /.*mysqladmin.*: /mysqladmin: /
+--exec $MYSQLADMIN -uroot -S $MASTER_MYSOCK -P $MASTER_MYPORT shutdown 2>&1
+--enable_result_log
+
+--source include/wait_until_disconnected.inc
+
+--echo
+--echo # 6. Clean-up
+--let $rpl_connection_name= server1
+--source include/rpl_connection.inc
+
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--let $rpl_server_number= 1
+--source include/rpl_start_server.inc
+
+--let $rpl_server_number= 1
+--source include/rpl_reconnect.inc
+
+# terminate send calling reap
+--echo # CR_SERVER_LOST, CR_SERVER_GONE_ERROR
+--error 2006,2013
+--reap
+
+--let $rpl_connection_name= server2
+--source include/rpl_connection.inc
+DROP TABLE test.t1;
+
+--let $rpl_connection_name= server_1
+--source include/rpl_connection.inc
+DROP TABLE test.t1;
+
+--source ../inc/group_replication_end.inc

sql/sql_plugin.cc

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2215,6 +2215,19 @@ static bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name)
              "Stop any active semisynchronous I/O threads on this slave first.");
     goto err;
   }
+
+  /* If Group Replication is in use, the plugin can't be uninstalled.
+   * The command STOP GROUP_REPLICATION should be used before uninstall.
+  */
+  if (plugin->ref_count && !strcmp(name->str, "group_replication"))
+  {
+    mysql_mutex_unlock(&LOCK_plugin);
+    my_error(ER_PLUGIN_CANNOT_BE_UNINSTALLED, MYF(0), name->str,
+             "Plugin is busy, it cannot be uninstalled. To force a"
+             " stop run STOP GROUP_REPLICATION and then UNINSTALL"
+             " PLUGIN group_replication.");
+    goto err;
+  }
 #endif
 
   plugin->state= PLUGIN_IS_DELETED;