WL#9852: XCom transport abstraction

XCom is a communication framework with a PAXOS implementation that
supports MySQL Group Replication.
It can be conceptually divided into two sections:
The PAXOS algorithm and a Communication infrastructure.

Although keeping the Communication infrastructure under our control
had given us an important advantage in terms of performance and fine
tuning, it doubles our work in areas where the server is already mature
(Secure Communications, Fragmentation).

We reused existing Server facilities in order to establish connections
among GCS group members. That allowed us to create authenticated and
secure connections with MySQL protocol.

RB: 25153

Author: tiagoportelajorge

include/my_command.h

@@ -93,6 +93,7 @@ enum enum_server_command {
   COM_BINLOG_DUMP_GTID,
   COM_RESET_CONNECTION, /**< See @ref page_protocol_com_reset_connection */
   COM_CLONE,
+  COM_SUBSCRIBE_GROUP_REPLICATION_STREAM,
   /* don't forget to update const char *command_name[] in sql_parse.cc */
 
   /* Must be last */

include/mysql.h.pp

@@ -86,6 +86,7 @@
   COM_BINLOG_DUMP_GTID,
   COM_RESET_CONNECTION,
   COM_CLONE,
+  COM_SUBSCRIBE_GROUP_REPLICATION_STREAM,
   COM_END
 };
 #include "my_compress.h"

include/mysql/plugin_audit.h.pp

@@ -171,6 +171,7 @@
   COM_BINLOG_DUMP_GTID,
   COM_RESET_CONNECTION,
   COM_CLONE,
+  COM_SUBSCRIBE_GROUP_REPLICATION_STREAM,
   COM_END
 };
 #include "my_sqlcommand.h"

include/mysql/plugin_group_replication.h

@@ -83,6 +83,9 @@ struct GROUP_REPLICATION_GROUP_MEMBERS_CALLBACKS {
                           size_t length);
   void (*set_member_version)(void *const context, const char &value,
                              size_t length);
+  void (*set_member_incoming_communication_protocol)(void *const context,
+                                                     const char &value,
+                                                     size_t length);
 };
 
 /*

include/mysql/thread_type.h

@@ -45,7 +45,8 @@ enum enum_thread_type {
   SYSTEM_THREAD_DD_RESTART = 1024,
   SYSTEM_THREAD_SERVER_INITIALIZE = 2048,
   SYSTEM_THREAD_INIT_FILE = 4096,
-  SYSTEM_THREAD_SERVER_UPGRADE = 8192
+  SYSTEM_THREAD_SERVER_UPGRADE = 8192,
+  SYSTEM_THREAD_GROUP_REPLICATION_CONNECTION = 16384
 };
 
 #endif /* THREAD_TYPE_INCLUDED */

include/sql_common.h

@@ -48,7 +48,10 @@ struct MEM_ROOT;
 extern "C" {
 #endif
 
-extern const char *unknown_sqlstate;
+#ifdef IMPORT_UNKNOWN_SQLSTATE
+__declspec(dllimport)
+#endif
+    extern const char *unknown_sqlstate;
 extern const char *cant_connect_sqlstate;
 extern const char *not_error_sqlstate;
 

mysql-test/collections/coverage.ignore

@@ -5,6 +5,7 @@
 
 plugin/x/tests/driver/
 unittest/gunit/xplugin
+unittest/gunit/libmysqlgcs
 extra/protobuf
 extra/icu
 extra/zstd

mysql-test/collections/default.push

@@ -7,20 +7,13 @@
 #
 # Non-Windows platforms
 #
-perl mysql-test-run.pl --timer --testcase-timeout=30 --suite-timeout=300 --debug-server --force  --comment=all-default-debug --vardir=var-all-default --skip-combinations --unit-tests-report --no-skip --exclude-platform=windows --skip-ndb
-
+perl mysql-test-run.pl --timer --testcase-timeout=30 --suite-timeout=300 --debug-server --force  --comment=all-default-debug --vardir=var-all-default --skip-combinations --unit-tests-report --no-skip --exclude-platform=windows
 #
 # Windows - run RPL separately with 8 threads
 #
-perl mysql-test-run.pl --timer --testcase-timeout=30 --suite-timeout=300 --debug-server --force  --comment=all-default-debug --vardir=var-all-default --skip-combinations --skip-rpl --unit-tests-report --no-skip --platform=windows --skip-ndb
+perl mysql-test-run.pl --timer --testcase-timeout=30 --suite-timeout=300 --debug-server --force  --comment=all-default-debug --vardir=var-all-default --skip-combinations --skip-rpl --unit-tests-report --no-skip --platform=windows
 perl mysql-test-run.pl --timer --testcase-timeout=30 --suite-timeout=300 --debug-server --force --parallel=8 --comment=rpl-suites --vardir=var-rpl-suites --skip-combinations --suite=rpl,rpl_gtid,rpl_nogtid --platform=windows
-
-#
-# Run all Ndb tests separately.
-# For builds without Ndb support no tests will be attempted.
-#
-perl mysql-test-run.pl --timer --testcase-timeout=30 --suite-timeout=300 --debug-server --force --comment=ndb --vardir=var-ndb --skip-combinations --no-skip --with-ndb-only
-
 #
 # Group Replication
 perl mysql-test-run.pl --timer --force --parallel=6 --comment=group_replication --vardir=var-group_replication --suite=group_replication
+

mysql-test/collections/default.weekly

@@ -191,13 +191,20 @@ perl mysql-test-run.pl --timer --force --parallel=6 --comment=group_replication_
 
 #GR with group_replication_ssl_mode = REQUIRED
 perl mysql-test-run.pl --timer --force --parallel=6 --comment=group_replication_ssl_mode_required --vardir=var-gr_ssl_mode_required --suite=group_replication  --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--loose-group_replication_ssl_mode=REQUIRED
+
+#GR with group_replication_ssl_mode = REQUIRED and MySQL Stack
+perl mysql-test-run.pl --timer --force --parallel=6 --comment=group_replication_ssl_mode_required_mysql --vardir=var-gr_ssl_mode_required_mysql --suite=group_replication  --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--loose-group_replication_ssl_mode=REQUIRED --mysqld=--loose-group_replication_communication_stack=MySQL
+
 #
 # b) DEBUG
 #
 
 #GR with loose-group_replication_gtid_assignment_block_size=1 on debug build
 perl mysql-test-run.pl --timer --debug-server --force --parallel=6 --comment=group_replication_gtid_assignment_block_size_1-debug --vardir=var-gr_gtid_assignment_block_size_1-debug --suite=group_replication  --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--loose-group_replication_gtid_assignment_block_size=1
 
+#GR with MySQL Stack on debug build
+perl mysql-test-run.pl --timer --debug-server --force --parallel=6 --comment=group_replication_mysql-debug --vardir=var-group_replication_mysql-debug --suite=group_replication  --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--loose-group_replication_communication_stack=MySQL
+
 #GR with MTS on debug build
 perl mysql-test-run.pl --timer --debug-server --force --parallel=6 --comment=group_replication_parallel_applier-debug --vardir=var-gr_parallel_applier-debug --suite=group_replication  --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--replica-parallel-workers=4 --mysqld=--replica-parallel-type=logical_clock --mysqld=--replica_preserve_commit_order=ON
 perl mysql-test-run.pl --timer --debug-server --force --parallel=6 --comment=group_replication_ssl_mode_required-debug --vardir=var-gr_ssl_mode_required-debug --suite=group_replication  --big-test --testcase-timeout=60 --suite-timeout=360 --mysqld=--loose-group_replication_ssl_mode=REQUIRED

mysql-test/collections/disabled.def

@@ -94,3 +94,13 @@ sys_vars.innodb_log_writer_threads_basic : Bug#32129814 SYS_VARS.INNODB_LOG_WRIT
 sysschema.v_wait_classes_global_by_avg_latency : BUG#21550054 Test fails too often.
 
 # x plugin suite tests
+
+# Group Replication tests.
+
+# These will only run in XCom because of the following issue:
+group_replication.gr_join_with_suspect_member : BUG#31704262 XCOM CREATES NODES WITH PAST GHOST VIEWS
+group_replication.gr_leave_with_suspect_member : BUG#31704262 XCOM CREATES NODES WITH PAST GHOST VIEWS
+group_replication.gr_majority_loss_restored_after_timeout : BUG#31704262 XCOM CREATES NODES WITH PAST GHOST VIEWS
+group_replication.gr_suspect_member_resumes_after_crash_join_retries : BUG#32458239 GR_SUSPECT_MEMBER_RESUMES_AFTER_CRASH_JOIN_RETRIES IS NOT STABLE
+group_replication.gr_majority_loss_5_to_2 @windows : Bug#32860569 SPORADIC GROUP REPLICATION MAJORITY TEST FAILURES IN WINDOWS
+group_replication.gr_majority_loss_5_to_2_recovery @windows : Bug#32860569 SPORADIC GROUP REPLICATION MAJORITY TEST FAILURES IN WINDOWS

mysql-test/include/gr_configuration.inc

@@ -16,6 +16,8 @@ if ($group_replication_group_name == "")
 # Save current connection.
 --let $_old_connection= $CURRENT_CONNECTION
 
+#check which stack are we using
+--let $have_xcom_stack= `SELECT @@group_replication_communication_stack LIKE 'XCOM'`
 
 # Default values
 --let $_group_replication_group_seeds=
@@ -50,7 +52,15 @@ while ($_rpl_server <= $_rpl_server_max)
 --let $_rpl_server= $_rpl_gcs_server_number
 while ($_rpl_server)
 {
-  --let $_rpl_gcs_server_port= \$SERVER_GR_PORT_$_rpl_server
+  --let $_rpl_gcs_server_port=
+  if ($have_xcom_stack)
+  {
+    --let $_rpl_gcs_server_port= \$SERVER_GR_PORT_$_rpl_server
+  }
+  if (!$have_xcom_stack)
+  {
+    --let $_rpl_gcs_server_port= \$SERVER_MYPORT_$_rpl_server
+  }
 
   # Append comma to list of addresses.
   if ($_group_replication_group_seeds != "")
@@ -86,7 +96,15 @@ while ($_rpl_server)
   }
 
   # Local address.
-  --let $_rpl_gcs_server_port= \$SERVER_GR_PORT_$_rpl_server
+  --let $_rpl_gcs_server_port=
+  if ($have_xcom_stack)
+  {
+    --let $_rpl_gcs_server_port= \$SERVER_GR_PORT_$_rpl_server
+  }
+  if (!$have_xcom_stack)
+  {
+    --let $_rpl_gcs_server_port= \$SERVER_MYPORT_$_rpl_server
+  }
   --let $_group_replication_local_address= $_rpl_gcs_server_address:$_rpl_gcs_server_port
 
   # Set configuration on server.

mysql-test/include/have_group_replication_mysql_communication_stack.inc

@@ -0,0 +1,10 @@
+# ==== Purpose ====
+#
+# Ensure that group replication is running using MySQL protocol stack
+#
+
+--let $_have_mysql_stack= `SELECT @@group_replication_communication_stack LIKE 'MYSQL'`
+if (!$_have_mysql_stack)
+{
+  --skip Test requires MySQL Protocol Stack configured in Group Replication
+}

mysql-test/include/have_group_replication_xcom_communication_stack.inc

@@ -0,0 +1,10 @@
+# ==== Purpose ====
+#
+# Ensure that group replication is running using MySQL protocol stack
+#
+
+--let $_have_xcom_stack= `SELECT @@group_replication_communication_stack LIKE 'XCOM'`
+if (!$_have_xcom_stack)
+{
+  --skip Test requires XCom Protocol Stack configured in Group Replication
+}

mysql-test/include/mtr_warnings.sql

@@ -197,7 +197,7 @@ INSERT INTO global_suppressions VALUES
  ("slave SQL thread aborted"),
  ("Slave: .*Duplicate entry"),
 
- /* 
+ /*
     innodb_dedicated_server warning which raised if innodb_buffer_pool_size,
     innodb_log_file_size or innodb_flush_method is specified.
  */
@@ -312,6 +312,12 @@ INSERT INTO global_suppressions VALUES
  ("Members removed from the group.*"),
  ("Error while sending message for group replication recovery"),
  ("Slave SQL for channel 'group_replication_recovery': ... The slave coordinator and worker threads are .*"),
+ ("A message intended for a client cannot be sent there as no client-session is attached. Therefore, we're sending the information to the error-log instead: MY-001160 - Got an error writing communication packets.*"),
+ ("A message intended for a client cannot be sent there as no client-session is attached. Therefore, we're sending the information to the error-log instead: MY-001158 - Got an error reading communication packets.*"),
+ ("Failed to establish MySQL client connection in Group Replication.*."),
+ ("\\[GCS\\] client closed the signalling connection .*"),
+ ("\\[GCS\\] local_server: client closed the signalling connection.*"),
+ ("\\[GCS\\] local_server: error reading from the signalling connection.*"),
 
  /*
    Warnings/errors related to SSL connection by mysqlx

mysql-test/include/spawn_monitoring_process.inc

@@ -20,7 +20,6 @@
 let $have_windows = `SELECT CONVERT(@@version_compile_os using latin1) IN ("Win32", "Win64", "Windows")`;
 
 if (!$have_windows) {
-
   --disable_query_log
   SET SESSION sql_log_bin= 0;
   SET @auto_increment_increment_save= @@SESSION.auto_increment_increment;
@@ -40,10 +39,24 @@ if (!$have_windows) {
    ORDER BY VARIABLE_NAME;
   if($rpl_group_replication)
   {
+    --let $gr_ssl_disabled= `SELECT @@group_replication_ssl_mode LIKE 'DISABLED'`
+    if(!$gr_ssl_disabled) {
+       INSERT INTO test.r_vars (var_name, var_value)
+         SELECT * FROM performance_schema.global_variables
+         WHERE VARIABLE_NAME LIKE 'ssl_ca'
+         OR VARIABLE_NAME LIKE 'ssl_cert'
+         OR VARIABLE_NAME LIKE 'ssl_key'
+         ORDER BY VARIABLE_NAME;
+    }
+
     INSERT INTO test.r_vars (var_name, var_value)
      SELECT * FROM performance_schema.global_variables
      WHERE VARIABLE_NAME LIKE 'group_replication_ssl_mode'
      OR VARIABLE_NAME LIKE 'group_replication_gtid_assignment_block_size'
+     OR VARIABLE_NAME LIKE 'group_replication_ssl_mode'
+     OR VARIABLE_NAME LIKE 'group_replication_recovery_ssl_ca'
+     OR VARIABLE_NAME LIKE 'group_replication_recovery_ssl_cert'
+     OR VARIABLE_NAME LIKE 'group_replication_recovery_ssl_key'
      ORDER BY VARIABLE_NAME;
   }
   --let $suite_mysqld_settings=

mysql-test/r/events_bugs.result

@@ -570,7 +570,7 @@ USE test;
 SHOW GRANTS FOR CURRENT_USER;
 Grants for root@localhost
 GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, SHUTDOWN, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, SUPER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER, CREATE TABLESPACE, CREATE ROLE, DROP ROLE ON *.* TO `root`@`localhost` WITH GRANT OPTION
-GRANT APPLICATION_PASSWORD_ADMIN,AUDIT_ADMIN,BACKUP_ADMIN,BINLOG_ADMIN,BINLOG_ENCRYPTION_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,ENCRYPTION_KEY_ADMIN,FLUSH_OPTIMIZER_COSTS,FLUSH_STATUS,FLUSH_TABLES,FLUSH_USER_RESOURCES,GROUP_REPLICATION_ADMIN,INNODB_REDO_LOG_ARCHIVE,INNODB_REDO_LOG_ENABLE,PERSIST_RO_VARIABLES_ADMIN,REPLICATION_APPLIER,REPLICATION_SLAVE_ADMIN,RESOURCE_GROUP_ADMIN,RESOURCE_GROUP_USER,ROLE_ADMIN,SERVICE_CONNECTION_ADMIN,SESSION_VARIABLES_ADMIN,SET_USER_ID,SHOW_ROUTINE,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN,TABLE_ENCRYPTION_ADMIN,XA_RECOVER_ADMIN ON *.* TO `root`@`localhost` WITH GRANT OPTION
+GRANT APPLICATION_PASSWORD_ADMIN,AUDIT_ADMIN,BACKUP_ADMIN,BINLOG_ADMIN,BINLOG_ENCRYPTION_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,ENCRYPTION_KEY_ADMIN,FLUSH_OPTIMIZER_COSTS,FLUSH_STATUS,FLUSH_TABLES,FLUSH_USER_RESOURCES,GROUP_REPLICATION_ADMIN,GROUP_REPLICATION_STREAM,INNODB_REDO_LOG_ARCHIVE,INNODB_REDO_LOG_ENABLE,PERSIST_RO_VARIABLES_ADMIN,REPLICATION_APPLIER,REPLICATION_SLAVE_ADMIN,RESOURCE_GROUP_ADMIN,RESOURCE_GROUP_USER,ROLE_ADMIN,SERVICE_CONNECTION_ADMIN,SESSION_VARIABLES_ADMIN,SET_USER_ID,SHOW_ROUTINE,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN,TABLE_ENCRYPTION_ADMIN,XA_RECOVER_ADMIN ON *.* TO `root`@`localhost` WITH GRANT OPTION
 GRANT PROXY ON ''@'' TO 'root'@'localhost' WITH GRANT OPTION
 SET GLOBAL event_scheduler = ON;
 CREATE TABLE events_test.event_log