MDEV-22707: galera got stuck after flush tables

Deadlock is possible between applier thread and local committing thread with active FLUSH TABLE.
Applier thread should skip table share checks and locks when opening table.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/MDEV-22707.result

@@ -0,0 +1,26 @@
+CREATE TABLE t1(f2 INT) ENGINE=InnoDB;
+connect node_1_applier_thd, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET GLOBAL debug_dbug = "+d,sync.wsrep_apply_cb";
+connection node_2;
+SET SESSION wsrep_sync_wait = 0;
+INSERT INTO t1 (f2) VALUES (2);
+connection node_1_applier_thd;
+SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
+connection node_1;
+SET SESSION wsrep_sync_wait = 0;
+SET DEBUG_SYNC = 'wsrep_before_replication SIGNAL wsrep_before_replication_reached WAIT_FOR continue';
+INSERT INTO t1 (f2) VALUES (1);
+connect node_1_flush_thd, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET DEBUG_SYNC="now WAIT_FOR wsrep_before_replication_reached";
+SET GLOBAL debug_dbug = "+d,sync.wsrep_before_mdl_wait";
+FLUSH TABLES;
+connect node_1_sync_release_thd, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET GLOBAL debug_dbug = "";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_before_mdl_wait";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
+SET DEBUG_SYNC = "now SIGNAL continue";
+SET DEBUG_SYNC = "RESET";
+connection node_1;
+connection node_1_flush_thd;
+connection node_2;
+DROP TABLE t1;

mysql-test/suite/galera/t/MDEV-22707.test

@@ -0,0 +1,51 @@
+#
+# MDEV-22707 galera got stuck after flush tables
+#
+
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+
+CREATE TABLE t1(f2 INT) ENGINE=InnoDB;
+
+--connect node_1_applier_thd, 127.0.0.1, root, , test, $NODE_MYPORT_1
+SET GLOBAL debug_dbug = "+d,sync.wsrep_apply_cb";
+
+--connection node_2
+SET SESSION wsrep_sync_wait = 0;
+--send INSERT INTO t1 (f2) VALUES (2)
+
+--connection node_1_applier_thd
+# Wait for `sync.wsrep_apply_cb_reached` signal
+SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
+
+--connection node_1
+SET SESSION wsrep_sync_wait = 0;
+SET DEBUG_SYNC = 'wsrep_before_replication SIGNAL wsrep_before_replication_reached WAIT_FOR continue';
+--send INSERT INTO t1 (f2) VALUES (1)
+
+--connect node_1_flush_thd, 127.0.0.1, root, , test, $NODE_MYPORT_1
+SET DEBUG_SYNC="now WAIT_FOR wsrep_before_replication_reached";
+SET GLOBAL debug_dbug = "+d,sync.wsrep_before_mdl_wait";
+--send FLUSH TABLES
+
+--connect node_1_sync_release_thd, 127.0.0.1, root, , test, $NODE_MYPORT_1
+# First clear all DBUG points
+SET GLOBAL debug_dbug = "";
+# Now signal threads to continue execution
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_before_mdl_wait";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
+SET DEBUG_SYNC = "now SIGNAL continue";
+SET DEBUG_SYNC = "RESET";
+
+--connection node_1
+--reap
+
+--connection node_1_flush_thd
+--reap
+
+--connection node_2
+--reap
+
+DROP TABLE t1;

sql/sql_base.cc

@@ -1859,7 +1859,12 @@ bool open_table(THD *thd, TABLE_LIST *table_list, Open_table_context *ot_ctx)
     DBUG_RETURN(FALSE);
   }
 
+#ifdef WITH_WSREP
+  if (!((flags & MYSQL_OPEN_IGNORE_FLUSH) ||
+        (wsrep_on(thd) && thd->wsrep_applier)))
+#else
   if (!(flags & MYSQL_OPEN_IGNORE_FLUSH))
+#endif
   {
     if (share->tdc->flushed)
     {