MDEV-28948 FLUSH BINARY LOGS waits/hangs on mysql_mutex_unlock(&LOCK_…

…index)

The hang may be caused by a 1pc branch that was fixed by MDEV-26031 in
10.6 and up. That commit did not look relevant in 10.5 and below
so  was not pushed to the low branches.

To possibly tackle the reported issue
the MDEV-26031 is backported now with a test that
unlike 10.6 does not expose the former bug in 10.5.
It is only needed for checking a refined logics
inside MYSQL_BIN_LOG::write_transaction_to_binlog.
The latter is made to do away with xid-unlogging (which is suspected
to have been at fault) for xid-less transaction.

mysql-test/suite/binlog/r/binlog_1pc.result

@@ -0,0 +1,34 @@
+RESET MASTER;
+CREATE TABLE t (f INT) ENGINE=INNODB;
+CREATE TABLE ta (f INT) ENGINE=Aria;
+BEGIN;
+INSERT INTO t  SET f = 1;
+INSERT INTO ta SET f = 1;
+COMMIT;
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO ta SET f = 1
+master-bin.000001	#	Query	#	#	COMMIT
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO t  SET f = 1
+master-bin.000001	#	Query	#	#	COMMIT
+# Prove that binlog is resettable. Under the bug condition it was not.
+RESET MASTER;
+SET @@binlog_format = ROW;
+CREATE TABLE ta2 ENGINE = Aria SELECT * from t;
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE `ta2` (
+  `f` int(11) DEFAULT NULL
+) ENGINE=Aria PAGE_CHECKSUM=1
+master-bin.000001	#	Annotate_rows	#	#	CREATE TABLE ta2 ENGINE = Aria SELECT * from t
+master-bin.000001	#	Table_map	#	#	table_id: # (test.ta2)
+master-bin.000001	#	Write_rows_v1	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Query	#	#	COMMIT
+# Prove that binlog is resettable. Under the bug condition it was not.
+RESET MASTER;
+# Cleanup
+DROP TABLE ta2, ta, t;
+# End of the tests

mysql-test/suite/binlog/t/binlog_1pc.test

@@ -0,0 +1,39 @@
+# The test file contains tests specific to one phase commit
+# execution and binlogging.
+#
+# MDEV-26031 unnessary xid logging in one phase commit case
+#
+--source include/have_innodb.inc
+--source include/have_aria.inc
+--source include/have_binlog_format_mixed.inc
+
+RESET MASTER;
+
+CREATE TABLE t (f INT) ENGINE=INNODB;
+CREATE TABLE ta (f INT) ENGINE=Aria;
+
+--let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1)
+BEGIN;
+  INSERT INTO t  SET f = 1;
+  INSERT INTO ta SET f = 1;
+COMMIT;
+--let $binlog_file =  query_get_value(SHOW MASTER STATUS, File, 1)
+--source include/show_binlog_events.inc
+
+--echo # Prove that binlog is resettable. Under the bug condition it was not.
+RESET MASTER;
+
+--let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1)
+--let $binlog_file =  query_get_value(SHOW MASTER STATUS, File, 1)
+SET @@binlog_format = ROW;
+CREATE TABLE ta2 ENGINE = Aria SELECT * from t;
+--source include/show_binlog_events.inc
+
+--echo # Prove that binlog is resettable. Under the bug condition it was not.
+RESET MASTER;
+
+
+--echo # Cleanup
+DROP TABLE ta2, ta, t;
+
+--echo # End of the tests

sql/log.cc

@@ -2177,9 +2177,19 @@ static int binlog_commit(handlerton *hton, THD *thd, bool all)
   */
   if (likely(!error) && ending_trans(thd, all))
   {
-    error= is_preparing_xa(thd) ?
+    bool is_xa_prepare= is_preparing_xa(thd);
+
+    error= is_xa_prepare ?
       binlog_commit_flush_xa_prepare(thd, all, cache_mngr) :
       binlog_commit_flush_trx_cache (thd, all, cache_mngr);
+    // the user xa is unlogged on common exec path with the "empty" xa case
+    if (cache_mngr->need_unlog && !is_xa_prepare)
+    {
+      error=
+        mysql_bin_log.unlog(BINLOG_COOKIE_MAKE(cache_mngr->binlog_id,
+                                               cache_mngr->delayed_error), 1);
+      cache_mngr->need_unlog= false;
+    }
   }
   /*
     This is part of the stmt rollback.
@@ -7621,16 +7631,16 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd,
   entry.using_trx_cache= using_trx_cache;
   entry.need_unlog= is_preparing_xa(thd);
   ha_info= all ? thd->transaction->all.ha_list : thd->transaction->stmt.ha_list;
+  entry.end_event= end_ev;
+  auto has_xid= entry.end_event->get_type_code() == XID_EVENT;
 
-  for (; !entry.need_unlog && ha_info; ha_info= ha_info->next())
+  for (; has_xid && !entry.need_unlog && ha_info; ha_info= ha_info->next())
   {
     if (ha_info->is_started() && ha_info->ht() != binlog_hton &&
         !ha_info->ht()->commit_checkpoint_request)
       entry.need_unlog= true;
-    break;
   }
 
-  entry.end_event= end_ev;
   if (cache_mngr->stmt_cache.has_incident() ||
       cache_mngr->trx_cache.has_incident())
   {