MDEV-8294: Inconsistent behavior of slave parallel threads at runtime

There were some cases where the slave SQL thread could stop without
the pool of parallel replication worker threads being correctly
de-activated.

mysql-test/suite/rpl/r/rpl_parallel2.result

@@ -12,6 +12,23 @@ Note	1592	Unsafe statement written to the binary log using statement format sinc
 include/wait_for_slave_param.inc [Seconds_Behind_Master]
 Seconds_Behind_Master should be zero here because the slave is fully caught up and idle.
 Seconds_Behind_Master = '0'
+*** MDEV-8294: Inconsistent behavior of slave parallel threads at runtime ***
+INSERT INTO t1 VALUES (10,0);
+SET sql_log_bin= 0;
+DELETE FROM t1 WHERE a=10;
+SET sql_log_bin= 1;
+INSERT INTO t1 VALUES (10,0);
+SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
+a	b
+10	0
+include/wait_for_slave_sql_error.inc [errno=1062]
+SET GLOBAL slave_parallel_threads=8;
+STOP SLAVE;
+SET GLOBAL sql_slave_skip_counter= 1;
+include/start_slave.inc
+SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
+a	b
+10	0
 include/stop_slave.inc
 SET GLOBAL slave_parallel_threads=@old_parallel_threads;
 include/start_slave.inc

mysql-test/suite/rpl/t/rpl_parallel2.test

@@ -37,6 +37,48 @@ INSERT INTO t1 VALUES (1,sleep(2));
 --source include/show_slave_status.inc
 
 
+--echo *** MDEV-8294: Inconsistent behavior of slave parallel threads at runtime ***
+
+--connection server_1
+INSERT INTO t1 VALUES (10,0);
+# Force a duplicate key error on the slave.
+SET sql_log_bin= 0;
+DELETE FROM t1 WHERE a=10;
+SET sql_log_bin= 1;
+INSERT INTO t1 VALUES (10,0);
+--save_master_pos
+SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
+
+--connection server_2
+--let $slave_sql_errno= 1062
+--source include/wait_for_slave_sql_error.inc
+
+# At this point, the worker threads should have stopped also.
+--let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.processlist WHERE User = "system user" AND State = "Waiting for work from SQL thread";
+--source include/wait_condition.inc
+
+# Check that the pool can still be resized, but remains inactive as no slave
+# SQL thread is running.
+SET GLOBAL slave_parallel_threads=8;
+--let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.processlist WHERE User = "system user" AND State = "Waiting for work from SQL thread";
+--source include/wait_condition.inc
+
+STOP SLAVE;
+# At this point, the worker threads should have stopped.
+--let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.processlist WHERE User = "system user" AND State = "Waiting for work from SQL thread";
+--source include/wait_condition.inc
+
+
+SET GLOBAL sql_slave_skip_counter= 1;
+--source include/start_slave.inc
+# At this point, the worker threads should have been spawned.
+--let $wait_condition= SELECT COUNT(*)=8 FROM information_schema.processlist WHERE User = "system user" AND State = "Waiting for work from SQL thread";
+--source include/wait_condition.inc
+--sync_with_master
+SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
+
+
+# Clean up
 --connection server_2
 --source include/stop_slave.inc
 SET GLOBAL slave_parallel_threads=@old_parallel_threads;

sql/slave.cc

@@ -652,11 +652,10 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock)
       DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS);
 
     mysql_mutex_unlock(log_lock);
-
-    if (opt_slave_parallel_threads > 0 &&
-        !master_info_index->any_slave_sql_running())
-      rpl_parallel_inactivate_pool(&global_rpl_thread_pool);
   }
+  if (opt_slave_parallel_threads > 0 &&
+      !master_info_index->any_slave_sql_running())
+    rpl_parallel_inactivate_pool(&global_rpl_thread_pool);
   if (thread_mask & (SLAVE_IO|SLAVE_FORCE_ALL))
   {
     DBUG_PRINT("info",("Terminating IO thread"));
@@ -4820,10 +4819,8 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME,
   THD_CHECK_SENTRY(thd);
   rli->sql_driver_thd= 0;
   mysql_mutex_lock(&LOCK_thread_count);
-  THD_CHECK_SENTRY(thd);
   thd->rgi_fake= thd->rgi_slave= NULL;
   delete serial_rgi;
-  delete thd;
   mysql_mutex_unlock(&LOCK_thread_count);
  /*
   Note: the order of the broadcast and unlock calls below (first broadcast, then unlock)
@@ -4834,6 +4831,22 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME,
   DBUG_EXECUTE_IF("simulate_slave_delay_at_terminate_bug38694", sleep(5););
   mysql_mutex_unlock(&rli->run_lock);  // tell the world we are done
 
+  /*
+    Deactivate the parallel replication thread pool, if there are now no more
+    SQL threads running. Do this here, when we have released all locks, but
+    while our THD (and current_thd) is still valid.
+  */
+  mysql_mutex_lock(&LOCK_active_mi);
+  if (opt_slave_parallel_threads > 0 &&
+      !master_info_index->any_slave_sql_running())
+    rpl_parallel_inactivate_pool(&global_rpl_thread_pool);
+  mysql_mutex_unlock(&LOCK_active_mi);
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  THD_CHECK_SENTRY(thd);
+  delete thd;
+  mysql_mutex_unlock(&LOCK_thread_count);
+
   DBUG_LEAVE;                                   // Must match DBUG_ENTER()
   my_thread_end();
 #ifdef HAVE_OPENSSL