MDEV-18353 Shutdown may miss to wait for connection thread

* count CONNECT objects too
* wait for all CONNECT objects to disappear (to be converted to THDs)
  before killing THDs away

mysql-test/main/shutdown_not_windows.combinations

@@ -0,0 +1,5 @@
+[1tpc]
+--thread-handling=one-thread-per-connection
+
+[pot]
+--thread-handling=pool-of-threads

mysql-test/main/shutdown_not_windows.result

@@ -0,0 +1,8 @@
+#
+# MDEV-18353 Shutdown may miss to wait for connection thread
+#
+call mtr.add_suppression('Thread .* did not exit');
+set @old_dbug=@@global.debug_dbug;
+set global debug_dbug='+d,CONNECT_wait';
+select variable_value into @cons from information_schema.global_status where variable_name='connections';
+# restart

mysql-test/main/shutdown_not_windows.test

@@ -0,0 +1,14 @@
+source include/not_windows.inc;
+source include/not_embedded.inc;
+source include/have_debug.inc;
+--echo #
+--echo # MDEV-18353 Shutdown may miss to wait for connection thread
+--echo #
+call mtr.add_suppression('Thread .* did not exit');
+set @old_dbug=@@global.debug_dbug;
+set global debug_dbug='+d,CONNECT_wait';
+select variable_value into @cons from information_schema.global_status where variable_name='connections';
+exec $MYSQL -e 'select sleep(3600)' >/dev/null 2>&1 &;
+let $wait_condition= select variable_value>@cons from information_schema.global_status where variable_name='connections';
+source include/wait_condition.inc;
+source include/restart_mysqld.inc;

sql/mysqld.cc

@@ -456,7 +456,7 @@ ulong delay_key_write_options;
 uint protocol_version;
 uint lower_case_table_names;
 ulong tc_heuristic_recover= 0;
-Atomic_counter<uint32_t> THD_count::count;
+Atomic_counter<uint32_t> THD_count::count, CONNECT::count;
 bool shutdown_wait_for_slaves;
 Atomic_counter<uint32_t> slave_open_temp_tables;
 ulong thread_created;
@@ -1719,6 +1719,9 @@ static void close_connections(void)
 #endif
   end_thr_alarm(0);			 // Abort old alarms.
 
+  while (CONNECT::count)
+    my_sleep(100);
+
   /*
     First signal all threads that it's time to die
     This will give the threads some time to gracefully abort their
@@ -8036,7 +8039,7 @@ static int mysql_init_variables(void)
   mqh_used= 0;
   cleanup_done= 0;
   select_errors= dropping_tables= ha_open_options=0;
-  THD_count::count= kill_cached_threads= wake_thread= 0;
+  THD_count::count= CONNECT::count= kill_cached_threads= wake_thread= 0;
   slave_open_temp_tables= 0;
   cached_thread_count= 0;
   opt_endinfo= using_udf_functions= 0;

sql/sql_connect.cc

@@ -1360,6 +1360,14 @@ void do_handle_one_connection(CONNECT *connect)
     return;
   }
 
+  DBUG_EXECUTE_IF("CONNECT_wait",
+  {
+    extern MYSQL_SOCKET unix_sock;
+    DBUG_ASSERT(unix_sock.fd >= 0);
+    while (unix_sock.fd >= 0)
+      my_sleep(1000);
+  });
+
   /*
     If a thread was created to handle this connection:
     increment slow_launch_threads counter if it took more than
@@ -1373,10 +1381,10 @@ void do_handle_one_connection(CONNECT *connect)
     if (launch_time >= slow_launch_time*1000000L)
       statistic_increment(slow_launch_threads, &LOCK_status);
   }
-  delete connect;
 
-  /* Make THD visible in show processlist */
-  server_threads.insert(thd);
+  server_threads.insert(thd); // Make THD visible in show processlist
+
+  delete connect; // must be after server_threads.insert, see close_connections()
 
   thd->thr_create_utime= thr_create_utime;
   /* We need to set this because of time_out_user_resource_limits */
@@ -1482,6 +1490,7 @@ CONNECT::~CONNECT()
 {
   if (vio)
     vio_delete(vio);
+  count--;
 }
 
 

sql/sql_connect.h

@@ -42,11 +42,14 @@ class CONNECT : public ilink {
   bool thread_count_incremented;
   ulonglong    prior_thr_create_utime;
 
+  static Atomic_counter<uint32_t> count;
+
   CONNECT()
     :vio(0), host(0), scheduler(thread_scheduler), thread_id(0), real_id(0),
     extra_port(0),
     thread_count_incremented(0), prior_thr_create_utime(0)
   {
+    count++;
   };
   ~CONNECT();
   void close_and_delete();

sql/threadpool_common.cc

@@ -222,6 +222,14 @@ static THD* threadpool_add_connection(CONNECT *connect, void *scheduler_data)
 {
   THD *thd= NULL;
 
+  DBUG_EXECUTE_IF("CONNECT_wait",
+  {
+    extern MYSQL_SOCKET unix_sock;
+    DBUG_ASSERT(unix_sock.fd >= 0);
+    while (unix_sock.fd >= 0)
+      my_sleep(1000);
+  });
+
   /*
     Create a new connection context: mysys_thread_var and PSI thread
     Store them in THD.
@@ -247,8 +255,8 @@ static THD* threadpool_add_connection(CONNECT *connect, void *scheduler_data)
     }
     return NULL;
   }
-  delete connect;
-  server_threads.insert(thd);
+  server_threads.insert(thd); // Make THD visible in show processlist
+  delete connect; // must be after server_threads.insert, see close_connections()
   thd->set_mysys_var(mysys_var);
   thd->event_scheduler.data= scheduler_data;