Bug#11751904 : 'SHOW GLOBAL STATUS' CAUSES LOCK CONTENTION ON

LOCK_THREAD_COUNT

Analysis:
Lock duration for LOCK_thread_count is high when adding thread 
level status to global status variable in function 
calc_sum_of_all_status(). It performs global_thread_list traversal
in order to calculate the sum. Because of this, new connections 
are blocked till the traversal is completed.

In this scenario, major bottleneck is insertion(accepting new 
incoming connections) during traversal and not removal as it can 
wait till the traversal is completed.

Fix:
Take copy of THDs from global_thread_list and perform calculation
after releasing LOCK_thread_count. During traversal(on copied 
THDs), removal from global_thread_list is blocked using another 
mutex LOCK_thread_remove such that THD copied are valid during 
computation(otherwise remove destroys THD).

sql/global_threads.h

@@ -23,6 +23,7 @@
 class THD;
 
 extern mysql_mutex_t LOCK_thread_count;
+extern mysql_mutex_t LOCK_thread_remove;
 extern mysql_cond_t COND_thread_count;
 
 /*
@@ -39,6 +40,7 @@ extern mysql_cond_t COND_thread_count;
 typedef std::set<THD*>::iterator Thread_iterator;
 Thread_iterator global_thread_list_begin();
 Thread_iterator global_thread_list_end();
+void copy_global_thread_list(std::set<THD*> *new_copy);
 void add_global_thread(THD *);
 void remove_global_thread(THD *);
 

sql/mysqld.cc

@@ -685,7 +685,7 @@ SHOW_COMP_OPTION have_profiling;
 
 pthread_key(MEM_ROOT**,THR_MALLOC);
 pthread_key(THD*, THR_THD);
-mysql_mutex_t LOCK_thread_count;
+mysql_mutex_t LOCK_thread_count, LOCK_thread_remove;
 mysql_mutex_t
   LOCK_error_log, LOCK_uuid_generator,
   LOCK_delayed_insert, LOCK_delayed_status, LOCK_delayed_create,
@@ -797,6 +797,17 @@ Thread_iterator global_thread_list_end()
   return global_thread_list->end();
 }
 
+void copy_global_thread_list(std::set<THD*> *new_copy)
+{
+  mysql_mutex_assert_owner(&LOCK_thread_count);
+  try {
+    *new_copy= *global_thread_list;
+  }
+  catch (std::bad_alloc &ba)
+  {
+  } 
+}
+
 void add_global_thread(THD *thd)
 {
   DBUG_PRINT("info", ("add_global_thread %p", thd));
@@ -819,7 +830,9 @@ void remove_global_thread(THD *thd)
   mysql_mutex_assert_owner(&LOCK_thread_count);
   DBUG_ASSERT(thd->release_resources_done());
 
+  mysql_mutex_lock(&LOCK_thread_remove);
   const size_t num_erased= global_thread_list->erase(thd);
+  mysql_mutex_unlock(&LOCK_thread_remove);
   if (num_erased == 1)
     --global_thread_count;
   // Removing a THD that was never added is an error.
@@ -1908,6 +1921,7 @@ static void clean_up_mutexes()
   mysql_mutex_destroy(&LOCK_prepared_stmt_count);
   mysql_mutex_destroy(&LOCK_error_messages);
   mysql_cond_destroy(&COND_thread_count);
+  mysql_mutex_destroy(&LOCK_thread_remove);
   mysql_cond_destroy(&COND_thread_cache);
   mysql_cond_destroy(&COND_flush_thread_cache);
   mysql_cond_destroy(&COND_manager);
@@ -4060,6 +4074,8 @@ You should consider changing lower_case_table_names to 1 or 2",
 static int init_thread_environment()
 {
   mysql_mutex_init(key_LOCK_thread_count, &LOCK_thread_count, MY_MUTEX_INIT_FAST);
+  mysql_mutex_init(key_LOCK_thread_remove, 
+                   &LOCK_thread_remove, MY_MUTEX_INIT_FAST);
   mysql_rwlock_init(key_rwlock_LOCK_status, &LOCK_status);
   mysql_mutex_init(key_LOCK_delayed_insert,
                    &LOCK_delayed_insert, MY_MUTEX_INIT_FAST);
@@ -9024,6 +9040,7 @@ PSI_mutex_key
   key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data,
   key_LOCK_error_messages, key_LOG_INFO_lock, key_LOCK_thread_count,
   key_LOCK_log_throttle_qni;
+PSI_mutex_key key_LOCK_thread_remove;
 PSI_mutex_key key_RELAYLOG_LOCK_commit;
 PSI_mutex_key key_RELAYLOG_LOCK_commit_queue;
 PSI_mutex_key key_RELAYLOG_LOCK_done;
@@ -9101,6 +9118,7 @@ static PSI_mutex_info all_server_mutexes[]=
   { &key_LOCK_error_messages, "LOCK_error_messages", PSI_FLAG_GLOBAL},
   { &key_LOG_INFO_lock, "LOG_INFO::lock", 0},
   { &key_LOCK_thread_count, "LOCK_thread_count", PSI_FLAG_GLOBAL},
+  { &key_LOCK_thread_remove, "LOCK_thread_remove", PSI_FLAG_GLOBAL},
   { &key_LOCK_log_throttle_qni, "LOCK_log_throttle_qni", PSI_FLAG_GLOBAL},
   { &key_gtid_ensure_index_mutex, "Gtid_state", PSI_FLAG_GLOBAL}
 };

sql/mysqld.h

@@ -323,7 +323,7 @@ extern PSI_mutex_key
   key_mutex_slave_parallel_pend_jobs, key_mutex_mts_temp_tables_lock,
   key_mutex_slave_parallel_worker,
   key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data,
-  key_LOCK_error_messages, key_LOCK_thread_count,
+  key_LOCK_error_messages, key_LOCK_thread_count, key_LOCK_thread_remove,
   key_LOCK_log_throttle_qni;
 extern PSI_mutex_key key_RELAYLOG_LOCK_commit;
 extern PSI_mutex_key key_RELAYLOG_LOCK_commit_queue;

sql/sql_show.cc

@@ -2594,18 +2594,33 @@ void calc_sum_of_all_status(STATUS_VAR *to)
 {
   DBUG_ENTER("calc_sum_of_all_status");
 
+  /* take copy of global_thread_list */
+  std::set<THD*> global_thread_list_copy;
+
   mysql_mutex_lock(&LOCK_thread_count);
+  mysql_mutex_lock(&LOCK_thread_remove);
+  copy_global_thread_list(&global_thread_list_copy);
+
+  /* 
+    Allow inserts to global_thread_list. Newly added thd
+    will not be accounted for summary calculation.
+    removal from global_thread_list is blocked as LOCK_thread_remove
+    mutex is not released yet 
+  */
+  mysql_mutex_unlock(&LOCK_thread_count);
+  Thread_iterator it= global_thread_list_copy.begin();
+  Thread_iterator end= global_thread_list_copy.end();
 
-  Thread_iterator it= global_thread_list_begin();
-  Thread_iterator end= global_thread_list_end();
   /* Get global values as base */
   *to= global_status_var;
 
   /* Add to this status from existing threads */
   for (; it != end; ++it)
     add_to_status(to, &(*it)->status_var);
 
-  mysql_mutex_unlock(&LOCK_thread_count);
+  DEBUG_SYNC_C("inside_calc_sum");
+
+  mysql_mutex_unlock(&LOCK_thread_remove);
   DBUG_VOID_RETURN;
 }