MDEV-14526: MariaDB keeps crashing under load when query_cache_type is changed

The problem was in such scenario:
T1 - starts registering query and locked QC
T2 - starts disabling QC and wait for UNLOCK
T1 - unlock QC
T2 - disable QC and destroy signals without waiting for query unlock
T1 a) - not yet unlocked query in qc and crash on attempt to unlock because
        QC signals are destroyed
   b) if above was done before destruction, it execute end_of results first
      time at exit on after try_lock which see QC disables and return TRUE.
      But it do not reset query_cache_tls->first_query_block which lead to
      second call of end_of_result when diagnostic arena has already
      inappropriate status (not is_eof()).

Fix is:
  1) wait for all queries unlocked before destroying them by locking and
     unlocking
  2) remove query_cache_tls->first_query_block if QC disabled

Author: sanja-byelkin

mysql-test/r/query_cache_debug.result

@@ -220,3 +220,29 @@ RESET QUERY CACHE;
 DROP TABLE t1;
 SET GLOBAL query_cache_size= DEFAULT;
 SET GLOBAL query_cache_type= DEFAULT;
+#
+# MDEV-14526: MariaDB keeps crashing under load when
+# query_cache_type is changed
+#
+CREATE TABLE t1 (
+`id` int(10) NOT NULL AUTO_INCREMENT,
+`k` int(10) default '0',
+PRIMARY KEY (`id`))
+ENGINE=MyISAM;
+INSERT IGNORE INTO t1 VALUES
+(NULL,1),(NULL,8),(NULL,NULL),(NULL,NULL),(NULL,4),(NULL,9),(NULL,7),
+(NULL,3),(NULL,NULL),(NULL,2),(NULL,3),(NULL,NULL),(NULL,2),(NULL,7),
+(NULL,1),(NULL,2),(NULL,4),(NULL,NULL),(NULL,1),(NULL,1),(NULL,4);
+SET GLOBAL query_cache_size= 1024*1024;
+SET GLOBAL query_cache_type= 1;
+set debug_sync="wait_in_query_cache_store_query SIGNAL parked WAIT_FOR go";
+SELECT DISTINCT id FROM t1 WHERE id BETWEEN 5603 AND 16218 ORDER BY k;
+set debug_sync="now WAIT_FOR parked";
+SET GLOBAL query_cache_type= 0;
+set debug_sync="now SIGNAL go";
+id
+set debug_sync= 'RESET';
+DROP TABLE t1;
+SEt GLOBAL query_cache_size= DEFAULT;
+SEt GLOBAL query_cache_type= DEFAULT;
+# End of 5.5 tests

mysql-test/t/query_cache_debug.test

@@ -319,3 +319,55 @@ RESET QUERY CACHE;
 DROP TABLE t1;
 SET GLOBAL query_cache_size= DEFAULT;
 SET GLOBAL query_cache_type= DEFAULT;
+
+--echo #
+--echo # MDEV-14526: MariaDB keeps crashing under load when
+--echo # query_cache_type is changed
+--echo #
+
+CREATE TABLE t1 (
+  `id` int(10) NOT NULL AUTO_INCREMENT,
+  `k` int(10) default '0',
+  PRIMARY KEY (`id`))
+ENGINE=MyISAM;
+
+INSERT IGNORE INTO t1 VALUES
+  (NULL,1),(NULL,8),(NULL,NULL),(NULL,NULL),(NULL,4),(NULL,9),(NULL,7),
+  (NULL,3),(NULL,NULL),(NULL,2),(NULL,3),(NULL,NULL),(NULL,2),(NULL,7),
+  (NULL,1),(NULL,2),(NULL,4),(NULL,NULL),(NULL,1),(NULL,1),(NULL,4);
+
+SET GLOBAL query_cache_size= 1024*1024;
+SET GLOBAL query_cache_type= 1;
+
+--connect (con2,localhost,root,,test)
+--connect (con1,localhost,root,,test)
+set debug_sync="wait_in_query_cache_store_query SIGNAL parked WAIT_FOR go";
+--send
+
+	SELECT DISTINCT id FROM t1 WHERE id BETWEEN 5603 AND 16218 ORDER BY k;
+
+--connection default
+
+set debug_sync="now WAIT_FOR parked";
+--connection con2
+--send
+        SET GLOBAL query_cache_type= 0;
+
+--connection default
+set debug_sync="now SIGNAL go";
+
+--connection con1
+--reap
+--connection con2
+--reap
+
+# Cleanup
+--disconnect con1
+--disconnect con2
+--connection default
+set debug_sync= 'RESET';
+DROP TABLE t1;
+SEt GLOBAL query_cache_size= DEFAULT;
+SEt GLOBAL query_cache_type= DEFAULT;
+
+--echo # End of 5.5 tests

sql/sql_cache.cc

@@ -1184,7 +1184,11 @@ void Query_cache::end_of_result(THD *thd)
 #endif
 
   if (try_lock(thd, Query_cache::WAIT))
+  {
+    if (is_disabled())
+      query_cache_tls->first_query_block= NULL; // do not try again with QC
     DBUG_VOID_RETURN;
+  }
 
   query_block= query_cache_tls->first_query_block;
   if (query_block)
@@ -1556,6 +1560,8 @@ def_week_frmt: %lu, in_trans: %d, autocommit: %d",
 
         unlock();
 
+        DEBUG_SYNC(thd, "wait_in_query_cache_store_query");
+
 	// init_n_lock make query block locked
 	BLOCK_UNLOCK_WR(query_block);
       }
@@ -2679,20 +2685,31 @@ void Query_cache::make_disabled()
 
   This function frees all resources allocated by the cache.  You
   have to call init_cache() before using the cache again. This function
-  requires the structure_guard_mutex to be locked.
+  requires the cache to be locked (LOCKED_NO_WAIT, lock_and_suspend) or
+  disabling.
 */
 
 void Query_cache::free_cache()
 {
   DBUG_ENTER("Query_cache::free_cache");
 
+  DBUG_ASSERT(m_cache_lock_status == LOCKED_NO_WAIT ||
+              m_cache_status == DISABLE_REQUEST);
+
   /* Destroy locks */
   Query_cache_block *block= queries_blocks;
   if (block)
   {
     do
     {
       Query_cache_query *query= block->query();
+      /*
+         There will not be new requests but some maybe not finished yet,
+         so wait for them by trying lock/unlock
+      */
+      BLOCK_LOCK_WR(block);
+      BLOCK_UNLOCK_WR(block);
+
       mysql_rwlock_destroy(&query->lock);
       block= block->next;
     } while (block != queries_blocks);