From 2628fa2dba31a779b9bfc12bbcce0210e97f8c56 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Fri, 5 Feb 2021 12:52:48 +1100
Subject: [PATCH 01/11] MDEV-20857: perf schema conflict name filename_hash

filename_hash is a function from libiberty.a from the system
but also an expored name in the perf schema static library.

We'll use a different name.
---
 storage/perfschema/pfs_engine_table.cc |  4 ++--
 storage/perfschema/pfs_instr.cc        | 20 ++++++++++----------
 storage/perfschema/pfs_instr.h         |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/storage/perfschema/pfs_engine_table.cc b/storage/perfschema/pfs_engine_table.cc
index e2ad190c435dd..acab0e73a3d24 100644
--- a/storage/perfschema/pfs_engine_table.cc
+++ b/storage/perfschema/pfs_engine_table.cc
@@ -1233,11 +1233,11 @@ bool pfs_show_status(handlerton *hton, THD *thd,
       break;
     case 141:
       name= "(filename_hash).count";
-      size= filename_hash.count;
+      size= pfs_filename_hash.count;
       break;
     case 142:
       name= "(filename_hash).size";
-      size= filename_hash.size;
+      size= pfs_filename_hash.size;
       break;
     case 143:
       name= "(host_hash).count";
diff --git a/storage/perfschema/pfs_instr.cc b/storage/perfschema/pfs_instr.cc
index ca9e0385021bd..fd8da77fe4072 100644
--- a/storage/perfschema/pfs_instr.cc
+++ b/storage/perfschema/pfs_instr.cc
@@ -143,7 +143,7 @@ PFS_thread *thread_array= NULL;
   File instrumentation instances array.
   @sa file_max
   @sa file_lost
-  @sa filename_hash
+  @sa pfs_filename_hash
 */
 PFS_file *file_array= NULL;
 
@@ -189,8 +189,8 @@ static unsigned char *history_stmts_digest_token_array= NULL;
 static char *thread_session_connect_attrs_array= NULL;
 
 /** Hash table for instrumented files. */
-LF_HASH filename_hash;
-/** True if filename_hash is initialized. */
+LF_HASH pfs_filename_hash;
+/** True if pfs_filename_hash is initialized. */
 static bool filename_hash_inited= false;
 
 /**
@@ -586,7 +586,7 @@ int init_file_hash(void)
 {
   if ((! filename_hash_inited) && (file_max > 0))
   {
-    lf_hash_init(&filename_hash, sizeof(PFS_file*), LF_HASH_UNIQUE,
+    lf_hash_init(&pfs_filename_hash, sizeof(PFS_file*), LF_HASH_UNIQUE,
                  0, 0, filename_hash_get_key, &my_charset_bin);
     /* filename_hash.size= file_max; */
     filename_hash_inited= true;
@@ -599,7 +599,7 @@ void cleanup_file_hash(void)
 {
   if (filename_hash_inited)
   {
-    lf_hash_destroy(&filename_hash);
+    lf_hash_destroy(&pfs_filename_hash);
     filename_hash_inited= false;
   }
 }
@@ -1186,7 +1186,7 @@ void destroy_thread(PFS_thread *pfs)
 }
 
 /**
-  Get the hash pins for @filename_hash.
+  Get the hash pins for @pfs_filename_hash.
   @param thread The running thread.
   @returns The LF_HASH pins for the thread.
 */
@@ -1196,7 +1196,7 @@ LF_PINS* get_filename_hash_pins(PFS_thread *thread)
   {
     if (! filename_hash_inited)
       return NULL;
-    thread->m_filename_hash_pins= lf_hash_get_pins(&filename_hash);
+    thread->m_filename_hash_pins= lf_hash_get_pins(&pfs_filename_hash);
   }
   return thread->m_filename_hash_pins;
 }
@@ -1314,7 +1314,7 @@ find_or_create_file(PFS_thread *thread, PFS_file_class *klass,
 search:
 
   entry= reinterpret_cast<PFS_file**>
-    (lf_hash_search(&filename_hash, pins,
+    (lf_hash_search(&pfs_filename_hash, pins,
                     normalized_filename, normalized_length));
   if (entry && (entry != MY_ERRPTR))
   {
@@ -1359,7 +1359,7 @@ find_or_create_file(PFS_thread *thread, PFS_file_class *klass,
         pfs->m_identity= (const void *)pfs;
 
         int res;
-        res= lf_hash_insert(&filename_hash, thread->m_filename_hash_pins,
+        res= lf_hash_insert(&pfs_filename_hash, thread->m_filename_hash_pins,
                             &pfs);
         if (likely(res == 0))
         {
@@ -1426,7 +1426,7 @@ void destroy_file(PFS_thread *thread, PFS_file *pfs)
   LF_PINS *pins= get_filename_hash_pins(thread);
   DBUG_ASSERT(pins != NULL);
 
-  lf_hash_delete(&filename_hash, pins,
+  lf_hash_delete(&pfs_filename_hash, pins,
                  pfs->m_filename, pfs->m_filename_length);
   if (klass->is_singleton())
     klass->m_singleton= NULL;
diff --git a/storage/perfschema/pfs_instr.h b/storage/perfschema/pfs_instr.h
index 81bc52d1d7590..a5ff3b4a17dfd 100644
--- a/storage/perfschema/pfs_instr.h
+++ b/storage/perfschema/pfs_instr.h
@@ -698,7 +698,7 @@ void update_socket_derived_flags();
 /** Update derived flags for all instruments. */
 void update_instruments_derived_flags();
 
-extern LF_HASH filename_hash;
+extern LF_HASH pfs_filename_hash;
 
 /** @} */
 #endif

From f83e2ecc501573cc500c2ee420e868a40b7910f2 Mon Sep 17 00:00:00 2001
From: Sergei Petrunia <psergey@askmonty.org>
Date: Tue, 23 Feb 2021 23:38:57 +0300
Subject: [PATCH 02/11] MDEV-24953: 10.5.9 crashes with large IN() list

The problem was in and_all_keys(), the code of MDEV-9759 which calculates
the new tree weight:

First, it didn't take into account the case when

(next->next_key_part=tmp) == NULL

and dereferenced a NULL pointer when getting tmp->weight.

Second, "if (param->alloced_sel_args > SEL_ARG::MAX_SEL_ARGS) break"
could leave the loop with incorrect value of weight.

Fixed by introducing SEL_ARG::update_weight_locally() and calling it
at the end of the function. This allows to avoid caring about all the
above cases.
---
 mysql-test/main/range_notembedded.result | 16 ++++++++-
 mysql-test/main/range_notembedded.test   | 32 ++++++++++++++++-
 sql/opt_range.cc                         | 46 +++++++++++++++---------
 sql/opt_range.h                          |  2 ++
 4 files changed, 77 insertions(+), 19 deletions(-)

diff --git a/mysql-test/main/range_notembedded.result b/mysql-test/main/range_notembedded.result
index 87fa85d3ac6ce..0ecf47c892ef9 100644
--- a/mysql-test/main/range_notembedded.result
+++ b/mysql-test/main/range_notembedded.result
@@ -159,7 +159,6 @@ left(@json, 2500)
     ]
 ]
 ## Repeat the above with a bit higher max_weight:
-set @tmp9750_weight=@@optimizer_max_sel_arg_weight;
 set optimizer_max_sel_arg_weight=120;
 explain select * from t1 where 
 kp1 in (1,2,3,4,5,6,7,8,9,10) and
@@ -225,3 +224,18 @@ SELECT *
 FROM mysql.help_relation ignore index (help_topic_id)
 WHERE (help_topic_id = 8 OR help_keyword_id = 0) AND help_keyword_id != 2 AND help_topic_id >= 1900;
 help_topic_id	help_keyword_id
+#
+# MDEV-24953: 10.5.9 crashes with large IN() list
+#
+CREATE TABLE t1 (
+notification_type_id smallint(4) unsigned NOT NULL DEFAULT 0,
+item_id int(10) unsigned NOT NULL DEFAULT 0,
+item_parent_id int(10) unsigned NOT NULL DEFAULT 0,
+user_id int(10) unsigned NOT NULL DEFAULT 0,
+PRIMARY KEY (notification_type_id,item_id,item_parent_id,user_id)
+);
+insert into t1 values (1,1,1,1), (2,2,2,2), (3,3,3,3);
+# Run crashing query
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	2	NULL	3	Using where
+drop table t1;
diff --git a/mysql-test/main/range_notembedded.test b/mysql-test/main/range_notembedded.test
index a70749ced6b55..5f6a05e8d9101 100644
--- a/mysql-test/main/range_notembedded.test
+++ b/mysql-test/main/range_notembedded.test
@@ -82,7 +82,6 @@ set @json= json_detailed(json_extract(@trace, '$**.setup_range_conditions'));
 select left(@json, 2500);
 
 --echo ## Repeat the above with a bit higher max_weight:
-set @tmp9750_weight=@@optimizer_max_sel_arg_weight;
 set optimizer_max_sel_arg_weight=120;
 explain select * from t1 where 
   kp1 in (1,2,3,4,5,6,7,8,9,10) and
@@ -110,3 +109,34 @@ SELECT *
 FROM mysql.help_relation ignore index (help_topic_id)
 WHERE (help_topic_id = 8 OR help_keyword_id = 0) AND help_keyword_id != 2 AND help_topic_id >= 1900;
 
+--echo #
+--echo # MDEV-24953: 10.5.9 crashes with large IN() list
+--echo #
+--source include/have_sequence.inc
+
+CREATE TABLE t1 (
+  notification_type_id smallint(4) unsigned NOT NULL DEFAULT 0,
+  item_id int(10) unsigned NOT NULL DEFAULT 0,
+  item_parent_id int(10) unsigned NOT NULL DEFAULT 0,
+  user_id int(10) unsigned NOT NULL DEFAULT 0,
+  PRIMARY KEY (notification_type_id,item_id,item_parent_id,user_id)
+);
+insert into t1 values (1,1,1,1), (2,2,2,2), (3,3,3,3);
+
+let $consts=`select group_concat(concat("'",seq,"'")) from seq_1_to_4642`;
+
+--echo # Run crashing query
+--disable_query_log
+eval
+explain
+DELETE FROM t1
+WHERE
+  notification_type_id IN (3, 4, 5, 6, 23)
+  AND
+  user_id = '5044'
+  AND
+  item_parent_id IN ($consts)
+;
+--enable_query_log
+
+drop table t1;
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 3684db40242f6..a02b6171a20e4 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -9800,7 +9800,6 @@ and_all_keys(RANGE_OPT_PARAM *param, SEL_ARG *key1, SEL_ARG *key2,
     key1->right= key1->left= &null_element;
     key1->next= key1->prev= 0;
   }
-  uint new_weight= 0;
 
   for (next=key1->first(); next ; next=next->next)
   {
@@ -9813,22 +9812,21 @@ and_all_keys(RANGE_OPT_PARAM *param, SEL_ARG *key1, SEL_ARG *key2,
 	continue;
       }
       next->next_key_part=tmp;
-      new_weight += 1 + tmp->weight;
       if (use_count)
 	next->increment_use_count(use_count);
       if (param->alloced_sel_args > SEL_ARG::MAX_SEL_ARGS)
         break;
     }
     else
-    {
-      new_weight += 1 + key2->weight;
       next->next_key_part=key2;
-    }
   }
   if (!key1)
     return &null_element;			// Impossible ranges
   key1->use_count++;
-  key1->weight= new_weight;
+
+  /* Re-compute the result tree's weight. */
+  key1->update_weight_locally();
+
   key1->max_part_no= MY_MAX(key2->max_part_no, key2->part+1);
   return key1;
 }
@@ -9992,6 +9990,30 @@ get_range(SEL_ARG **e1,SEL_ARG **e2,SEL_ARG *root1)
   return 0;
 }
 
+/*
+  @brief
+    Update the tree weight.
+
+  @detail
+    Utility function to be called on a SEL_ARG tree root after doing local
+    modifications concerning changes at this key part.
+    Assumes that the weight of the graphs connected via next_key_part is
+    up to dayte.
+*/
+void SEL_ARG::update_weight_locally()
+{
+  uint new_weight= 0;
+  const SEL_ARG *sl;
+  for (sl= first(); sl ; sl= sl->next)
+  {
+    new_weight++;
+    if (sl->next_key_part)
+      new_weight += sl->next_key_part->weight;
+  }
+  weight= new_weight;
+}
+
+
 #ifndef DBUG_OFF
 /*
   Verify SEL_TREE's weight.
@@ -10728,17 +10750,7 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
   key1->use_count++;
 
   /* Re-compute the result tree's weight. */
-  {
-    uint new_weight= 0;
-    const SEL_ARG *sl;
-    for (sl= key1->first(); sl ; sl= sl->next)
-    {
-      new_weight++;
-      if (sl->next_key_part)
-        new_weight += sl->next_key_part->weight;
-    }
-    key1->weight= new_weight;
-  }
+  key1->update_weight_locally();
 
   key1->max_part_no= max_part_no;
   return key1;
diff --git a/sql/opt_range.h b/sql/opt_range.h
index 50cd43c0e85ce..1014176ecc594 100644
--- a/sql/opt_range.h
+++ b/sql/opt_range.h
@@ -316,6 +316,8 @@ class SEL_ARG :public Sql_alloc
   */
   uint weight;
   enum { MAX_WEIGHT = 32000 };
+
+  void update_weight_locally();
 #ifndef DBUG_OFF
   uint verify_weight();
 #endif

From cea03285ecf79312ed70cbd00fbe4233c2ea040f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 24 Feb 2021 13:30:35 +0200
Subject: [PATCH 03/11] MDEV-24967 : Signal 11 on
 ha_innodb.cc::bg_wsrep_kill_trx line 18611

Null poiter reference in case where bf_thd has no trx .e.g. when
we have MDL-conflict.
---
 storage/innobase/handler/ha_innodb.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index d54987889fd70..f870cf7793b51 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -18688,10 +18688,12 @@ static void bg_wsrep_kill_trx(void *void_arg)
 		/* apparently victim trx was meanwhile rolled back.
 		tell bf thd not to wait, in case it already started to */
 		trx_t *trx= thd_to_trx(bf_thd);
-		if (lock_t *lock= trx->lock.wait_lock) {
-		  trx_mutex_enter(trx);
-		  lock_cancel_waiting_and_release(lock);
-		  trx_mutex_exit(trx);
+		/* note that bf_thd might not have trx e.g. in case of
+		MDL-conflict. */
+		if (lock_t *lock= (trx ? trx->lock.wait_lock : NULL)) {
+			trx_mutex_enter(trx);
+			lock_cancel_waiting_and_release(lock);
+			trx_mutex_exit(trx);
 		}
 		goto ret1;
 	}

From f2428b9c247a7c1d369a5b962108775eb191ec8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 24 Feb 2021 13:30:35 +0200
Subject: [PATCH 04/11] MDEV-24967 : Signal 11 on
 ha_innodb.cc::bg_wsrep_kill_trx line 18611

Null poiter reference in case where bf_thd has no trx .e.g. when
we have MDL-conflict.
---
 storage/innobase/handler/ha_innodb.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 7ca9e83a368fd..5746ac703f6e5 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -18598,10 +18598,12 @@ static void bg_wsrep_kill_trx(void *void_arg)
 		/* apparently victim trx was meanwhile rolled back.
 		tell bf thd not to wait, in case it already started to */
 		trx_t *trx= thd_to_trx(bf_thd);
-		if (lock_t *lock= trx->lock.wait_lock) {
-		  trx_mutex_enter(trx);
-		  lock_cancel_waiting_and_release(lock);
-		  trx_mutex_exit(trx);
+		/* note that bf_thd might not have trx e.g. in case of
+		MDL-conflict. */
+		if (lock_t *lock= (trx ? trx->lock.wait_lock : NULL)) {
+			trx_mutex_enter(trx);
+			lock_cancel_waiting_and_release(lock);
+			trx_mutex_exit(trx);
 		}
 		goto ret1;
 	}

From d1eeb4b83932660f8c8170a861734076bb6af546 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 24 Feb 2021 12:02:54 +0200
Subject: [PATCH 05/11] MDEV-24964 : Heap-buffer-overflow on wsrep_schema.cc
 ::remove_fragments

Problem was that we used heap allocated key using too small
array. Fixed by using dynamic memory allocation using actual
needed size.
---
 sql/wsrep_schema.cc | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/sql/wsrep_schema.cc b/sql/wsrep_schema.cc
index b2346aba508c1..e811d4e83171a 100644
--- a/sql/wsrep_schema.cc
+++ b/sql/wsrep_schema.cc
@@ -566,14 +566,24 @@ static int end_index_scan(TABLE* table) {
   return 0;
 }
 
-static void make_key(TABLE* table, uchar* key, key_part_map* map, int parts) {
+static void make_key(TABLE* table, uchar** key, key_part_map* map, int parts) {
   uint prefix_length= 0;
   KEY_PART_INFO* key_part= table->key_info->key_part;
+
   for (int i=0; i < parts; i++)
     prefix_length += key_part[i].store_length;
+
   *map= make_prev_keypart_map(parts);
-  key_copy(key, table->record[0], table->key_info, prefix_length);
+
+  if (!(*key= (uchar *) my_malloc(prefix_length + 1, MYF(MY_WME))))
+  {
+    WSREP_ERROR("Failed to allocate memory for key prefix_length %u", prefix_length);
+    assert(0);
+  }
+
+  key_copy(*key, table->record[0], table->key_info, prefix_length);
 }
+
 } /* namespace Wsrep_schema_impl */
 
 
@@ -958,7 +968,7 @@ int Wsrep_schema::update_fragment_meta(THD* thd,
 
   Wsrep_schema_impl::binlog_off binlog_off(thd);
   int error;
-  uchar key[MAX_KEY_LENGTH+MAX_FIELD_WIDTH];
+  uchar *key=NULL;
   key_part_map key_map= 0;
   TABLE* frag_table= 0;
 
@@ -973,7 +983,7 @@ int Wsrep_schema::update_fragment_meta(THD* thd,
   Wsrep_schema_impl::store(frag_table, 0, ws_meta.server_id());
   Wsrep_schema_impl::store(frag_table, 1, ws_meta.transaction_id().get());
   Wsrep_schema_impl::store(frag_table, 2, -1);
-  Wsrep_schema_impl::make_key(frag_table, key, &key_map, 3);
+  Wsrep_schema_impl::make_key(frag_table, &key, &key_map, 3);
 
   if ((error= Wsrep_schema_impl::init_for_index_scan(frag_table,
                                                      key, key_map)))
@@ -987,9 +997,11 @@ int Wsrep_schema::update_fragment_meta(THD* thd,
     }
     Wsrep_schema_impl::finish_stmt(thd);
     thd->lex->restore_backup_query_tables_list(&query_tables_list_backup);
+    my_free(key);
     DBUG_RETURN(1);
   }
 
+  my_free(key);
   /* Copy the original record to frag_table->record[1] */
   store_record(frag_table, record[1]);
 
@@ -1024,7 +1036,7 @@ static int remove_fragment(THD*                  thd,
               seqno.get());
   int ret= 0;
   int error;
-  uchar key[MAX_KEY_LENGTH+MAX_FIELD_WIDTH];
+  uchar *key= NULL;
   key_part_map key_map= 0;
 
   DBUG_ASSERT(server_id.is_undefined() == false);
@@ -1038,7 +1050,7 @@ static int remove_fragment(THD*                  thd,
   Wsrep_schema_impl::store(frag_table, 0, server_id);
   Wsrep_schema_impl::store(frag_table, 1, transaction_id.get());
   Wsrep_schema_impl::store(frag_table, 2, seqno.get());
-  Wsrep_schema_impl::make_key(frag_table, key, &key_map, 3);
+  Wsrep_schema_impl::make_key(frag_table, &key, &key_map, 3);
 
   if ((error= Wsrep_schema_impl::init_for_index_scan(frag_table,
                                                      key,
@@ -1060,6 +1072,8 @@ static int remove_fragment(THD*                  thd,
     ret= 1;
   }
 
+  if (key)
+    my_free(key);
   Wsrep_schema_impl::end_index_scan(frag_table);
   return ret;
 }
@@ -1147,7 +1161,7 @@ int Wsrep_schema::replay_transaction(THD* orig_thd,
   int ret= 1;
   int error;
   TABLE* frag_table= 0;
-  uchar key[MAX_KEY_LENGTH+MAX_FIELD_WIDTH];
+  uchar *key=NULL;
   key_part_map key_map= 0;
 
   for (std::vector<wsrep::seqno>::const_iterator i= fragments.begin();
@@ -1164,7 +1178,7 @@ int Wsrep_schema::replay_transaction(THD* orig_thd,
     Wsrep_schema_impl::store(frag_table, 0, ws_meta.server_id());
     Wsrep_schema_impl::store(frag_table, 1, ws_meta.transaction_id().get());
     Wsrep_schema_impl::store(frag_table, 2, i->get());
-    Wsrep_schema_impl::make_key(frag_table, key, &key_map, 3);
+    Wsrep_schema_impl::make_key(frag_table, &key, &key_map, 3);
 
     int error= Wsrep_schema_impl::init_for_index_scan(frag_table,
                                                       key,
@@ -1211,6 +1225,7 @@ int Wsrep_schema::replay_transaction(THD* orig_thd,
       Wsrep_schema_impl::finish_stmt(&thd);
       DBUG_RETURN(1);
     }
+
     error= Wsrep_schema_impl::init_for_index_scan(frag_table,
                                                   key,
                                                   key_map);
@@ -1224,6 +1239,7 @@ int Wsrep_schema::replay_transaction(THD* orig_thd,
     }
 
     error= Wsrep_schema_impl::delete_row(frag_table);
+
     if (error)
     {
       WSREP_WARN("Could not delete row from streaming log table: %d", error);
@@ -1233,8 +1249,12 @@ int Wsrep_schema::replay_transaction(THD* orig_thd,
     }
     Wsrep_schema_impl::end_index_scan(frag_table);
     Wsrep_schema_impl::finish_stmt(&thd);
+    my_free(key);
+    key= NULL;
   }
 
+  if (key)
+    my_free(key);
   DBUG_RETURN(ret);
 }
 

From bf6484e7bb4af3a3bc60289d86e4bde813f4e0c0 Mon Sep 17 00:00:00 2001
From: Igor Babaev <igor@askmonty.org>
Date: Wed, 24 Feb 2021 13:51:47 -0800
Subject: [PATCH 06/11] MDEV-24910 Crash with SELECT that uses table value
 constructor as a subselect

This bug caused crashes of the server when processing queries with table
value constructors (TVC) that contained subqueries and were used itself as
subselects. For such TVCs the following transformation is applied at the
prepare stage:
  VALUES (v1), ... (vn) => SELECT * FROM (VALUES (v1), ... (vn)) tvc_x.
This transformation allows to reduce the problem of evaluation of TVCs used
as subselects to the problem of evaluation of regular subselects.
The transformation is implemented in the wrap_tvc(). The code the function
to mimic the behaviour of the parser when processing the result of the
transformation. However this imitation was not free of some flaws. First
the function called the method exclude() that completely destroyed the
select tree structures below the transformed TVC. Second the function
used the procedure mysql_new_select to create st_select_lex nodes for
both wrapping select of the transformation and TVC. This also led to
constructing of invalid select tree structures.
The patch actually re-engineers the code of wrap_tvc().

Approved by Oleksandr Byelkin <sanja@mariadb.com>
---
 mysql-test/main/table_value_constr.result     | 110 +++++++++++++++++-
 mysql-test/main/table_value_constr.test       |  57 +++++++++
 .../compat/oracle/r/table_value_constr.result |   4 +-
 sql/sql_lex.cc                                |  23 ++++
 sql/sql_lex.h                                 |   1 +
 sql/sql_tvc.cc                                |  67 ++++++-----
 6 files changed, 231 insertions(+), 31 deletions(-)

diff --git a/mysql-test/main/table_value_constr.result b/mysql-test/main/table_value_constr.result
index e112aca78af79..603f21a772d39 100644
--- a/mysql-test/main/table_value_constr.result
+++ b/mysql-test/main/table_value_constr.result
@@ -748,7 +748,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 3	MATERIALIZED	<derived2>	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
-Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` semi join ((values (1)) `tvc_0`) where 1
+Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` semi join ((values (1)) `tvc_0`) where 1
 explain extended select * from t1
 where a in (select * from (values (1)) as tvc_0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -983,7 +983,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 3	MATERIALIZED	<derived2>	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
-Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` semi join ((values (1),(2)) `tvc_0`) where 1
+Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` semi join ((values (1),(2)) `tvc_0`) where 1
 explain extended select * from t1 
 where a = any (select * from (values (1),(2)) as tvc_0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -2775,4 +2775,110 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	3	Using where
 drop table t1;
+#
+# MDEV-24910: TVC containing subquery used as a subselect
+#
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (7), (1);
+create table t2 (b int) engine=myisam;
+insert into t2 values (1), (2);
+select (values ((select 2))) from t2;
+(values ((select 2)))
+2
+2
+explain select (values ((select 2))) from t2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	
+4	SUBQUERY	<derived2>	ALL	NULL	NULL	NULL	NULL	2	
+2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1249	Select 3 was reduced during optimization
+prepare stmt from "select (values ((select 2))) from t2";
+execute stmt;
+(values ((select 2)))
+2
+2
+execute stmt;
+(values ((select 2)))
+2
+2
+deallocate prepare stmt;
+select (values ((select * from t1 where a > 10))) from t2;
+(values ((select * from t1 where a > 10)))
+NULL
+NULL
+explain select (values ((select * from t1 where a > 10))) from t2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	
+4	SUBQUERY	<derived2>	ALL	NULL	NULL	NULL	NULL	2	
+2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+3	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	3	Using where
+prepare stmt from "select (values ((select * from t1 where a > 10))) from t2";
+execute stmt;
+(values ((select * from t1 where a > 10)))
+NULL
+NULL
+execute stmt;
+(values ((select * from t1 where a > 10)))
+NULL
+NULL
+deallocate prepare stmt;
+create table t3 (a int);
+insert into t3 values
+(3), (7), (7), (1), (3), (9), (7), (9), (8), (7), (8);
+create view v1 as select count(a) as c from t3 group by a;
+select
+(values ((select * from t3 where a in (select * from v1))));
+(values ((select * from t3 where a in (select * from v1))))
+1
+explain select
+(values ((select * from t3 where a in (select * from v1))));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+6	SUBQUERY	<derived2>	ALL	NULL	NULL	NULL	NULL	2	
+2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+3	SUBQUERY	t3	ALL	NULL	NULL	NULL	NULL	11	
+3	SUBQUERY	<subquery4>	eq_ref	distinct_key	distinct_key	8	func	1	Using where
+4	MATERIALIZED	<derived5>	ALL	NULL	NULL	NULL	NULL	11	
+5	DERIVED	t3	ALL	NULL	NULL	NULL	NULL	11	Using temporary; Using filesort
+prepare stmt from "select
+(values ((select * from t3 where a in (select * from v1))))";
+execute stmt;
+(values ((select * from t3 where a in (select * from v1))))
+1
+execute stmt;
+(values ((select * from t3 where a in (select * from v1))))
+1
+deallocate prepare stmt;
+select
+(values ((select * from t3
+where a > 10 and a in (select * from v1))));
+(values ((select * from t3
+where a > 10 and a in (select * from v1))))
+NULL
+explain select
+(values ((select * from t3
+where a > 10 and a in (select * from v1))));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+6	SUBQUERY	<derived2>	ALL	NULL	NULL	NULL	NULL	2	
+2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+3	SUBQUERY	t3	ALL	NULL	NULL	NULL	NULL	11	Using where
+3	SUBQUERY	<subquery4>	eq_ref	distinct_key	distinct_key	8	func	1	Using where
+4	MATERIALIZED	<derived5>	ALL	NULL	NULL	NULL	NULL	11	
+5	DERIVED	t3	ALL	NULL	NULL	NULL	NULL	11	Using temporary; Using filesort
+prepare stmt from "select
+(values ((select * from t3
+where a > 10 and a in (select * from v1))))";
+execute stmt;
+(values ((select * from t3
+where a > 10 and a in (select * from v1))))
+NULL
+execute stmt;
+(values ((select * from t3
+where a > 10 and a in (select * from v1))))
+NULL
+deallocate prepare stmt;
+drop view v1;
+drop table t1,t2,t3;
 End of 10.3 tests
diff --git a/mysql-test/main/table_value_constr.test b/mysql-test/main/table_value_constr.test
index 0a78fd9b386fd..2246a19d306f3 100644
--- a/mysql-test/main/table_value_constr.test
+++ b/mysql-test/main/table_value_constr.test
@@ -1459,4 +1459,61 @@ eval explain $q3;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-24910: TVC containing subquery used as a subselect
+--echo #
+
+create table t1 (a int) engine=myisam;
+insert into t1 values (3), (7), (1);
+create table t2 (b int) engine=myisam;
+insert into t2 values (1), (2);
+
+let $q1=
+select (values ((select 2))) from t2;
+eval $q1;
+eval explain $q1;
+eval prepare stmt from "$q1";
+execute stmt;
+execute stmt;
+deallocate prepare stmt;
+
+let $q2=
+select (values ((select * from t1 where a > 10))) from t2;
+eval $q2;
+eval explain $q2;
+eval prepare stmt from "$q2";
+execute stmt;
+execute stmt;
+deallocate prepare stmt;
+
+create table t3 (a int);
+insert into t3 values
+ (3), (7), (7), (1), (3), (9), (7), (9), (8), (7), (8);
+
+create view v1 as select count(a) as c from t3 group by a;
+
+let $q3=
+select
+(values ((select * from t3 where a in (select * from v1))));
+eval $q3;
+eval explain $q3;
+eval prepare stmt from "$q3";
+execute stmt;
+execute stmt;
+deallocate prepare stmt;
+
+let $q4=
+select
+(values ((select * from t3
+          where a > 10 and a in (select * from v1))));
+eval $q4;
+eval explain $q4;
+eval prepare stmt from "$q4";
+execute stmt;
+execute stmt;
+deallocate prepare stmt;
+
+drop view v1;
+drop table t1,t2,t3;
+
 --echo End of 10.3 tests
diff --git a/mysql-test/suite/compat/oracle/r/table_value_constr.result b/mysql-test/suite/compat/oracle/r/table_value_constr.result
index d4f8e28fe0787..539329554d5e5 100644
--- a/mysql-test/suite/compat/oracle/r/table_value_constr.result
+++ b/mysql-test/suite/compat/oracle/r/table_value_constr.result
@@ -746,7 +746,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 3	MATERIALIZED	<derived2>	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
-Note	1003	select "test"."t1"."a" AS "a","test"."t1"."b" AS "b" from "test"."t1" semi join ((values (1)) "tvc_0") where 1
+Note	1003	/* select#1 */ select "test"."t1"."a" AS "a","test"."t1"."b" AS "b" from "test"."t1" semi join ((values (1)) "tvc_0") where 1
 explain extended select * from t1
 where a in (select * from (values (1)) as tvc_0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -981,7 +981,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 3	MATERIALIZED	<derived2>	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
-Note	1003	select "test"."t1"."a" AS "a","test"."t1"."b" AS "b" from "test"."t1" semi join ((values (1),(2)) "tvc_0") where 1
+Note	1003	/* select#1 */ select "test"."t1"."a" AS "a","test"."t1"."b" AS "b" from "test"."t1" semi join ((values (1),(2)) "tvc_0") where 1
 explain extended select * from t1 
 where a = any (select * from (values (1),(2)) as tvc_0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index 9766a28757e1a..70d795c145d19 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -2464,9 +2464,32 @@ void st_select_lex_node::add_slave(st_select_lex_node *slave_arg)
   {
     slave= slave_arg;
     slave_arg->master= this;
+    slave->prev= &master->slave;
+    slave->next= 0;
   }
 }
 
+/*
+  @brief
+    Substitute this node in select tree for a newly creates node
+
+  @param  subst the node to substitute for
+
+  @details
+    The function substitute this node in the select tree for a newly
+    created node subst. This node is just removed from the tree but all
+    its link fields and the attached sub-tree remain untouched.
+*/
+
+void st_select_lex_node::substitute_in_tree(st_select_lex_node *subst)
+{
+  if ((subst->next= next))
+    next->prev= &subst->next;
+  subst->prev= prev;
+  (*prev)= subst;
+  subst->master= master;
+}
+
 
 /*
   include on level down (but do not link)
diff --git a/sql/sql_lex.h b/sql/sql_lex.h
index 77b4e15aaf03d..979e212c1f68c 100644
--- a/sql/sql_lex.h
+++ b/sql/sql_lex.h
@@ -711,6 +711,7 @@ class st_select_lex_node {
   void include_global(st_select_lex_node **plink);
   void exclude();
   void exclude_from_tree();
+  void substitute_in_tree(st_select_lex_node *subst);
 
   void set_slave(st_select_lex_node *slave_arg) { slave= slave_arg; }
   void move_node(st_select_lex_node *where_to_move)
diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc
index 0a5f6687e179c..0a771b592e4e1 100644
--- a/sql/sql_tvc.cc
+++ b/sql/sql_tvc.cc
@@ -654,44 +654,61 @@ st_select_lex *wrap_tvc(THD *thd, st_select_lex *tvc_sl,
 
   Query_arena backup;
   Query_arena *arena= thd->activate_stmt_arena_if_needed(&backup);
+
+  Item *item;
+  SELECT_LEX *wrapper_sl;
+  SELECT_LEX_UNIT *derived_unit;
+
   /*
-    Create SELECT_LEX of the select used in the result of transformation
+    Create SELECT_LEX wrapper_sl of the select used in the result
+    of the transformation
   */
-  lex->current_select= tvc_sl;
-  if (mysql_new_select(lex, 0, NULL))
+  if (!(wrapper_sl= new (thd->mem_root) SELECT_LEX()))
     goto err;
-  mysql_init_select(lex);
-  /* Create item list as '*' for the subquery SQ */
-  Item *item;
-  SELECT_LEX *wrapper_sl;
-  wrapper_sl= lex->current_select;
+  wrapper_sl->select_number= ++thd->lex->stmt_lex->current_select_number;
+  wrapper_sl->parent_lex= lex; /* Used in init_query. */
+  wrapper_sl->init_query();
+  wrapper_sl->init_select();
+
+  wrapper_sl->nest_level= tvc_sl->nest_level;
+  wrapper_sl->parsing_place= tvc_sl->parsing_place;
   wrapper_sl->linkage= tvc_sl->linkage;
-  wrapper_sl->parsing_place= SELECT_LIST;
+
+  lex->current_select= wrapper_sl;
   item= new (thd->mem_root) Item_field(thd, &wrapper_sl->context,
                                        NULL, NULL, &star_clex_str);
   if (item == NULL || add_item_to_list(thd, item))
     goto err;
   (wrapper_sl->with_wild)++;
-  
-  /* Exclude SELECT with TVC */
-  tvc_sl->exclude();
+
+  /* Include the newly created select into the global list of selects */
+  wrapper_sl->include_global((st_select_lex_node**)&lex->all_selects_list);
+
+  /* Substitute select node used of TVC for the newly created select */
+  tvc_sl->substitute_in_tree(wrapper_sl);
+
   /*
-    Create derived table DT that will wrap TVC in the result of transformation
+    Create a unit for the substituted select used for TVC and attach it
+    to the the wrapper select wrapper_sl as the only unit. The created
+    unit is the unit for the derived table tvc_x of the transformation.
   */
-  SELECT_LEX *tvc_select; // select for tvc
-  SELECT_LEX_UNIT *derived_unit; // unit for tvc_select
-  if (mysql_new_select(lex, 1, tvc_sl))
+  if (!(derived_unit= new (thd->mem_root) SELECT_LEX_UNIT()))
     goto err;
-  tvc_select= lex->current_select;
-  derived_unit= tvc_select->master_unit();
-  tvc_select->linkage= DERIVED_TABLE_TYPE;
+  derived_unit->init_query();
+  derived_unit->thd= thd;
+  derived_unit->include_down(wrapper_sl);
 
-  lex->current_select= wrapper_sl;
+  /*
+    Attach the select used of TVC as the only slave to the unit for
+    the derived table tvc_x of the transformation
+  */
+  derived_unit->add_slave(tvc_sl);
+  tvc_sl->linkage= DERIVED_TABLE_TYPE;
 
   /*
-    Create the name of the wrapping derived table and
-    add it to the FROM list of the wrapper
-   */
+    Generate the name of the derived table created for TVC and
+    add it to the FROM list of the wrapping select
+  */
   Table_ident *ti;
   LEX_CSTRING alias;
   TABLE_LIST *derived_tab;
@@ -710,10 +727,6 @@ st_select_lex *wrap_tvc(THD *thd, st_select_lex *tvc_sl,
   wrapper_sl->table_list.first->derived_type= DTYPE_TABLE | DTYPE_MATERIALIZE;
   lex->derived_tables|= DERIVED_SUBQUERY;
 
-  wrapper_sl->where= 0;
-  wrapper_sl->set_braces(false);
-  derived_unit->set_with_clause(0);
-
   if (arena)
     thd->restore_active_arena(arena, &backup);
   thd->lex->result= save_result;

From 1635686b509111c10cdb0842a0dabc0ef07bdf56 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Mon, 15 Feb 2021 12:31:31 +1100
Subject: [PATCH 07/11] MDEV-23510: arm64 lf_hash alignment of pointers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

volatile != atomic.

volatile has no memory barrier schemantics, its for mmaped IO
so lets allow some optimizer gains and stop pretending it helps
with memory atomicity.

The MDEV lists a SEGV an assumption is made that an address was
partially read. As C packs structs strictly in order and on arm64 the
cache line size is 128 bits. A pointer (link - 64 bits), followed
by a hashnr (uint32 - 32 bits), leaves the following key (uchar *
64 bits), neither naturally aligned to any pointer and worse, split
across a cache line which is the processors view of an atomic
reservation of memory.

lf_dynarray_lvalue is assumed to return a 64 bit aligned address.

As a solution move the 32bit hashnr to the end so we don't get the
*key pointer split across two cache lines.

Tested by: Krunal Bauskar
Reviewer: Marko Mäkelä
---
 mysys/lf_hash.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/mysys/lf_hash.c b/mysys/lf_hash.c
index 0dd0c1a94c620..a7553b349deb5 100644
--- a/mysys/lf_hash.c
+++ b/mysys/lf_hash.c
@@ -31,10 +31,10 @@
 
 /* An element of the list */
 typedef struct {
-  intptr volatile link; /* a pointer to the next element in a list and a flag */
-  uint32 hashnr;        /* reversed hash number, for sorting                 */
+  intptr link;   /* a pointer to the next element in a list and a flag */
   const uchar *key;
   size_t keylen;
+  uint32 hashnr; /* reversed hash number, for sorting */
   /*
     data is stored here, directly after the keylen.
     thus the pointer to data is (void*)(slist_element_ptr+1)
@@ -48,7 +48,7 @@ const int LF_HASH_OVERHEAD= sizeof(LF_SLIST);
   in a list) from l_find to l_insert/l_delete
 */
 typedef struct {
-  intptr volatile *prev;
+  intptr *prev;
   LF_SLIST *curr, *next;
 } CURSOR;
 
@@ -85,7 +85,7 @@ typedef struct {
              0 - ok
              1 - error (callbck returned 1)
 */
-static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
+static int l_find(LF_SLIST **head, CHARSET_INFO *cs, uint32 hashnr,
                  const uchar *key, uint keylen, CURSOR *cursor, LF_PINS *pins,
                  my_hash_walk_action callback)
 {
@@ -168,7 +168,7 @@ static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
     it uses pins[0..2], on return all pins are removed.
     if there're nodes with the same key value, a new node is added before them.
 */
-static LF_SLIST *l_insert(LF_SLIST * volatile *head, CHARSET_INFO *cs,
+static LF_SLIST *l_insert(LF_SLIST **head, CHARSET_INFO *cs,
                          LF_SLIST *node, LF_PINS *pins, uint flags)
 {
   CURSOR         cursor;
@@ -220,7 +220,7 @@ static LF_SLIST *l_insert(LF_SLIST * volatile *head, CHARSET_INFO *cs,
   NOTE
     it uses pins[0..2], on return all pins are removed.
 */
-static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
+static int l_delete(LF_SLIST **head, CHARSET_INFO *cs, uint32 hashnr,
                    const uchar *key, uint keylen, LF_PINS *pins)
 {
   CURSOR cursor;
@@ -278,7 +278,7 @@ static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
     it uses pins[0..2], on return the pin[2] keeps the node found
     all other pins are removed.
 */
-static LF_SLIST *l_search(LF_SLIST * volatile *head, CHARSET_INFO *cs,
+static LF_SLIST *l_search(LF_SLIST **head, CHARSET_INFO *cs,
                          uint32 hashnr, const uchar *key, uint keylen,
                          LF_PINS *pins)
 {
@@ -319,7 +319,7 @@ static inline my_hash_value_type calc_hash(CHARSET_INFO *cs,
 
 #define MAX_LOAD 1.0    /* average number of elements in a bucket */
 
-static int initialize_bucket(LF_HASH *, LF_SLIST * volatile*, uint, LF_PINS *);
+static int initialize_bucket(LF_HASH *, LF_SLIST **, uint, LF_PINS *);
 
 static void default_initializer(LF_HASH *hash, void *dst, const void *src)
 {
@@ -398,7 +398,7 @@ void lf_hash_destroy(LF_HASH *hash)
 int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data)
 {
   int csize, bucket, hashnr;
-  LF_SLIST *node, * volatile *el;
+  LF_SLIST *node, **el;
 
   node= (LF_SLIST *)lf_alloc_new(pins);
   if (unlikely(!node))
@@ -437,7 +437,7 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data)
 */
 int lf_hash_delete(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen)
 {
-  LF_SLIST * volatile *el;
+  LF_SLIST **el;
   uint bucket, hashnr;
 
   hashnr= hash->hash_function(hash->charset, (uchar *)key, keylen) & INT_MAX32;
@@ -473,7 +473,7 @@ void *lf_hash_search_using_hash_value(LF_HASH *hash, LF_PINS *pins,
                                       my_hash_value_type hashnr,
                                       const void *key, uint keylen)
 {
-  LF_SLIST * volatile *el, *found;
+  LF_SLIST **el, *found;
   uint bucket;
 
   /* hide OOM errors - if we cannot initialize a bucket, try the previous one */
@@ -507,7 +507,7 @@ int lf_hash_iterate(LF_HASH *hash, LF_PINS *pins,
   CURSOR cursor;
   uint bucket= 0;
   int res;
-  LF_SLIST * volatile *el;
+  LF_SLIST **el;
 
   el= lf_dynarray_lvalue(&hash->array, bucket);
   if (unlikely(!el))
@@ -539,13 +539,13 @@ static const uchar *dummy_key= (uchar*)"";
     0 - ok
    -1 - out of memory
 */
-static int initialize_bucket(LF_HASH *hash, LF_SLIST * volatile *node,
+static int initialize_bucket(LF_HASH *hash, LF_SLIST **node,
                               uint bucket, LF_PINS *pins)
 {
   uint parent= my_clear_highest_bit(bucket);
   LF_SLIST *dummy= (LF_SLIST *)my_malloc(sizeof(LF_SLIST), MYF(MY_WME));
   LF_SLIST **tmp= 0, *cur;
-  LF_SLIST * volatile *el= lf_dynarray_lvalue(&hash->array, parent);
+  LF_SLIST **el= lf_dynarray_lvalue(&hash->array, parent);
   if (unlikely(!el || !dummy))
     return -1;
   if (*el == NULL && bucket &&

From e0ba68ba34f5623cfa3c61e2e1966971703297f5 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Mon, 15 Feb 2021 12:31:31 +1100
Subject: [PATCH 08/11] MDEV-23510: arm64 lf_hash alignment of pointers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Like the 10.2 version 1635686b509111c10cdb0842a0dabc0ef07bdf56,
except C++ on internal functions for my_assume_aligned.

volatile != atomic.

volatile has no memory barrier schemantics, its for mmaped IO
so lets allow some optimizer gains and stop pretending it helps
with memory atomicity.

The MDEV lists a SEGV an assumption is made that an address was
partially read. As C packs structs strictly in order and on arm64 the
cache line size is 128 bits. A pointer (link - 64 bits), followed
by a hashnr (uint32 - 32 bits), leaves the following key (uchar *
64 bits), neither naturally aligned to any pointer and worse, split
across a cache line which is the processors view of an atomic
reservation of memory.

lf_dynarray_lvalue is assumed to return a 64 bit aligned address.

As a solution move the 32bit hashnr to the end so we don't get the
*key pointer split across two cache lines.

Tested by: Krunal Bauskar
Reviewer: Marko Mäkelä
---
 include/lf.h                    |  2 +-
 mysys/CMakeLists.txt            |  2 +-
 mysys/{lf_hash.c => lf_hash.cc} | 55 ++++++++++++++++++++-------------
 3 files changed, 35 insertions(+), 24 deletions(-)
 rename mysys/{lf_hash.c => lf_hash.cc} (90%)

diff --git a/include/lf.h b/include/lf.h
index 88ac644c349f0..267a66aeeaf2d 100644
--- a/include/lf.h
+++ b/include/lf.h
@@ -125,7 +125,7 @@ void *lf_alloc_new(LF_PINS *pins);
 C_MODE_END
 
 /*
-  extendible hash, lf_hash.c
+  extendible hash, lf_hash.cc
 */
 #include <hash.h>
 
diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt
index 6aab788f12c7b..d73d1c68b52e2 100644
--- a/mysys/CMakeLists.txt
+++ b/mysys/CMakeLists.txt
@@ -39,7 +39,7 @@ SET(MYSYS_SOURCES  array.c charset-def.c charset.c crc32ieee.cc my_default.c
                                 tree.c typelib.c base64.c my_memmem.c
                                 my_getpagesize.c
                                 guess_malloc_library.c
-				lf_alloc-pin.c lf_dynarray.c lf_hash.c
+				lf_alloc-pin.c lf_dynarray.c lf_hash.cc
                                 safemalloc.c my_new.cc
 				my_getncpus.c my_safehash.c my_chmod.c my_rnd.c
                                 my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c ../sql-common/my_time.c
diff --git a/mysys/lf_hash.c b/mysys/lf_hash.cc
similarity index 90%
rename from mysys/lf_hash.c
rename to mysys/lf_hash.cc
index fd005b4584ba1..bc3504bc0dd73 100644
--- a/mysys/lf_hash.c
+++ b/mysys/lf_hash.cc
@@ -28,13 +28,14 @@
 #include <my_bit.h>
 #include <lf.h>
 #include "my_cpu.h"
+#include "assume_aligned.h"
 
 /* An element of the list */
 typedef struct {
-  intptr volatile link; /* a pointer to the next element in a list and a flag */
-  uint32 hashnr;        /* reversed hash number, for sorting                 */
+  intptr link;   /* a pointer to the next element in a list and a flag */
   const uchar *key;
   size_t keylen;
+  uint32 hashnr; /* reversed hash number, for sorting */
   /*
     data is stored here, directly after the keylen.
     thus the pointer to data is (void*)(slist_element_ptr+1)
@@ -48,7 +49,7 @@ const int LF_HASH_OVERHEAD= sizeof(LF_SLIST);
   in a list) from l_find to l_insert/l_delete
 */
 typedef struct {
-  intptr volatile *prev;
+  intptr *prev;
   LF_SLIST *curr, *next;
 } CURSOR;
 
@@ -85,7 +86,7 @@ typedef struct {
              0 - ok
              1 - error (callbck returned 1)
 */
-static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
+static int l_find(LF_SLIST **head, CHARSET_INFO *cs, uint32 hashnr,
                  const uchar *key, size_t keylen, CURSOR *cursor, LF_PINS *pins,
                  my_hash_walk_action callback)
 {
@@ -98,11 +99,11 @@ static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
   DBUG_ASSERT(!keylen || !callback);    /* should not be set both */
 
 retry:
-  cursor->prev= (intptr *)head;
+  cursor->prev= (intptr *) my_assume_aligned<sizeof(intptr)>(head);
   do { /* PTR() isn't necessary below, head is a dummy node */
-    cursor->curr= (LF_SLIST *)(*cursor->prev);
+    cursor->curr= my_assume_aligned<sizeof(LF_SLIST *)>((LF_SLIST *)(*cursor->prev));
     lf_pin(pins, 1, cursor->curr);
-  } while (my_atomic_loadptr((void**)cursor->prev) != cursor->curr &&
+  } while (my_atomic_loadptr((void **)my_assume_aligned<sizeof(LF_SLIST *)>(cursor->prev)) != cursor->curr &&
            LF_BACKOFF());
   for (;;)
   {
@@ -111,11 +112,14 @@ static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
 
     cur_hashnr= cursor->curr->hashnr;
     cur_keylen= cursor->curr->keylen;
+    /* The key element needs to be aligned, not necessary what it points to */
+    my_assume_aligned<sizeof(const uchar *)>(&cursor->curr->key);
     cur_key= cursor->curr->key;
 
     do {
+      /* attempting to my_assume_aligned onlink below broke the implementation */
       link= cursor->curr->link;
-      cursor->next= PTR(link);
+      cursor->next= my_assume_aligned<sizeof(LF_SLIST *)>(PTR(link));
       lf_pin(pins, 0, cursor->next);
     } while (link != cursor->curr->link && LF_BACKOFF());
 
@@ -155,6 +159,10 @@ static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
   }
 }
 
+
+/* static l_find is the only user my_assume_aligned, keep the rest as c scoped */
+C_MODE_START
+
 /*
   DESCRIPTION
     insert a 'node' in the list that starts from 'head' in the correct
@@ -168,7 +176,7 @@ static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
     it uses pins[0..2], on return all pins are removed.
     if there're nodes with the same key value, a new node is added before them.
 */
-static LF_SLIST *l_insert(LF_SLIST * volatile *head, CHARSET_INFO *cs,
+static LF_SLIST *l_insert(LF_SLIST **head, CHARSET_INFO *cs,
                          LF_SLIST *node, LF_PINS *pins, uint flags)
 {
   CURSOR         cursor;
@@ -220,7 +228,7 @@ static LF_SLIST *l_insert(LF_SLIST * volatile *head, CHARSET_INFO *cs,
   NOTE
     it uses pins[0..2], on return all pins are removed.
 */
-static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
+static int l_delete(LF_SLIST **head, CHARSET_INFO *cs, uint32 hashnr,
                    const uchar *key, uint keylen, LF_PINS *pins)
 {
   CURSOR cursor;
@@ -278,7 +286,7 @@ static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
     it uses pins[0..2], on return the pin[2] keeps the node found
     all other pins are removed.
 */
-static LF_SLIST *l_search(LF_SLIST * volatile *head, CHARSET_INFO *cs,
+static LF_SLIST *l_search(LF_SLIST **head, CHARSET_INFO *cs,
                          uint32 hashnr, const uchar *key, uint keylen,
                          LF_PINS *pins)
 {
@@ -319,13 +327,14 @@ static inline my_hash_value_type calc_hash(CHARSET_INFO *cs,
 
 #define MAX_LOAD 1.0    /* average number of elements in a bucket */
 
-static int initialize_bucket(LF_HASH *, LF_SLIST * volatile*, uint, LF_PINS *);
+static int initialize_bucket(LF_HASH *, LF_SLIST **, uint, LF_PINS *);
 
 static void default_initializer(LF_HASH *hash, void *dst, const void *src)
 {
   memcpy(dst, src, hash->element_size);
 }
 
+
 /*
   Initializes lf_hash, the arguments are compatible with hash_init
 
@@ -398,7 +407,7 @@ void lf_hash_destroy(LF_HASH *hash)
 int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data)
 {
   int csize, bucket, hashnr;
-  LF_SLIST *node, * volatile *el;
+  LF_SLIST *node, **el;
 
   node= (LF_SLIST *)lf_alloc_new(pins);
   if (unlikely(!node))
@@ -407,7 +416,7 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data)
   node->key= hash_key(hash, (uchar *)(node+1), &node->keylen);
   hashnr= hash->hash_function(hash->charset, node->key, node->keylen) & INT_MAX32;
   bucket= hashnr % hash->size;
-  el= lf_dynarray_lvalue(&hash->array, bucket);
+  el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket);
   if (unlikely(!el))
     return -1;
   if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins)))
@@ -437,7 +446,7 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data)
 */
 int lf_hash_delete(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen)
 {
-  LF_SLIST * volatile *el;
+  LF_SLIST **el;
   uint bucket, hashnr;
 
   hashnr= hash->hash_function(hash->charset, (uchar *)key, keylen) & INT_MAX32;
@@ -445,7 +454,7 @@ int lf_hash_delete(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen)
   /* hide OOM errors - if we cannot initialize a bucket, try the previous one */
   for (bucket= hashnr % hash->size; ;bucket= my_clear_highest_bit(bucket))
   {
-    el= lf_dynarray_lvalue(&hash->array, bucket);
+    el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket);
     if (el && (*el || initialize_bucket(hash, el, bucket, pins) == 0))
       break;
     if (unlikely(bucket == 0))
@@ -473,13 +482,13 @@ void *lf_hash_search_using_hash_value(LF_HASH *hash, LF_PINS *pins,
                                       my_hash_value_type hashnr,
                                       const void *key, uint keylen)
 {
-  LF_SLIST * volatile *el, *found;
+  LF_SLIST **el, *found;
   uint bucket;
 
   /* hide OOM errors - if we cannot initialize a bucket, try the previous one */
   for (bucket= hashnr % hash->size; ;bucket= my_clear_highest_bit(bucket))
   {
-    el= lf_dynarray_lvalue(&hash->array, bucket);
+    el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket);
     if (el && (*el || initialize_bucket(hash, el, bucket, pins) == 0))
       break;
     if (unlikely(bucket == 0))
@@ -507,9 +516,9 @@ int lf_hash_iterate(LF_HASH *hash, LF_PINS *pins,
   CURSOR cursor;
   uint bucket= 0;
   int res;
-  LF_SLIST * volatile *el;
+  LF_SLIST **el;
 
-  el= lf_dynarray_lvalue(&hash->array, bucket);
+  el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket);
   if (unlikely(!el))
     return 0; /* if there's no bucket==0, the hash is empty */
   if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins)))
@@ -539,14 +548,14 @@ static const uchar *dummy_key= (uchar*)"";
     0 - ok
    -1 - out of memory
 */
-static int initialize_bucket(LF_HASH *hash, LF_SLIST * volatile *node,
+static int initialize_bucket(LF_HASH *hash, LF_SLIST **node,
                               uint bucket, LF_PINS *pins)
 {
   uint parent= my_clear_highest_bit(bucket);
   LF_SLIST *dummy= (LF_SLIST *)my_malloc(key_memory_lf_slist,
                                          sizeof(LF_SLIST), MYF(MY_WME));
   LF_SLIST **tmp= 0, *cur;
-  LF_SLIST * volatile *el= lf_dynarray_lvalue(&hash->array, parent);
+  LF_SLIST **el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, parent);
   if (unlikely(!el || !dummy))
     return -1;
   if (*el == NULL && bucket &&
@@ -574,3 +583,5 @@ static int initialize_bucket(LF_HASH *hash, LF_SLIST * volatile *node,
   */
   return 0;
 }
+
+C_MODE_END

From 577c970c520852b7bd9ad0b7e38d1883760b79bd Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Fri, 29 Jan 2021 09:37:22 +1100
Subject: [PATCH 09/11] MDEV-24728: Debian include client caching_sha2_password
 plugin

Backport of 4bc31a904f22

Include client libraries for auth caching_sha2_password and
sha256_password in the libmariadb3 client library package.
---
 debian/libmariadb3.install | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install
index 065adc67a91c7..faeeb3026d63f 100644
--- a/debian/libmariadb3.install
+++ b/debian/libmariadb3.install
@@ -4,3 +4,5 @@ usr/lib/*/libmariadb.so.*
 usr/lib/mysql/plugin/dialog.so
 usr/lib/mysql/plugin/mysql_clear_password.so
 usr/lib/mysql/plugin/client_ed25519.so
+usr/lib/mysql/plugin/sha256_password.so
+usr/lib/mysql/plugin/caching_sha2_password.so

From 48b5f8a5444af05c460b6142b0574b789a4e7882 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Thu, 25 Feb 2021 15:44:45 +1100
Subject: [PATCH 10/11] mysys: lf_hash - fix l_search size_t keylen

Correcting an incorrect merge from 10.2
---
 mysys/lf_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mysys/lf_hash.c b/mysys/lf_hash.c
index c8d80f377e5d5..911d7228c0e11 100644
--- a/mysys/lf_hash.c
+++ b/mysys/lf_hash.c
@@ -86,7 +86,7 @@ typedef struct {
              1 - error (callbck returned 1)
 */
 static int l_find(LF_SLIST **head, CHARSET_INFO *cs, uint32 hashnr,
-                 const uchar *key, uint keylen, CURSOR *cursor, LF_PINS *pins,
+                 const uchar *key, size_t keylen, CURSOR *cursor, LF_PINS *pins,
                  my_hash_walk_action callback)
 {
   uint32       cur_hashnr;

From 0a95c922c8a2c1625ec6756b1e5c523e1cc4f73d Mon Sep 17 00:00:00 2001
From: Varun Gupta <varun.gupta@mariadb.com>
Date: Thu, 25 Feb 2021 12:42:01 +0530
Subject: [PATCH 11/11] Fixed the innodb_ext_key test by adding replace_column

---
 mysql-test/r/innodb_ext_key.result | 2 +-
 mysql-test/t/innodb_ext_key.test   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/innodb_ext_key.result b/mysql-test/r/innodb_ext_key.result
index 880d7a8bceb68..76e6cd000f947 100644
--- a/mysql-test/r/innodb_ext_key.result
+++ b/mysql-test/r/innodb_ext_key.result
@@ -804,5 +804,5 @@ PRIMARY KEY(pk)
 INSERT INTO t2 SELECT a,a FROM t1;
 EXPLAIN SELECT pk FROM t2 FORCE INDEX(k1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	index	NULL	k1	23	NULL	10	Using index
+1	SIMPLE	t2	index	NULL	k1	23	NULL	#	Using index
 DROP TABLE t1,t2;
diff --git a/mysql-test/t/innodb_ext_key.test b/mysql-test/t/innodb_ext_key.test
index de9ca10ff1bfb..04ebcc1e41ba5 100644
--- a/mysql-test/t/innodb_ext_key.test
+++ b/mysql-test/t/innodb_ext_key.test
@@ -622,6 +622,7 @@ CREATE TABLE t2 (
 )ENGINE=INNODB;
 
 INSERT INTO t2 SELECT a,a FROM t1;
+--replace_column 9 #
 EXPLAIN SELECT pk FROM t2 FORCE INDEX(k1);
 
 DROP TABLE t1,t2;