Bug#28957660 Regression: Crash in select_lex::flatten_subqueries

Followup patch: Ensure that semi-join transformation sequence is deterministic.

Sorting of semi-join transformation candidate objects in
SELECT_LEX::flatten_subqueries was not deterministic and could vary
from platform to platform. The sort criterion has been improved by
adding position in the candidate array as the last criterion, thus
making sorting fully deterministic.

Author: roylyseng

mysql-test/r/subquery_exists.result

@@ -486,7 +486,7 @@ EXISTS (SELECT * FROM t1) AND
 EXISTS (SELECT * FROM t1);
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
-3	SUBQUERY	t1	NULL	ALL	NULL	NULL	NULL	NULL	1	100.00	NULL
+32	SUBQUERY	t1	NULL	ALL	NULL	NULL	NULL	NULL	1	100.00	NULL
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) semi join (`test`.`t1`) where 0
 SELECT * FROM t1

sql/sql_const.h

@@ -106,8 +106,10 @@
   element, such as a random function or a non-deterministic function.
   Expressions containing this bit cannot be evaluated once and then cached,
   they must be evaluated at latest possible point.
+  MAX_TABLES_FOR_SIZE adds the pseudo bits and is used for sizing purposes only.
 */
-#define MAX_TABLES (sizeof(table_map) * 8 - 3) /* Max tables in join */
+#define MAX_TABLES_FOR_SIZE (sizeof(table_map) * 8)  ///< Use for sizing ONLY
+#define MAX_TABLES (MAX_TABLES_FOR_SIZE - 3)         ///< Max tables in join
 #define INNER_TABLE_BIT (((table_map)1) << (MAX_TABLES + 0))
 #define OUTER_REF_TABLE_BIT (((table_map)1) << (MAX_TABLES + 1))
 #define RAND_TABLE_BIT (((table_map)1) << (MAX_TABLES + 2))

sql/sql_resolver.cc

@@ -2996,7 +2996,11 @@ static bool replace_subcondition(THD *thd, Item **tree, Item *old_cond,
     temporary table.
     We deal with this problem by flattening children's subqueries first and
     then using a heuristic rule to determine each subquery predicate's
-    "priority".
+    priority, which is calculated in this order:
+
+    1. Prefer dependent subqueries over non-dependent ones
+    2. Prefer subqueries with many tables over those with fewer tables
+    3. Prefer early subqueries over later ones (to make sort deterministic)
 
   @returns false if success, true if error
 */
@@ -3031,7 +3035,8 @@ bool SELECT_LEX::flatten_subqueries(THD *thd) {
     there are cases where flattening is not possible and only the parent can
     know.
    */
-  for (subq = subq_begin; subq < subq_end; subq++) {
+  uint subq_no;
+  for (subq = subq_begin, subq_no = 0; subq < subq_end; subq++, subq_no++) {
     // Transformation of IN and EXISTS subqueries is supported
     DBUG_ASSERT((*subq)->substype() == Item_subselect::IN_SUBS ||
                 (*subq)->substype() == Item_subselect::EXISTS_SUBS);
@@ -3041,10 +3046,16 @@ bool SELECT_LEX::flatten_subqueries(THD *thd) {
     // Check that we proceeded bottom-up
     DBUG_ASSERT(child_select->sj_candidates == NULL);
 
+    bool dependent = (*subq)->unit->uncacheable & UNCACHEABLE_DEPENDENT;
     (*subq)->sj_convert_priority =
-        (((*subq)->unit->uncacheable & UNCACHEABLE_DEPENDENT) ? MAX_TABLES
-                                                              : 0) +
-        child_select->leaf_table_count;
+        (((dependent * MAX_TABLES_FOR_SIZE) +  // dependent subqueries first
+          child_select->leaf_table_count) *
+         65536) +           // then with many tables
+        (65536 - subq_no);  // then based on position
+    /*
+      We may actually allocate more than 64k subqueries in a query block,
+      but this is so unlikely that we ignore the impact it may have on sorting.
+    */
   }
 
   /*