MDEV-35206: Assertion failure in JOIN::dbug_verify_sj_inner_tables

bsrikanth-mariadb · bsrikanth-mariadb · commit 8be6d2540baa · 2025-10-20T09:56:05.000+05:30
A nested select query is crashing in when optimizer_join_limit_pref_ratio=10
and optimizer_search_depth=1 due to an assertion failure in
JOIN::dbug_verify_sj_inner_tables().

In sql_select.cc#choose_plan(), there are 2 back-2-back calls to
greedy_search(). The first one is invoked to build a join plan
that can short-cut ORDER BY...LIMIT, while the second invocation
to not consider short-cut.

The greedy_search() should start with a value of join-&gt;cur_sj_inner_tables
set to 0. However, the first greedy_search() call left the value of
join-&gt;cur_sj_inner_tables to "6". This caused the assert to fail in
dbug_verify_sj_inner_tables() as soon as the second greedy_search() started,
where in it was expecting a value of 0.

Similar problem is noticed with cur_embedding_map in the case of nested
joins, and nested_join counter.

introduced init_join_plan_search_state() which is called
from start of greedy_search() and optimize_straight join
that does all the needed initialization by :
  1. setting 0's to cur_sj_inner_tables, and cur_embedding_map
  2. invoking reset_nj_counters().
diff --git a/mysql-test/main/join_nested.result b/mysql-test/main/join_nested.result
@@ -2062,4 +2062,67 @@ LEFT JOIN  (t1 a LEFT JOIN t1 b ON t1.i = b.i)  ON c.i = t1.i);
 1
 1
 DROP TABLE t1;
+#
+# MDEV-35206: Assertion in JOIN::dbug_verify_sj_inner_tables
+#
+SET @save_optimizer_join_limit_pref_ratio= @@optimizer_join_limit_pref_ratio;
+SET @save_optimizer_search_depth= @@optimizer_search_depth;
+CREATE TABLE t1 (c1 VARCHAR(64) DEFAULT NULL, c2 VARCHAR(8) DEFAULT NULL);
+INSERT INTO t1 (c1) values ('one');
+INSERT INTO t1 (c2) values ('2');
+SET optimizer_join_limit_pref_ratio=10;
+SET optimizer_search_depth=1;
+SELECT 
+c1 
+FROM 
+t1
+WHERE 
+c2 IN (SELECT c2
+FROM t1
+WHERE c1 IN (SELECT c1 
+FROM t1
+WHERE c1 IN (NULL)
+)
+)
+ORDER BY c1 LIMIT 1;
+c1
+DROP TABLE t1;
+#
+# similar issue with join::cur_embedding_map
+#
+CREATE TABLE t10 (a int, b int, index(b));
+INSERT INTO t10 SELECT seq, seq FROM seq_1_to_10;
+CREATE TABLE t11(a int, b int);
+CREATE TABLE t12(a int, b int, index(b));
+INSERT INTO t11 select seq, seq FROM seq_1_to_20;
+INSERT INTO t12 select seq, seq FROM seq_1_to_40;
+CREATE TABLE t13(a int, b int);
+CREATE TABLE t14(a int, b int, index(b));
+INSERT INTO t13 select seq, seq FROM seq_1_to_20;
+INSERT INTO t14 select seq, seq FROM seq_1_to_40;
+ANALYZE TABLE t10, t11, t12;
+Table	Op	Msg_type	Msg_text
+test.t10	analyze	status	Engine-independent statistics collected
+test.t10	analyze	status	Table is already up to date
+test.t11	analyze	status	Engine-independent statistics collected
+test.t11	analyze	status	OK
+test.t12	analyze	status	Engine-independent statistics collected
+test.t12	analyze	status	Table is already up to date
+EXPLAIN SELECT *
+FROM
+t10 LEFT JOIN 
+(
+t11 JOIN t12 ON t11.b=t12.b 
+left join (t13 join t14 on t13.b=t14.b) on t13.a=t11.a
+) ON t10.a=t11.a
+ORDER BY t10.b LIMIT 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t10	ALL	NULL	NULL	NULL	NULL	10	Using temporary; Using filesort
+1	SIMPLE	t11	ALL	NULL	NULL	NULL	NULL	20	Using where
+1	SIMPLE	t12	ref	b	b	5	test.t11.b	1	
+1	SIMPLE	t13	ALL	NULL	NULL	NULL	NULL	20	Using where
+1	SIMPLE	t14	ref	b	b	5	test.t13.b	1	
+DROP TABLE t10, t11, t12, t13, t14;
+SET optimizer_join_limit_pref_ratio= @save_optimizer_join_limit_pref_ratio;
+SET optimizer_search_depth= @save_optimizer_search_depth;
 # end of 10.11 tests
diff --git a/mysql-test/main/join_nested.test b/mysql-test/main/join_nested.test
@@ -2,6 +2,7 @@
 --disable_warnings
 DROP TABLE IF EXISTS t0,t1,t2,t3,t4,t5,t6,t7,t8,t9;
 --enable_warnings
+--source include/have_sequence.inc
 
 SET @save_optimizer_switch=@@optimizer_switch;
 SET optimizer_switch=ifnull(@optimizer_switch_for_join_nested_test,'outer_join_with_cache=off');
@@ -1470,4 +1471,69 @@ SELECT 1 FROM  t1 WHERE i IN
     LEFT JOIN  (t1 a LEFT JOIN t1 b ON t1.i = b.i)  ON c.i = t1.i);
 
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-35206: Assertion in JOIN::dbug_verify_sj_inner_tables
+--echo #
+
+SET @save_optimizer_join_limit_pref_ratio= @@optimizer_join_limit_pref_ratio;
+SET @save_optimizer_search_depth= @@optimizer_search_depth;
+
+CREATE TABLE t1 (c1 VARCHAR(64) DEFAULT NULL, c2 VARCHAR(8) DEFAULT NULL);
+INSERT INTO t1 (c1) values ('one');
+INSERT INTO t1 (c2) values ('2');
+
+SET optimizer_join_limit_pref_ratio=10;
+SET optimizer_search_depth=1;
+
+SELECT 
+  c1 
+FROM 
+  t1
+WHERE 
+  c2 IN (SELECT c2
+          FROM t1
+          WHERE c1 IN (SELECT c1 
+                        FROM t1
+                        WHERE c1 IN (NULL)
+                      )
+        )
+ORDER BY c1 LIMIT 1;
+
+DROP TABLE t1;
+
+--echo #
+--echo # similar issue with join::cur_embedding_map
+--echo #
+CREATE TABLE t10 (a int, b int, index(b));
+INSERT INTO t10 SELECT seq, seq FROM seq_1_to_10;
+
+CREATE TABLE t11(a int, b int);
+CREATE TABLE t12(a int, b int, index(b));
+
+INSERT INTO t11 select seq, seq FROM seq_1_to_20;
+INSERT INTO t12 select seq, seq FROM seq_1_to_40;
+
+CREATE TABLE t13(a int, b int);
+CREATE TABLE t14(a int, b int, index(b));
+
+INSERT INTO t13 select seq, seq FROM seq_1_to_20;
+INSERT INTO t14 select seq, seq FROM seq_1_to_40;
+
+ANALYZE TABLE t10, t11, t12;
+
+EXPLAIN SELECT *
+FROM
+  t10 LEFT JOIN 
+  (
+    t11 JOIN t12 ON t11.b=t12.b 
+     left join (t13 join t14 on t13.b=t14.b) on t13.a=t11.a
+  ) ON t10.a=t11.a
+ORDER BY t10.b LIMIT 1;
+
+DROP TABLE t10, t11, t12, t13, t14;
+
+SET optimizer_join_limit_pref_ratio= @save_optimizer_join_limit_pref_ratio;
+SET optimizer_search_depth= @save_optimizer_search_depth;
+
 --echo # end of 10.11 tests
diff --git a/mysql-test/main/join_nested_jcl6.result b/mysql-test/main/join_nested_jcl6.result
@@ -2071,6 +2071,69 @@ LEFT JOIN  (t1 a LEFT JOIN t1 b ON t1.i = b.i)  ON c.i = t1.i);
 1
 1
 DROP TABLE t1;
+#
+# MDEV-35206: Assertion in JOIN::dbug_verify_sj_inner_tables
+#
+SET @save_optimizer_join_limit_pref_ratio= @@optimizer_join_limit_pref_ratio;
+SET @save_optimizer_search_depth= @@optimizer_search_depth;
+CREATE TABLE t1 (c1 VARCHAR(64) DEFAULT NULL, c2 VARCHAR(8) DEFAULT NULL);
+INSERT INTO t1 (c1) values ('one');
+INSERT INTO t1 (c2) values ('2');
+SET optimizer_join_limit_pref_ratio=10;
+SET optimizer_search_depth=1;
+SELECT 
+c1 
+FROM 
+t1
+WHERE 
+c2 IN (SELECT c2
+FROM t1
+WHERE c1 IN (SELECT c1 
+FROM t1
+WHERE c1 IN (NULL)
+)
+)
+ORDER BY c1 LIMIT 1;
+c1
+DROP TABLE t1;
+#
+# similar issue with join::cur_embedding_map
+#
+CREATE TABLE t10 (a int, b int, index(b));
+INSERT INTO t10 SELECT seq, seq FROM seq_1_to_10;
+CREATE TABLE t11(a int, b int);
+CREATE TABLE t12(a int, b int, index(b));
+INSERT INTO t11 select seq, seq FROM seq_1_to_20;
+INSERT INTO t12 select seq, seq FROM seq_1_to_40;
+CREATE TABLE t13(a int, b int);
+CREATE TABLE t14(a int, b int, index(b));
+INSERT INTO t13 select seq, seq FROM seq_1_to_20;
+INSERT INTO t14 select seq, seq FROM seq_1_to_40;
+ANALYZE TABLE t10, t11, t12;
+Table	Op	Msg_type	Msg_text
+test.t10	analyze	status	Engine-independent statistics collected
+test.t10	analyze	status	Table is already up to date
+test.t11	analyze	status	Engine-independent statistics collected
+test.t11	analyze	status	OK
+test.t12	analyze	status	Engine-independent statistics collected
+test.t12	analyze	status	Table is already up to date
+EXPLAIN SELECT *
+FROM
+t10 LEFT JOIN 
+(
+t11 JOIN t12 ON t11.b=t12.b 
+left join (t13 join t14 on t13.b=t14.b) on t13.a=t11.a
+) ON t10.a=t11.a
+ORDER BY t10.b LIMIT 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t10	ALL	NULL	NULL	NULL	NULL	10	Using temporary; Using filesort
+1	SIMPLE	t11	hash_ALL	NULL	#hash#$hj	5	test.t10.a	20	Using where; Using join buffer (flat, BNLH join)
+1	SIMPLE	t12	ref	b	b	5	test.t11.b	1	Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan
+1	SIMPLE	t13	hash_ALL	NULL	#hash#$hj	5	test.t10.a	20	Using where; Using join buffer (incremental, BNLH join)
+1	SIMPLE	t14	ref	b	b	5	test.t13.b	1	Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan
+DROP TABLE t10, t11, t12, t13, t14;
+SET optimizer_join_limit_pref_ratio= @save_optimizer_join_limit_pref_ratio;
+SET optimizer_search_depth= @save_optimizer_search_depth;
 # end of 10.11 tests
 CREATE TABLE t5 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b));
 CREATE TABLE t6 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b));
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
@@ -342,6 +342,7 @@ static
 void compute_part_of_sort_key_for_equals(JOIN *join, TABLE *table,
                                          Item_field *item_field,
                                          key_map *col_keys);
+static void init_join_plan_search_state(JOIN *join);
 
 #ifndef DBUG_OFF
 
@@ -9396,11 +9397,9 @@ choose_plan(JOIN *join, table_map join_tables)
   DBUG_ENTER("choose_plan");
 
   join->limit_optimization_mode= false;
-  join->cur_embedding_map= 0;
   join->extra_heuristic_pruning= false;
   join->prune_level= join->thd->variables.optimizer_prune_level;
 
-  reset_nj_counters(join, join->join_list);
   qsort_cmp2 jtab_sort_func;
 
   if (join->emb_sjm_nest)
@@ -9440,11 +9439,6 @@ choose_plan(JOIN *join, table_map join_tables)
   {
     choose_initial_table_order(join);
   }
-  /*
-    Note: constant tables are already in the join prefix. We don't
-    put them into the cur_sj_inner_tables, though.
-  */
-  join->cur_sj_inner_tables= 0;
 
   if (straight_join)
   {
@@ -9781,6 +9775,8 @@ optimize_straight_join(JOIN *join, table_map remaining_tables)
   POSITION  loose_scan_pos;
   THD *thd= join->thd;
 
+  init_join_plan_search_state(join);
+
   for (JOIN_TAB **pos= join->best_ref + idx ; (s= *pos) ; pos++)
   {
     POSITION *position= join->positions + idx;
@@ -9926,6 +9922,8 @@ greedy_search(JOIN      *join,
   uint      n_tables __attribute__((unused));
   DBUG_ENTER("greedy_search");
 
+  init_join_plan_search_state(join);
+
   /* number of tables that remain to be optimized */
   usable_tables= (join->emb_sjm_nest ?
                   (join->emb_sjm_nest->sj_inner_tables &
@@ -18993,6 +18991,7 @@ static bool check_interleaving_with_nj(JOIN_TAB *next_tab)
     if (!next_emb->sj_on_expr)
     {
       next_emb->nested_join->counter++;
+      DBUG_ASSERT(next_emb->nested_join->counter <= next_emb->nested_join->n_tables);
       if (next_emb->nested_join->counter == 1)
       {
         /* 
@@ -33073,6 +33072,12 @@ bool JOIN::transform_all_conds_and_on_exprs_in_join_list(
   return false;
 }
 
+static void init_join_plan_search_state(JOIN *join)
+{
+  join->cur_sj_inner_tables= 0;
+  join->cur_embedding_map= 0;
+  reset_nj_counters(join, join->join_list);
+}
 
 /**
   @} (end of group Query_Optimizer)
