MDEV-28871: Assert ... failed in JOIN::dbug_verify_sj_inner_tables...

optimize_semi_joins() calls update_sj_state() to update semi-join
optimization state in the JOIN class.

greedy_search() algorithm considers different join prefixes,
and then picks one table to put into the join prefix.
Most of the semi-join optimization state is in the table's entry
in the join->positions[cur_prefix_size].

However, it also needs to call update_sj_state() to update the
semi-join optimization state in the JOIN class.

There is one exception, which is the cause of this bug: when we're
inside optimize_semi_join_nests() and are optimizing a subquery,
optimize_semi_joins() does nothing, it doesn't call update_sj_state().

greedy_search() must not do that either.

mysql-test/main/subselect_sj.result

@@ -3327,4 +3327,17 @@ ERROR HY000: Illegal parameter data types geometry and int for operation '='
 EXECUTE stmt;
 ERROR HY000: Illegal parameter data types geometry and int for operation '='
 DROP TABLE t1, t2;
+#
+# MDEV-28871: Assert ... failed in JOIN::dbug_verify_sj_inner_tables with low optimizer_search_depth
+#
+set @tmp_28871=@@optimizer_search_depth;
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (b INT);
+INSERT INTO t1 VALUES (1),(2);
+INSERT INTO t1 VALUES (3),(4);
+SET optimizer_search_depth= 1;
+SELECT * FROM t1 WHERE a IN (SELECT b FROM t2 JOIN t1);
+a
+DROP TABLE t1, t2;
+set optimizer_search_depth= @tmp_28871;
 set optimizer_switch=@subselect_sj_tmp;

mysql-test/main/subselect_sj.test

@@ -3016,5 +3016,22 @@ EXECUTE stmt;
 
 DROP TABLE t1, t2;
 
+--echo #
+--echo # MDEV-28871: Assert ... failed in JOIN::dbug_verify_sj_inner_tables with low optimizer_search_depth
+--echo #
+set @tmp_28871=@@optimizer_search_depth;
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (b INT);
+
+# Data is optional, fails either way
+INSERT INTO t1 VALUES (1),(2);
+INSERT INTO t1 VALUES (3),(4);
+
+SET optimizer_search_depth= 1;
+
+SELECT * FROM t1 WHERE a IN (SELECT b FROM t2 JOIN t1);
+DROP TABLE t1, t2;
+set optimizer_search_depth= @tmp_28871;
+
 # The following command must be the last one the file 
 set optimizer_switch=@subselect_sj_tmp;

mysql-test/main/subselect_sj_jcl6.result

@@ -3338,6 +3338,19 @@ ERROR HY000: Illegal parameter data types geometry and int for operation '='
 EXECUTE stmt;
 ERROR HY000: Illegal parameter data types geometry and int for operation '='
 DROP TABLE t1, t2;
+#
+# MDEV-28871: Assert ... failed in JOIN::dbug_verify_sj_inner_tables with low optimizer_search_depth
+#
+set @tmp_28871=@@optimizer_search_depth;
+CREATE TABLE t1 (a INT);
+CREATE TABLE t2 (b INT);
+INSERT INTO t1 VALUES (1),(2);
+INSERT INTO t1 VALUES (3),(4);
+SET optimizer_search_depth= 1;
+SELECT * FROM t1 WHERE a IN (SELECT b FROM t2 JOIN t1);
+a
+DROP TABLE t1, t2;
+set optimizer_search_depth= @tmp_28871;
 set optimizer_switch=@subselect_sj_tmp;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off

sql/opt_subselect.cc

@@ -2894,6 +2894,7 @@ void optimize_semi_joins(JOIN *join, table_map remaining_tables, uint idx,
 void update_sj_state(JOIN *join, const JOIN_TAB *new_tab,
                      uint idx, table_map remaining_tables)
 {
+  DBUG_ASSERT(!join->emb_sjm_nest);
   if (TABLE_LIST *emb_sj_nest= new_tab->emb_sj_nest)
   {
     join->cur_sj_inner_tables |= emb_sj_nest->sj_inner_tables;

sql/sql_select.cc

@@ -8210,7 +8210,8 @@ greedy_search(JOIN      *join,
       picked semi-join operation is in best_pos->...picker, but we need to
       update the global state in the JOIN object, too.
     */
-    update_sj_state(join, best_table, idx, remaining_tables);
+    if (!join->emb_sjm_nest)
+      update_sj_state(join, best_table, idx, remaining_tables);
 
     /* find the position of 'best_table' in 'join->best_ref' */
     best_idx= idx;