MDEV-35164: optimizer_join_limit_pref_ratio: assertion when the ORDER BY table becomes constant

Assertion failure has happened due to this scenario:

A query was ran with optimizer_join_limit_pref_ratio=1.
The query had "ORDER BY t1.col LIMIT N".
The optimizer set join->limit_shortcut_applicable=1.
Then, table t1 was marked as constant.
The code in choose_query_plan() still set join->limit_optimization_mode=1
which caused the optimizer to only consider t1 as the first non-const table.
But t1 was already put into the join prefix as the constant table.
The optimizer couldn't produce any join order at all and crashed.

Fixed by not searching for shortcut plan if ORDER BY table is a constant.
We will not try to do sorting anyway in this case (and LIMIT short-cutting
will be done for any join order).

Author: spetrunia

mysql-test/main/order_by_limit_join.result

@@ -457,6 +457,32 @@ JS
 ]
 set optimizer_search_depth=@tmp_osd;
 set optimizer_trace=@tmp_os;
+# An extra testcase for MDEV-35164 (its main testcase is below).
+alter table t1 add unique key(col2);
+insert into t10 select * from t10;
+insert into t10 select * from t10;
+analyze table t10;
+Table	Op	Msg_type	Msg_text
+test.t10	analyze	status	Engine-independent statistics collected
+test.t10	analyze	status	OK
+# This will not crash and also show that sorting is not done when
+#   ORDER BY only refers to const table columns:
+explain
+select
+*
+from
+t1
+join t10 on t1.a=t10.a
+join t11 on t1.b=t11.b
+where
+t1.col2=3
+order by
+t1.col1
+limit 10;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	const	col2,a,b	col2	5	const	1	
+1	SIMPLE	t11	ref	b	b	5	const	1	
+1	SIMPLE	t10	ref	a	a	5	const	2	
 drop table t1, t10, t11;
 #
 # MDEV-35072: Assertion failure with optimizer_join_limit_pref_ratio and 1-table select
@@ -468,4 +494,15 @@ SELECT * FROM t1  ORDER BY c1 LIMIT 1;
 c1
 1
 DROP TABLE t1;
+#
+# MDEV-35164: optimizer_join_limit_pref_ratio: assertion when the ORDER BY table becomes constant
+# Original testcase:
+#
+SET optimizer_join_limit_pref_ratio=1;
+CREATE TABLE t1 (a INT KEY,b INT, KEY(b)) ;
+INSERT INTO t1 VALUES (2,NULL);
+INSERT INTO t1 VALUES (5,NULL);
+SELECT * FROM t1 NATURAL JOIN t1 AS t2 WHERE t1.b=NULL ORDER BY t1.a LIMIT 1;
+a	b
+DROP TABLE t1;
 set optimizer_join_limit_pref_ratio=default;

mysql-test/main/order_by_limit_join.test

@@ -205,9 +205,29 @@ set @trace=(select trace from information_schema.optimizer_trace);
 --source include/optimizer_trace_no_costs.inc
 select json_detailed(json_extract(@trace, '$**.join_limit_shortcut_choice')) as JS;
 
-
 set optimizer_search_depth=@tmp_osd;
 set optimizer_trace=@tmp_os;
+
+--echo # An extra testcase for MDEV-35164 (its main testcase is below).
+alter table t1 add unique key(col2);
+insert into t10 select * from t10;
+insert into t10 select * from t10;
+analyze table t10;
+
+--echo # This will not crash and also show that sorting is not done when
+--echo #   ORDER BY only refers to const table columns:
+explain
+select
+  *
+from
+  t1
+  join t10 on t1.a=t10.a
+  join t11 on t1.b=t11.b
+where
+ t1.col2=3
+order by
+  t1.col1
+limit 10;
 drop table t1, t10, t11;
 
 --echo #
@@ -219,4 +239,15 @@ INSERT INTO t1  VALUES (1),(2);
 SELECT * FROM t1  ORDER BY c1 LIMIT 1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-35164: optimizer_join_limit_pref_ratio: assertion when the ORDER BY table becomes constant
+--echo # Original testcase:
+--echo #
+SET optimizer_join_limit_pref_ratio=1;
+CREATE TABLE t1 (a INT KEY,b INT, KEY(b)) ;
+INSERT INTO t1 VALUES (2,NULL);
+INSERT INTO t1 VALUES (5,NULL);
+SELECT * FROM t1 NATURAL JOIN t1 AS t2 WHERE t1.b=NULL ORDER BY t1.a LIMIT 1;
+DROP TABLE t1;
+
 set optimizer_join_limit_pref_ratio=default;

sql/sql_select.cc

@@ -9306,8 +9306,18 @@ choose_plan(JOIN *join, table_map join_tables)
     double limit_cost= DBL_MAX;
     POSITION *limit_plan= NULL;
 
-    /* First, build a join plan that can short-cut ORDER BY...LIMIT */
-    if (join->limit_shortcut_applicable && !join->emb_sjm_nest)
+    /*
+      First, build a join plan that can short-cut ORDER BY...LIMIT.
+      Do it if
+      (1) The SELECT in query makes it possible to do short-cutting for
+          some table TBL.
+      (2) We are optimizing the whole JOIN, not a semi-join nest
+      (3) The table TBL has not been marked as constant (in this case,
+          ORDER BY LIMIT will be optimized away)
+    */
+    if (join->limit_shortcut_applicable &&                  // (1)
+        !join->emb_sjm_nest &&                              // (2)
+        !(join->sort_by_table->map & join->const_table_map)) //(3)
     {
       bool res;
       Json_writer_object wrapper(join->thd);