MDEV-7565: Server crash with Signal 6 (part 2)

Preparation of subselect moved earlier (before checks which needs it prepared).

Author: sanja-byelkin

mysql-test/r/subselect.result

@@ -7015,3 +7015,15 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL

mysql-test/r/subselect_no_mat.result

@@ -7012,6 +7012,18 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
 @@optimizer_switch like '%materialization=on%'

mysql-test/r/subselect_no_opts.result

@@ -7010,4 +7010,16 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set @optimizer_switch_for_subselect_test=null;

mysql-test/r/subselect_no_scache.result

@@ -7021,6 +7021,18 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
 @@optimizer_switch like '%subquery_cache=on%'

mysql-test/r/subselect_no_semijoin.result

@@ -7010,5 +7010,17 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;

mysql-test/t/subselect.test

@@ -5890,3 +5890,15 @@ DROP TABLE t1,t2;
 create table t1 (c int);
 select exists(select 1 from t1 group by `c` in (select `c` from t1));
 drop table t1;
+
+--echo #
+--echo # MDEV-7565: Server crash with Signal 6 (part 2)
+--echo #
+Select 
+  (Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+    Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+  ) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+

sql/opt_subselect.cc

@@ -703,6 +703,12 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
       if (!optimizer_flag(thd, OPTIMIZER_SWITCH_IN_TO_EXISTS) &&
           !optimizer_flag(thd, OPTIMIZER_SWITCH_MATERIALIZATION))
         my_error(ER_ILLEGAL_SUBQUERY_OPTIMIZER_SWITCHES, MYF(0));
+      /*
+        Transform each subquery predicate according to its overloaded
+        transformer.
+      */
+      if (subselect->select_transformer(join))
+        DBUG_RETURN(-1);
 
       /*
         If the subquery predicate is IN/=ANY, analyse and set all possible
@@ -754,12 +760,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         allany_subs->add_strategy(strategy);
       }
 
-      /*
-        Transform each subquery predicate according to its overloaded
-        transformer.
-      */
-      if (subselect->select_transformer(join))
-        DBUG_RETURN(-1);
     }
   }
   DBUG_RETURN(0);