MDEV-29362 Crash with query using constant subquery as left part of I…

…N subquery

If a query has a HAVING clause that contains a predicate with a constant
IN subquery whose lef part in its turn is a subquery and the predicate is
subject to pushdown from HAVING to WHERE then execution of the query could
cause a crash of the server.

The cause of the problem was the missing implementation of the walk()
method for the class Item_in_optimizer. As a result in some cases the left
operand of the Item_in_optimizer condition could be traversed twice by
the walk procedure. For many call-back functions used as an argument of
this procedure it does not matter. Yet it matters for the call-back
function cleanup_excluding_immutables_processor() used in pushdown of
predicates from HAVING to WHERE. If the processed item is marked with
the IMMUTABLE_FL flag then the processor just removes this flag, otherwise
it performs cleanup of the item making it unfixed. If an item is marked
with an the IMMUTABLE_FL and it traversed with this processor twice then
it becomes unfixed after the second traversal though the flag indicates
that the item should not be cleaned up.

Approved by Oleksandr Byelkin <sanja@mariadb.com>

mysql-test/main/subselect4.result

@@ -3213,4 +3213,83 @@ FROM x
 )
 );
 ERROR 21000: Operand should contain 2 column(s)
+#
+# MDEV-29362: Constant subquery used as left part of IN subquery
+#
+CREATE TABLE t1 (a int) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (15), (1), (2);
+CREATE TABLE t2 (b int) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (15), (1);
+CREATE TABLE t3 (c int) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (15), (1);
+SET optimizer_switch='condition_pushdown_from_having=off';
+SELECT a FROM t1 GROUP BY a
+HAVING a = ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  ) + 1;
+a
+2
+SELECT a FROM t1 GROUP BY a
+HAVING a IN ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  );
+a
+1
+SET optimizer_switch='condition_pushdown_from_having=on';
+SELECT a FROM t1 GROUP BY a
+HAVING a = ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  ) + 1;
+a
+2
+SELECT a FROM t1 GROUP BY a
+HAVING a IN ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  );
+a
+1
+EXPLAIN FORMAT=JSON SELECT a FROM t1 GROUP BY a
+HAVING a = ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  ) + 1;
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "table": {
+      "table_name": "t1",
+      "access_type": "ALL",
+      "rows": 3,
+      "filtered": 100,
+      "attached_condition": "t1.a = <cache>(<in_optimizer>((subquery#2),<exists>(subquery#3)) + 1)"
+    },
+    "subqueries": [
+      {
+        "query_block": {
+          "select_id": 3,
+          "having_condition": "trigcond(t3.c is null)",
+          "table": {
+            "table_name": "t3",
+            "access_type": "ALL",
+            "rows": 2,
+            "filtered": 100,
+            "attached_condition": "trigcond(1 = t3.c or t3.c is null)"
+          }
+        }
+      },
+      {
+        "query_block": {
+          "select_id": 2,
+          "table": {
+            "table_name": "t2",
+            "access_type": "ALL",
+            "rows": 2,
+            "filtered": 100,
+            "attached_condition": "t2.b = 1"
+          }
+        }
+      }
+    ]
+  }
+}
+PREPARE stmt FROM "SELECT a FROM t1 GROUP BY a
+HAVING a = ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  ) + 1";
+EXECUTE stmt;
+a
+2
+EXECUTE stmt;
+a
+2
+DEALLOCATE PREPARE stmt;
+DROP TABLE t1,t2,t3;
 # End of 10.4 tests

mysql-test/main/subselect4.test

@@ -2629,5 +2629,41 @@ SELECT
          )
   );
 
+--echo #
+--echo # MDEV-29362: Constant subquery used as left part of IN subquery
+--echo #
+
+CREATE TABLE t1 (a int) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (15), (1), (2);
+CREATE TABLE t2 (b int) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (15), (1);
+CREATE TABLE t3 (c int) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (15), (1);
+
+let $q1=
+SELECT a FROM t1 GROUP BY a
+  HAVING a = ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  ) + 1;
+let $q2=
+SELECT a FROM t1 GROUP BY a
+  HAVING a IN ( (SELECT b FROM t2 where b=1) IN (SELECT c FROM t3)  );
+
+SET optimizer_switch='condition_pushdown_from_having=off';
+
+eval $q1;
+eval $q2;
+
+SET optimizer_switch='condition_pushdown_from_having=on';
+
+eval $q1;
+eval $q2;
+
+eval EXPLAIN FORMAT=JSON $q1;
+
+eval PREPARE stmt FROM "$q1";
+EXECUTE stmt;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+DROP TABLE t1,t2,t3;
 
 --echo # End of 10.4 tests

sql/item_cmpfunc.cc

@@ -1444,6 +1444,23 @@ bool Item_in_optimizer::invisible_mode()
 }
 
 
+bool Item_in_optimizer::walk(Item_processor processor,
+                             bool walk_subquery,
+                             void *arg)
+{
+  bool res= FALSE;
+  if (args[1]->type() == Item::SUBSELECT_ITEM &&
+      ((Item_subselect *)args[1])->substype() != Item_subselect::EXISTS_SUBS &&
+      !(((Item_subselect *)args[1])->substype() == Item_subselect::IN_SUBS &&
+        ((Item_in_subselect *)args[1])->test_strategy(SUBS_IN_TO_EXISTS)))
+    res= args[0]->walk(processor, walk_subquery, arg);
+  if (!res)
+    res= args[1]->walk(processor, walk_subquery, arg);
+
+  return res || (this->*processor)(arg);
+}
+
+
 /**
   Add an expression cache for this subquery if it is needed
 

sql/item_cmpfunc.h

@@ -385,6 +385,7 @@ class Item_in_optimizer: public Item_bool_func
   bool eval_not_null_tables(void *opt_arg);
   void fix_after_pullout(st_select_lex *new_parent, Item **ref, bool merge);
   bool invisible_mode();
+  bool walk(Item_processor processor, bool walk_subquery, void *arg);
   void reset_cache() { cache= NULL; }
   virtual void print(String *str, enum_query_type query_type);
   void restore_first_argument();