MDEV-35913 Assertion `m_comparator.cmp_type() != ROW_RESULT' in Item_func_in

Check earlier that a column is a virtual column, before checking that all
arguments are constants, to avoid this assertion in the case of non-virtual
columns.

Author: DaveGosselin-MariaDB

mysql-test/main/type_row.result

@@ -72,3 +72,20 @@ SET sql_mode=DEFAULT;
 #
 # End of 10.7 tests
 #
+#
+# Start of 11.8 tests
+#
+#
+# MDEV-35913 Assertion `m_comparator.cmp_type() != ROW_RESULT'
+#
+CREATE TABLE t1 (a TEXT UNIQUE);
+INSERT INTO t1 VALUES('1');
+SELECT 1 FROM t1 WHERE ROW(a, (a,a)) IN ((1, (1,1)),(2, (2,1)));
+1
+1
+DROP TABLE t1;
+CREATE TABLE t1 (a INT KEY,b INT,vb DATE AS (b) VIRTUAL,KEY(vb));
+SELECT * FROM t1 WHERE b<7 AND (a,b) NOT IN ((1,2),(8,9),(5,1));
+a	b	vb
+DROP TABLE t1;
+# End of 11.8 tests

mysql-test/main/type_row.test

@@ -88,3 +88,22 @@ SET sql_mode=DEFAULT;
 --echo #
 --echo # End of 10.7 tests
 --echo #
+
+--echo #
+--echo # Start of 11.8 tests
+--echo #
+
+--echo #
+--echo # MDEV-35913 Assertion `m_comparator.cmp_type() != ROW_RESULT'
+--echo #
+
+CREATE TABLE t1 (a TEXT UNIQUE);
+INSERT INTO t1 VALUES('1');
+SELECT 1 FROM t1 WHERE ROW(a, (a,a)) IN ((1, (1,1)),(2, (2,1)));
+DROP TABLE t1;
+
+CREATE TABLE t1 (a INT KEY,b INT,vb DATE AS (b) VIRTUAL,KEY(vb));
+SELECT * FROM t1 WHERE b<7 AND (a,b) NOT IN ((1,2),(8,9),(5,1));
+DROP TABLE t1;
+
+--echo # End of 11.8 tests

sql/opt_vcol_substitution.cc

@@ -454,14 +454,17 @@ Item* Item_func_null_predicate::vcol_subst_transformer(THD *thd, uchar *arg)
 Item* Item_func_in::vcol_subst_transformer(THD *thd, uchar *arg)
 {
   Vcol_subst_context *ctx= (Vcol_subst_context*)arg;
+  Field *vcol_field= nullptr;
 
-  /* Check that all arguments inside IN() are constants */
-  if (!compatible_types_scalar_bisection_possible())
+  /*
+    Check that the left hand side of IN() is a virtual column expression and
+    that all arguments inside IN() are constants.
+  */
+  if (!(vcol_field= is_vcol_expr(ctx, args[0])) ||
+      !compatible_types_scalar_bisection_possible())
     return this;
 
-  Field *vcol_field;
-  if ((vcol_field= is_vcol_expr(ctx, args[0])))
-    subst_vcol_if_compatible(ctx, this, &args[0], vcol_field);
+  subst_vcol_if_compatible(ctx, this, &args[0], vcol_field);
   return this;
 }