MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed …

…and in Item_type_holder::val_decimal on SELECT

The bug is fixed by the patch ported from MySQL. See the comprehensive
description below.

commit 455c4e8810c76430719b1a08a63ca0f69f44678a
Author: Guilhem Bichot <guilhem.bichot@oracle.com>
Date:   Fri Mar 13 17:51:27 2015 +0100

    Bug#17668844: CRASH/ASSERT AT ITEM_TYPE_HOLDER::VAL_STR IN ITEM.C

    We have a predicate of the form:
    literal_row <=> (a UNION)

    The subquery is constant, so Item_cache objects are used for its
    SELECT list.
    In order, this happens:
    - Item_subselect::fix_fields() calls select_lex_unit::prepare,
    where we create Item_type_holder's
    (appended to unit->types list), create the tmp table (using type info
    found in unit->types), and call fill_item_list() to put the
    Item_field's of this table into unit->item_list.
    - Item_subselect::fix_length_and_dec() calls set_row() which
    makes Item_cache's of the subquery wrap the Item_type_holder's
    - When/if a first result row is found for the subquery,
    Item_cache's are re-pointed to unit->item_list
    (i.e. Item_field objects which reference the UNION's tmp table
    columns) (see call to Item_singlerow_subselect::store()).
    - In our subquery, no result row is found, so the Item_cache's
    still wrap Item_type_holder's; evaluating '<=>' reads the
    value of those, but Item_type_holder objects are not expected to be
    evaluated.

    Fix: instead of putting unit->types into Item_cache, and later
    replacing with unit->item_list, put unit->item_list in Item_cache from
    the start.

Approved by Oleksandr Byelkin <sanja@mariadb.com>

mysql-test/main/subselect.result

@@ -1320,7 +1320,7 @@ a
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` int(3) DEFAULT NULL
+  `a` int(11) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 create table t1 (a int);
@@ -7498,5 +7498,44 @@ ERROR HY000: Illegal parameter data types row and boolean for operation '='
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 ERROR HY000: Illegal parameter data types row and boolean for operation '='
 #
+# MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+# in Item_type_holder::val_decimal on SELECT
+#
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+a
+1
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+a
+1
+UPDATE t1 SET a = 0
+WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT + 1 / + 1, a FROM t1
+WHERE a > -0 + 1) IN (SELECT a, a);
+a
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+WITH RECURSIVE x (x) AS (
+SELECT 1 INTERSECT
+SELECT -(SELECT 1.000000 AS x
+UNION
+SELECT 1.000000 ORDER BY NOT x < 'x',
+-(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+) + 1 FROM x
+)
+SELECT DISTINCT x, 1, NULL, 1.000000
+FROM x
+WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+(SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+ORDER BY x ASC, x DESC, x;
+ERROR 22007: Truncated incorrect DECIMAL value: 'x'
+DROP TABLE t1, x;
+#
 # End of 10.4 tests
 #

mysql-test/main/subselect.test

@@ -6346,6 +6346,50 @@ SELECT ROW(1,2) = 1 IN (SELECT 1 UNION SELECT 2);
 --error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 
+--echo #
+--echo # MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+--echo # in Item_type_holder::val_decimal on SELECT
+--echo #
+
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+
+UPDATE t1 SET a = 0
+  WHERE (SELECT a, a WHERE a < 0 INTERSECT
+         SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+                        SELECT + 1 / + 1, a FROM t1
+  WHERE a > -0 + 1) IN (SELECT a, a);
+
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+let $q= WITH RECURSIVE x (x) AS (
+  SELECT 1 INTERSECT
+  SELECT -(SELECT 1.000000 AS x
+           UNION
+           SELECT 1.000000 ORDER BY NOT x < 'x',
+                                    -(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+                                    WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+           ) + 1 FROM x
+  )
+  SELECT DISTINCT x, 1, NULL, 1.000000
+  FROM x
+  WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+        (SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+        ORDER BY x ASC, x DESC, x;
+
+--error ER_TRUNCATED_WRONG_VALUE
+eval $q;
+
+DROP TABLE t1, x;
+
 --echo #
 --echo # End of 10.4 tests
 --echo #

mysql-test/main/subselect_no_exists_to_in.result

@@ -1324,7 +1324,7 @@ a
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` int(3) DEFAULT NULL
+  `a` int(11) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 create table t1 (a int);
@@ -7498,6 +7498,45 @@ ERROR HY000: Illegal parameter data types row and boolean for operation '='
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 ERROR HY000: Illegal parameter data types row and boolean for operation '='
 #
+# MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+# in Item_type_holder::val_decimal on SELECT
+#
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+a
+1
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+a
+1
+UPDATE t1 SET a = 0
+WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT + 1 / + 1, a FROM t1
+WHERE a > -0 + 1) IN (SELECT a, a);
+a
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+WITH RECURSIVE x (x) AS (
+SELECT 1 INTERSECT
+SELECT -(SELECT 1.000000 AS x
+UNION
+SELECT 1.000000 ORDER BY NOT x < 'x',
+-(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+) + 1 FROM x
+)
+SELECT DISTINCT x, 1, NULL, 1.000000
+FROM x
+WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+(SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+ORDER BY x ASC, x DESC, x;
+ERROR 22007: Truncated incorrect DECIMAL value: 'x'
+DROP TABLE t1, x;
+#
 # End of 10.4 tests
 #
 set optimizer_switch=default;

mysql-test/main/subselect_no_mat.result

@@ -1327,7 +1327,7 @@ a
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` int(3) DEFAULT NULL
+  `a` int(11) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 create table t1 (a int);
@@ -7491,6 +7491,45 @@ ERROR HY000: Illegal parameter data types row and boolean for operation '='
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 ERROR HY000: Illegal parameter data types row and boolean for operation '='
 #
+# MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+# in Item_type_holder::val_decimal on SELECT
+#
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+a
+1
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+a
+1
+UPDATE t1 SET a = 0
+WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT + 1 / + 1, a FROM t1
+WHERE a > -0 + 1) IN (SELECT a, a);
+a
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+WITH RECURSIVE x (x) AS (
+SELECT 1 INTERSECT
+SELECT -(SELECT 1.000000 AS x
+UNION
+SELECT 1.000000 ORDER BY NOT x < 'x',
+-(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+) + 1 FROM x
+)
+SELECT DISTINCT x, 1, NULL, 1.000000
+FROM x
+WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+(SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+ORDER BY x ASC, x DESC, x;
+ERROR 22007: Truncated incorrect DECIMAL value: 'x'
+DROP TABLE t1, x;
+#
 # End of 10.4 tests
 #
 set optimizer_switch=default;

mysql-test/main/subselect_no_opts.result

@@ -1323,7 +1323,7 @@ a
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` int(3) DEFAULT NULL
+  `a` int(11) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 create table t1 (a int);
@@ -7489,6 +7489,45 @@ ERROR HY000: Illegal parameter data types row and boolean for operation '='
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 ERROR HY000: Illegal parameter data types row and boolean for operation '='
 #
+# MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+# in Item_type_holder::val_decimal on SELECT
+#
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+a
+1
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+a
+1
+UPDATE t1 SET a = 0
+WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT + 1 / + 1, a FROM t1
+WHERE a > -0 + 1) IN (SELECT a, a);
+a
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+WITH RECURSIVE x (x) AS (
+SELECT 1 INTERSECT
+SELECT -(SELECT 1.000000 AS x
+UNION
+SELECT 1.000000 ORDER BY NOT x < 'x',
+-(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+) + 1 FROM x
+)
+SELECT DISTINCT x, 1, NULL, 1.000000
+FROM x
+WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+(SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+ORDER BY x ASC, x DESC, x;
+ERROR 22007: Truncated incorrect DECIMAL value: 'x'
+DROP TABLE t1, x;
+#
 # End of 10.4 tests
 #
 set @optimizer_switch_for_subselect_test=null;

mysql-test/main/subselect_no_scache.result

@@ -1326,7 +1326,7 @@ a
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` int(3) DEFAULT NULL
+  `a` int(11) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 create table t1 (a int);
@@ -7504,6 +7504,45 @@ ERROR HY000: Illegal parameter data types row and boolean for operation '='
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 ERROR HY000: Illegal parameter data types row and boolean for operation '='
 #
+# MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+# in Item_type_holder::val_decimal on SELECT
+#
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+a
+1
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+a
+1
+UPDATE t1 SET a = 0
+WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT + 1 / + 1, a FROM t1
+WHERE a > -0 + 1) IN (SELECT a, a);
+a
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+WITH RECURSIVE x (x) AS (
+SELECT 1 INTERSECT
+SELECT -(SELECT 1.000000 AS x
+UNION
+SELECT 1.000000 ORDER BY NOT x < 'x',
+-(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+) + 1 FROM x
+)
+SELECT DISTINCT x, 1, NULL, 1.000000
+FROM x
+WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+(SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+ORDER BY x ASC, x DESC, x;
+ERROR 22007: Truncated incorrect DECIMAL value: 'x'
+DROP TABLE t1, x;
+#
 # End of 10.4 tests
 #
 set optimizer_switch=default;

mysql-test/main/subselect_no_semijoin.result

@@ -1323,7 +1323,7 @@ a
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` int(3) DEFAULT NULL
+  `a` int(11) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 create table t1 (a int);
@@ -7489,6 +7489,45 @@ ERROR HY000: Illegal parameter data types row and boolean for operation '='
 SELECT ROW(1,2) = (1 = ANY (SELECT 1 UNION SELECT 2));
 ERROR HY000: Illegal parameter data types row and boolean for operation '='
 #
+# MDEV-29070 SIGSEGV in my_decimal::operator= and Assertion `0' failed
+# in Item_type_holder::val_decimal on SELECT
+#
+CREATE TABLE t1(a INT UNIQUE);
+INSERT INTO t1(a) VALUES (1);
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT 1, 1);
+a
+1
+SELECT a FROM t1 WHERE (SELECT a, a UNION SELECT 1, a FROM t1) IN (SELECT a, a);
+a
+1
+UPDATE t1 SET a = 0
+WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT +1 / +1, a FROM t1 WHERE a > -0+1) IN (SELECT a, a);
+SELECT a FROM t1 WHERE (SELECT a, a WHERE a < 0 INTERSECT
+SELECT + 1 / + 1, a FROM t1
+WHERE a > -0 + 1) IN (SELECT a, a);
+a
+CREATE TABLE x (x INT);
+INSERT INTO x (x) VALUES (1);
+UPDATE x SET x = 1 WHERE x = 1;
+INSERT INTO x (x) VALUES (1), (1);
+WITH RECURSIVE x (x) AS (
+SELECT 1 INTERSECT
+SELECT -(SELECT 1.000000 AS x
+UNION
+SELECT 1.000000 ORDER BY NOT x < 'x',
+-(SELECT 1 + x/1.000000 IN (1, 1) FROM x
+WHERE x ORDER BY 1 - x) DESC LIMIT 1 OFFSET 1
+) + 1 FROM x
+)
+SELECT DISTINCT x, 1, NULL, 1.000000
+FROM x
+WHERE (SELECT (SELECT x WHERE x IN (SELECT x FROM x))) >
+(SELECT (SELECT x ORDER BY x = x OR (x = 1 AND x = 1) DESC))
+ORDER BY x ASC, x DESC, x;
+ERROR 22007: Truncated incorrect DECIMAL value: 'x'
+DROP TABLE t1, x;
+#
 # End of 10.4 tests
 #
 #

sql/item_subselect.cc

@@ -3910,14 +3910,14 @@ bool subselect_union_engine::fix_length_and_dec(Item_cache **row)
 
   if (unit->first_select()->item_list.elements == 1)
   {
-    if (set_row(unit->types, row))
+    if (set_row(unit->item_list, row))
       return TRUE;
     item->collation.set(row[0]->collation);
   }
   else
   {
     bool maybe_null_saved= maybe_null;
-    if (set_row(unit->types, row))
+    if (set_row(unit->item_list, row))
       return TRUE;
     maybe_null= maybe_null_saved;
   }