Bug#18411494 WRONG COMPARSION ON BIG DECIMAL VALUES

Problem: integer literals may be converted to floats for
comparison with decimal data. If the integers are large,
we may lose precision, and give wrong results.

Fix: for
  <non-const decimal expression> <cmp> <const string expression>
  or
  <const string expression> <cmp> <non-const decimal expression>
we override the compare_type chosen by item_cmp_type(), and
do comparison as decimal rather than float.

(cherry picked from commit mysql/mysql-server@1cf3489 and edited by Johannes Weißl <jargon@molb.org>)

Author: Tor Didriksen

mysql-test/r/type_newdecimal.result

@@ -2243,3 +2243,42 @@ DROP TABLE t1;
 #
 # End of 10.1 tests
 #
+#
+# Bug#18408499 UNSIGNED BIGINT HIGH VALUES 
+#              WRONG NUMERICAL COMPARISON RESULTS
+#
+CREATE TABLE t1(value DECIMAL(24,0) NOT NULL);
+INSERT INTO t1(value) 
+VALUES('100000000000000000000001'),
+('100000000000000000000002'), 
+('100000000000000000000003');
+SELECT * FROM t1 WHERE value = '100000000000000000000002';
+value
+100000000000000000000002
+SELECT * FROM t1 WHERE '100000000000000000000002' = value;
+value
+100000000000000000000002
+SELECT * FROM t1 WHERE value + 0 = '100000000000000000000002';
+value
+100000000000000000000002
+SELECT * FROM t1 WHERE value = 100000000000000000000002;
+value
+100000000000000000000002
+SELECT * FROM t1 WHERE value + 0 = 100000000000000000000002;
+value
+100000000000000000000002
+PREPARE stmt FROM 'SELECT * FROM t1 WHERE value = ?';
+set @a="100000000000000000000002";
+EXECUTE stmt using @a;
+value
+100000000000000000000002
+set @a=100000000000000000000002;
+EXECUTE stmt using @a;
+value
+100000000000000000000002
+DEALLOCATE PREPARE stmt;
+ALTER TABLE t1 ADD INDEX value (value);
+SELECT * FROM t1 WHERE value = '100000000000000000000002';
+value
+100000000000000000000002
+DROP TABLE t1;

mysql-test/t/type_newdecimal.test

@@ -1738,3 +1738,30 @@ DROP TABLE t1;
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+--echo #
+--echo # Bug#18408499 UNSIGNED BIGINT HIGH VALUES 
+--echo #              WRONG NUMERICAL COMPARISON RESULTS
+--echo #
+
+CREATE TABLE t1(value DECIMAL(24,0) NOT NULL);
+INSERT INTO t1(value) 
+VALUES('100000000000000000000001'),
+      ('100000000000000000000002'), 
+      ('100000000000000000000003');
+SELECT * FROM t1 WHERE value = '100000000000000000000002';
+SELECT * FROM t1 WHERE '100000000000000000000002' = value;
+SELECT * FROM t1 WHERE value + 0 = '100000000000000000000002';
+SELECT * FROM t1 WHERE value = 100000000000000000000002;
+SELECT * FROM t1 WHERE value + 0 = 100000000000000000000002;
+
+PREPARE stmt FROM 'SELECT * FROM t1 WHERE value = ?';
+set @a="100000000000000000000002";
+EXECUTE stmt using @a;
+set @a=100000000000000000000002;
+EXECUTE stmt using @a;
+DEALLOCATE PREPARE stmt;
+
+ALTER TABLE t1 ADD INDEX value (value);
+SELECT * FROM t1 WHERE value = '100000000000000000000002';
+DROP TABLE t1;

sql/item_cmpfunc.cc

@@ -655,6 +655,22 @@ int Arg_comparator::set_cmp_func(Item_func_or_sum *owner_arg,
     return 0;
   }
 
+  if (m_compare_type == REAL_RESULT &&
+      (((*a)->result_type() == DECIMAL_RESULT && !(*a)->const_item() &&
+        (*b)->result_type() == STRING_RESULT  &&  (*b)->const_item()) ||
+       ((*b)->result_type() == DECIMAL_RESULT && !(*b)->const_item() &&
+        (*a)->result_type() == STRING_RESULT  &&  (*a)->const_item())))
+  {
+    /*
+     <non-const decimal expression> <cmp> <const string expression>
+     or
+     <const string expression> <cmp> <non-const decimal expression>
+
+     Do comparison as decimal rather than float, in order not to lose precision.
+    */
+    m_compare_type= DECIMAL_RESULT;
+  }
+
   if (m_compare_type == INT_RESULT &&
       (*a)->field_type() == MYSQL_TYPE_YEAR &&
       (*b)->field_type() == MYSQL_TYPE_YEAR)