MDEV-23282 FLOAT(53,0) badly handles out-of-range values

truncate_double() did not take into account the max_value limit in case when dec<NOT_FIXED_DEC.
MariaDB · Jul 27, 2020 · 29851b6 · 29851b6
1 parent 8460db1
commit 29851b6
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 4 deletions.
diff --git a/mysql-test/r/type_float.result b/mysql-test/r/type_float.result
@@ -672,5 +672,26 @@ Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (`test`.`t1`.`a` = 2010e0)
 DROP TABLE t1;
 #
+# MDEV-23282 FLOAT(53,0) badly handles out-of-range values
+#
+CREATE OR REPLACE TABLE t1 (c1 FLOAT NOT NULL, c2 FLOAT NOT NULL);
+INSERT IGNORE INTO t1 VALUES (1e+40, -1e+40);
+Warnings:
+Warning	1264	Out of range value for column 'c1' at row 1
+Warning	1264	Out of range value for column 'c2' at row 1
+SELECT c1, c2 FROM t1;
+c1	c2
+3.40282e38	-3.40282e38
+DROP TABLE t1;
+CREATE OR REPLACE TABLE t1 (c1 FLOAT(53,0) NOT NULL, c2 FLOAT(53,0) NOT NULL);
+INSERT IGNORE INTO t1 VALUES (1e+40, -1e+40);
+Warnings:
+Warning	1264	Out of range value for column 'c1' at row 1
+Warning	1264	Out of range value for column 'c2' at row 1
+SELECT c1, c2 FROM t1;
+c1	c2
+340282346638528860000000000000000000000	-340282346638528860000000000000000000000
+DROP TABLE t1;
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/suite/engines/iuds/r/insert_decimal.result b/mysql-test/suite/engines/iuds/r/insert_decimal.result
@@ -1892,9 +1892,13 @@ Warnings:
 Warning	1264	Out of range value for column 'c1' at row 3
 INSERT INTO t5 VALUES('1e+52','-1e+52','1e+52',5),('1e-52','-1e-52','1e-52',6);
 Warnings:
+Warning	1264	Out of range value for column 'c1' at row 1
+Warning	1264	Out of range value for column 'c2' at row 1
 Warning	1264	Out of range value for column 'c3' at row 1
 INSERT INTO t5 VALUES('1e+53','-1e+53','1e+53',7),('1e-53','-1e-53','1e-53',8);
 Warnings:
+Warning	1264	Out of range value for column 'c1' at row 1
+Warning	1264	Out of range value for column 'c2' at row 1
 Warning	1264	Out of range value for column 'c3' at row 1
 SELECT * FROM t5;
 c1	c2	c3	c4

diff --git a/mysql-test/t/type_float.test b/mysql-test/t/type_float.test
@@ -485,6 +485,20 @@ EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=2010e0 AND a>=2010e0;
 DROP TABLE t1;
 
 
+--echo #
+--echo # MDEV-23282 FLOAT(53,0) badly handles out-of-range values
+--echo #
+
+CREATE OR REPLACE TABLE t1 (c1 FLOAT NOT NULL, c2 FLOAT NOT NULL);
+INSERT IGNORE INTO t1 VALUES (1e+40, -1e+40);
+SELECT c1, c2 FROM t1;
+DROP TABLE t1;
+
+CREATE OR REPLACE TABLE t1 (c1 FLOAT(53,0) NOT NULL, c2 FLOAT(53,0) NOT NULL);
+INSERT IGNORE INTO t1 VALUES (1e+40, -1e+40);
+SELECT c1, c2 FROM t1;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.1 tests
 --echo #
diff --git a/sql/field.cc b/sql/field.cc
@@ -4627,11 +4627,12 @@ int truncate_double(double *nr, uint field_length, uint dec,
   {
     uint order= field_length - dec;
     uint step= array_elements(log_10) - 1;
-    max_value= 1.0;
+    double max_value_by_dec= 1.0;
     for (; order > step; order-= step)
-      max_value*= log_10[step];
-    max_value*= log_10[order];
-    max_value-= 1.0 / log_10[dec];
+      max_value_by_dec*= log_10[step];
+    max_value_by_dec*= log_10[order];
+    max_value_by_dec-= 1.0 / log_10[dec];
+    set_if_smaller(max_value, max_value_by_dec);
 
     /* Check for infinity so we don't get NaN in calculations */
     if (!my_isinf(res))