Bug#34554755: DISTINCT on FLOAT values gives duplicates with SQL_BIG_RESULT

kahatlen · kahatlen · commit f8b4c9997c0f · 2022-09-06T15:23:00.000+02:00
Values returned from operators that convert a value to FLOAT (that is,
CAST(... AS FLOAT), CONVERT(..., FLOAT) or JSON_VALUE(... RETURNING
FLOAT)) may have extra precision in their internal representation,
since they are stored in double precision internally. This sometimes
causes unexpected results when checking such values for equality.

The symptoms include SELECT DISTINCT returning duplicates and
comparison operators incorrectly reporting two equal values as not
equal.

Fixed by stripping off the extra double precision from the values
before returning them. Also make conversion from float to string in
these conversion operators use float format instead of double format.

Change-Id: Id3dc86c6d791fce34a9e5b10a5b6097c336ed534
diff --git a/mysql-test/r/cast.result b/mysql-test/r/cast.result
@@ -2276,3 +2276,19 @@ Warning	1287	Setting user variables within expressions is deprecated and will be
 Warning	3020	Invalid argument for logarithm
 set @a:= @b;
 do convert(@a using latin1);
+#
+# Bug#34554755: DISTINCT on FLOAT values gives duplicates
+#               with SQL_BIG_RESULT
+#
+SELECT CAST(1111111111111111 AS FLOAT) = CAST(1111111111111110 AS FLOAT) AS eq;
+eq
+1
+SELECT CAST(CAST(1111111111111111 AS FLOAT) AS CHAR) AS v;
+v
+1.11111e15
+SELECT CAST(CAST(1111111111111111 AS FLOAT) AS SIGNED) AS v;
+v
+1111111126482944
+SELECT CAST(CAST(1111111111111111 AS FLOAT) AS DECIMAL(20,2)) AS v;
+v
+1111111126482944.00
diff --git a/mysql-test/r/distinct.result b/mysql-test/r/distinct.result
@@ -1414,10 +1414,6 @@ CAST(v AS FLOAT)
 -2007570000
 -2007570000
 -2007570000
--2007570000
--2007570000
-2007570000
-2007570000
 2007570000
 2007570000
 2007570000
diff --git a/mysql-test/suite/json/r/json_value.result b/mysql-test/suite/json/r/json_value.result
@@ -862,7 +862,7 @@ JSON_VALUE('1', '$' RETURNING JSON) + 1.0e0
 2
 SELECT JSON_VALUE('1.2', '$' RETURNING FLOAT) + 1.0e0;
 JSON_VALUE('1.2', '$' RETURNING FLOAT) + 1.0e0
-2.2
+2.200000047683716
 SELECT JSON_VALUE('1988', '$' RETURNING YEAR) + 1.0e0;
 JSON_VALUE('1988', '$' RETURNING YEAR) + 1.0e0
 1989
@@ -1020,11 +1020,15 @@ SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
 TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
 '$' RETURNING DECIMAL(14, 0)))
 2000-01-02 12:14:14
+# Warns because 20000102121314 gets truncated to fit in a FLOAT,
+# and is not a valid timestamp after truncation.
 SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
 '$' RETURNING FLOAT));
 TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
 '$' RETURNING FLOAT))
-2000-01-02 12:14:14
+NULL
+Warnings:
+Warning	1292	Incorrect datetime value: '20000102416384'
 SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
 '$' RETURNING DOUBLE));
 TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
@@ -1347,6 +1351,7 @@ Note	1003	/* select#1 */ select `test`.`t1`.`j` AS `j` from `test`.`t1` where (j
 SELECT * FROM t1
 WHERE JSON_VALUE(j, '$' RETURNING FLOAT) = CAST(3.3 AS FLOAT);
 j
+3.3
 # The index is not used if the JSON_VALUE expressions differ.
 EXPLAIN SELECT * FROM t1
 WHERE JSON_VALUE(j, '$' RETURNING DOUBLE) = CAST(3.3 AS FLOAT);
@@ -1811,3 +1816,34 @@ SELECT JSON_VALUE(JSON_OBJECT('a', 'value'), '$.a'),
 JSON_VALUE(JSON_OBJECT('a', NULL), '$.a');
 JSON_VALUE(JSON_OBJECT('a', 'value'), '$.a')	JSON_VALUE(JSON_OBJECT('a', NULL), '$.a')
 value	NULL
+#
+# Bug#34554755: DISTINCT on FLOAT values gives duplicates
+#               with SQL_BIG_RESULT
+#
+SELECT JSON_VALUE('1111111111111111', '$' RETURNING FLOAT) =
+JSON_VALUE('1111111111111110', '$' RETURNING FLOAT) AS eq;
+eq
+1
+SELECT CAST(JSON_VALUE('1111111111111111', '$' RETURNING FLOAT) AS CHAR) AS v;
+v
+1.11111e15
+SELECT CAST(JSON_VALUE('1111111111111111', '$' RETURNING FLOAT) AS SIGNED) AS v;
+v
+1111111126482944
+SELECT CAST(JSON_VALUE('1111111111111111', '$' RETURNING FLOAT)
+AS DECIMAL(20,2)) AS res;
+res
+1111111126482944.00
+SELECT CAST(JSON_VALUE('{}', '$.a' RETURNING FLOAT
+DEFAULT 1111111111111111 ON EMPTY) AS CHAR) AS v;
+v
+1.11111e15
+SELECT CAST(JSON_VALUE('{}', '$.a' RETURNING FLOAT
+DEFAULT 1111111111111111 ON EMPTY) AS SIGNED) AS v;
+v
+1111111126482944
+SELECT CAST(JSON_VALUE('{}', '$.a' RETURNING FLOAT
+DEFAULT 1111111111111111 ON EMPTY)
+AS DECIMAL(20,2)) AS v;
+v
+1111111126482944.00
diff --git a/mysql-test/suite/json/t/json_value.test b/mysql-test/suite/json/t/json_value.test
@@ -706,6 +706,8 @@ SET TIMESTAMP=DEFAULT;
 SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('"2000-01-02 12:13:14"', '$'));
 SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
                                           '$' RETURNING DECIMAL(14, 0)));
+--echo # Warns because 20000102121314 gets truncated to fit in a FLOAT,
+--echo # and is not a valid timestamp after truncation.
 SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
                                           '$' RETURNING FLOAT));
 SELECT TIMESTAMPADD(MINUTE, 1, JSON_VALUE('20000102121314',
@@ -1193,3 +1195,26 @@ SELECT JSON_VALUE(JSON_OBJECT('a', FALSE), '$.a' RETURNING SIGNED),
 # As CHAR:
 SELECT JSON_VALUE(JSON_OBJECT('a', 'value'), '$.a'),
        JSON_VALUE(JSON_OBJECT('a', NULL), '$.a');
+
+--echo #
+--echo # Bug#34554755: DISTINCT on FLOAT values gives duplicates
+--echo #               with SQL_BIG_RESULT
+--echo #
+
+# These two integers map to the same FLOAT value, so the comparison
+# should return TRUE.
+SELECT JSON_VALUE('1111111111111111', '$' RETURNING FLOAT) =
+       JSON_VALUE('1111111111111110', '$' RETURNING FLOAT) AS eq;
+
+# Converting the FLOAT to other types should not keep the full DOUBLE precision.
+SELECT CAST(JSON_VALUE('1111111111111111', '$' RETURNING FLOAT) AS CHAR) AS v;
+SELECT CAST(JSON_VALUE('1111111111111111', '$' RETURNING FLOAT) AS SIGNED) AS v;
+SELECT CAST(JSON_VALUE('1111111111111111', '$' RETURNING FLOAT)
+            AS DECIMAL(20,2)) AS res;
+SELECT CAST(JSON_VALUE('{}', '$.a' RETURNING FLOAT
+                       DEFAULT 1111111111111111 ON EMPTY) AS CHAR) AS v;
+SELECT CAST(JSON_VALUE('{}', '$.a' RETURNING FLOAT
+                       DEFAULT 1111111111111111 ON EMPTY) AS SIGNED) AS v;
+SELECT CAST(JSON_VALUE('{}', '$.a' RETURNING FLOAT
+                       DEFAULT 1111111111111111 ON EMPTY)
+            AS DECIMAL(20,2)) AS v;
diff --git a/mysql-test/suite/rpl/r/rpl_cast.result b/mysql-test/suite/rpl/r/rpl_cast.result
@@ -33,11 +33,11 @@ id	bt	vt	tt	ft	it	itu	jt	dt	dc	dat	yt
 5	json	4827410.7404	-87484.874897447	-4873280	5398432	8549	8392	-892472.8748931	49852.53985000	2020-02-20	2020
 SELECT * FROM wl529_insert;
 f1	f2	f3	f4	f5	f6	f7	f8	f9	f10	f11
-0	837498713.84728	8240921.0982421	9385.8251953125	1234	12984	3980921	98453.8324732	4824.892470000000000000000000000000	20110403	2012.0000000000
-0	-494804.48724	48744.84783	1234.122314453125	-1234	938432	9218421	384734.84732	589239.948091000000000000000000000000	20101111	2010.0000000000
-0	7848094.874821	48973847.8743	3875329	3473	3984010	1	-8478.89247	-402984.874200000000000000000000000000	19800101	1980.0000000000
-0	908249214.8742	7634109.984304	-9824821	-98347	3814030	8742.43	-384731.837438	-4824.892470000000000000000000000000	19700101	1970.0000000000
-0	4827410.7404	-87484.874897447	-4873285	5398430	8549	8392	-892472.8748931	49852.539850000000000000000000000000	20200220	2020.0000000000
+0	837498688	8240921	9385.8251953125	1234	12984	3980921	98453.8359375	4824.892578125000000000000000000000	20110404	2012.0000000000
+0	-494804.5	48744.84765625	1234.122314453125	-1234	938432	9218421	384734.84375	589239.937500000000000000000000000000	20101112	2010.0000000000
+0	7848095	48973848	3875329	3473	3984010	1	-8478.892578125	-402984.875000000000000000000000000000	19800100	1980.0000000000
+0	908249216	7634110	-9824821	-98347	3814030	8742.4296875	-384731.84375	-4824.892578125000000000000000000000	19700100	1970.0000000000
+0	4827410.5	-87484.875	-4873285	5398430	8549	8392	-892472.875	49852.539062500000000000000000000000	20200220	2020.0000000000
 0	837498713.84728	8240921.0982421	9385.8251953125	1234	12984	3980921	98453.8324732	4824.892470000000000000000000000000	20110403	2012.0000000000
 0	-494804.48724	48744.84783	1234.122314453125	-1234	938432	9218421	384734.84732	589239.948091000000000000000000000000	20101111	2010.0000000000
 0	7848094.874821	48973847.8743	3875329	3473	3984010	1	-8478.89247	-402984.874200000000000000000000000000	19800101	1980.0000000000
diff --git a/mysql-test/t/cast.test b/mysql-test/t/cast.test
@@ -1132,3 +1132,17 @@ DROP TABLE t;
 set @a:= st_aswkb(st_astext(@b:=log10(0.0)));
 set @a:= @b;
 do convert(@a using latin1);
+
+--echo #
+--echo # Bug#34554755: DISTINCT on FLOAT values gives duplicates
+--echo #               with SQL_BIG_RESULT
+--echo #
+
+# These two integers map to the same FLOAT value, so the comparison
+# should return TRUE.
+SELECT CAST(1111111111111111 AS FLOAT) = CAST(1111111111111110 AS FLOAT) AS eq;
+
+# Converting the FLOAT to other types should not keep the full DOUBLE precision.
+SELECT CAST(CAST(1111111111111111 AS FLOAT) AS CHAR) AS v;
+SELECT CAST(CAST(1111111111111111 AS FLOAT) AS SIGNED) AS v;
+SELECT CAST(CAST(1111111111111111 AS FLOAT) AS DECIMAL(20,2)) AS v;
diff --git a/mysql-test/t/distinct.test b/mysql-test/t/distinct.test
@@ -985,7 +985,6 @@ INSERT INTO t1 VALUES
  (2007568257), (2007568260), (2007570000), (2007567234), (2007567230);
 
 SELECT SQL_SMALL_RESULT DISTINCT CAST(v AS FLOAT) FROM t1;
---skip_if_hypergraph  # Incorrect results recorded. Bug#34554755.
 SELECT SQL_BIG_RESULT DISTINCT CAST(v AS FLOAT) FROM t1;
 
 DROP TABLE t1;
diff --git a/sql/filesort.cc b/sql/filesort.cc
@@ -1356,12 +1356,6 @@ size_t make_sortkey_from_item(Item *item, Item_result result_type,
     case REAL_RESULT: {
       assert(!is_varlen);
       double value = item->val_real();
-      if (item->data_type() == MYSQL_TYPE_FLOAT) {
-        // Get rid of extra precision. Otherwise, duplicate removal may make the
-        // wrong decision and treat equal values as distinct values due to
-        // differences in the insignificant digits.
-        value = static_cast<float>(value);
-      }
       if (item->null_value) {
         assert(item->is_nullable());
         *null_indicator = 0;
diff --git a/sql/item.cc b/sql/item.cc
@@ -263,7 +263,23 @@ String *Item::val_str_ascii(String *str) {
 String *Item::val_string_from_real(String *str) {
   double nr = val_real();
   if (null_value) return nullptr; /* purecov: inspected */
-  str->set_real(nr, decimals, &my_charset_bin);
+
+  char buffer[FLOATING_POINT_BUFFER];
+  size_t len;
+  if (data_type() == MYSQL_TYPE_FLOAT) {
+    len = my_gcvt(nr, MY_GCVT_ARG_FLOAT, MAX_FLOAT_STR_LENGTH, buffer,
+                  /*error=*/nullptr);
+  } else {
+    len = my_gcvt(nr, MY_GCVT_ARG_DOUBLE, MAX_DOUBLE_STR_LENGTH, buffer,
+                  /*error=*/nullptr);
+  }
+
+  uint dummy_errors;
+  if (str->copy(buffer, len, &my_charset_numeric, collation.collation,
+                &dummy_errors)) {
+    return error_str();
+  }
+
   return str;
 }
 
diff --git a/sql/item_func.cc b/sql/item_func.cc
@@ -1994,21 +1994,20 @@ void Item_typecast_decimal::print(const THD *thd, String *str,
 }
 
 String *Item_typecast_real::val_str(String *str) {
-  double res = val_real();
-  if (null_value) return nullptr;
-
-  str->set_real(res, decimals, collation.collation);
-  return str;
+  return val_string_from_real(str);
 }
 
 double Item_typecast_real::val_real() {
   double res = args[0]->val_real();
   null_value = args[0]->null_value;
   if (null_value) return 0.0;
-  if (data_type() == MYSQL_TYPE_FLOAT &&
-      ((res > std::numeric_limits<float>::max()) ||
-       res < std::numeric_limits<float>::lowest()))
-    return raise_float_overflow();
+  if (data_type() == MYSQL_TYPE_FLOAT) {
+    if (res > std::numeric_limits<float>::max() ||
+        res < std::numeric_limits<float>::lowest()) {
+      return raise_float_overflow();
+    }
+    res = static_cast<float>(res);
+  }
   return check_float_overflow(res);
 }
 
diff --git a/sql/item_json_func.cc b/sql/item_json_func.cc
@@ -4396,7 +4396,9 @@ Item_func_json_value::create_json_value_default(THD *thd, Item *item) {
         my_error(ER_DATA_OUT_OF_RANGE, MYF(0), "FLOAT DEFAULT", func_name());
         return nullptr;
       }
-      default_value->real_default = value;
+      // The value is within range of FLOAT. Finally, cast it to float to get
+      // rid of any extra (double) precision that doesn't fit in a FLOAT.
+      default_value->real_default = static_cast<float>(value);
       break;
     }
     case ITEM_CAST_DOUBLE: {
@@ -5229,7 +5231,14 @@ double Item_func_json_value::extract_real_value() {
 
   bool err = false;
   double value = wr.coerce_real(func_name(), CE_IGNORE, &err);
-  if (!err) return value;
+  if (!err) {
+    if (data_type() == MYSQL_TYPE_FLOAT) {
+      // Remove any extra (double) precision.
+      return static_cast<float>(value);
+    } else {
+      return value;
+    }
+  }
 
   if (handle_json_value_conversion_error(
           m_on_error, data_type() == MYSQL_TYPE_DOUBLE ? "DOUBLE" : "FLOAT",
