MDEV-8921 Wrong result for CAST(AVG(double_column) AS SIGNED)

Author: Alexander Barkov

mysql-test/r/func_group.result

@@ -2298,5 +2298,21 @@ id	avg	cast_avg
 2	9223372036854775807.0000	9223372036854775807
 DROP TABLE t1;
 #
+# MDEV-8921 Wrong result for CAST(AVG(double_column) AS SIGNED)
+#
+CREATE TABLE t1 (id INT, a DOUBLE);
+INSERT INTO t1 VALUES (1,0x7FFFFFFFFFFFFFFF),(2,0x7FFFFFFFFFFFFFFF);
+SELECT id, AVG(a) AS avg, CAST(MIN(a) AS SIGNED) AS cast_min,CAST(AVG(a) AS SIGNED) AS cast_avg FROM t1 GROUP BY id HAVING avg!=123 ORDER BY id;
+id	avg	cast_min	cast_avg
+1	9.223372036854776e18	9223372036854775807	9223372036854775807
+2	9.223372036854776e18	9223372036854775807	9223372036854775807
+DROP TABLE t1;
+CREATE TABLE t1 (a DOUBLE);
+INSERT INTO t1 VALUES (0x7FFFFFFFFFFFFFFF);
+SELECT MIN(a), SUM(a), CAST(SUM(a) AS SIGNED), CAST(AVG(a) AS SIGNED) FROM t1;
+MIN(a)	SUM(a)	CAST(SUM(a) AS SIGNED)	CAST(AVG(a) AS SIGNED)
+9.223372036854776e18	9.223372036854776e18	9223372036854775807	9223372036854775807
+DROP TABLE t1;
+#
 # End of 10.1 tests
 #

mysql-test/t/func_group.test

@@ -1581,6 +1581,18 @@ SELECT id, AVG(a) AS avg, CAST(MIN(a) AS SIGNED) AS cast_min FROM t1 GROUP BY id
 SELECT id, AVG(a) AS avg, CAST(AVG(a) AS SIGNED) AS cast_avg FROM t1 GROUP BY id HAVING avg!=123 ORDER BY id;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-8921 Wrong result for CAST(AVG(double_column) AS SIGNED)
+--echo #
+CREATE TABLE t1 (id INT, a DOUBLE);
+INSERT INTO t1 VALUES (1,0x7FFFFFFFFFFFFFFF),(2,0x7FFFFFFFFFFFFFFF);
+SELECT id, AVG(a) AS avg, CAST(MIN(a) AS SIGNED) AS cast_min,CAST(AVG(a) AS SIGNED) AS cast_avg FROM t1 GROUP BY id HAVING avg!=123 ORDER BY id;
+DROP TABLE t1;
+CREATE TABLE t1 (a DOUBLE);
+INSERT INTO t1 VALUES (0x7FFFFFFFFFFFFFFF);
+SELECT MIN(a), SUM(a), CAST(SUM(a) AS SIGNED), CAST(AVG(a) AS SIGNED) FROM t1;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

sql/item.cc

@@ -359,6 +359,14 @@ longlong Item::val_int_from_date()
 }
 
 
+longlong Item::val_int_from_real()
+{
+  DBUG_ASSERT(fixed == 1);
+  bool error;
+  return double_to_longlong(val_real(), false /*unsigned_flag*/, &error);
+}
+
+
 double Item::val_real_from_date()
 {
   DBUG_ASSERT(fixed == 1);

sql/item.h

@@ -1008,6 +1008,7 @@ class Item: public Value_source,
   my_decimal *val_decimal_from_time(my_decimal *decimal_value);
   longlong val_int_from_decimal();
   longlong val_int_from_date();
+  longlong val_int_from_real();
   double val_real_from_decimal();
   double val_real_from_date();
 

sql/item_sum.cc

@@ -1430,7 +1430,7 @@ longlong Item_sum_sum::val_int()
                    &result);
     return result;
   }
-  return (longlong) rint(val_real());
+  return val_int_from_real();
 }
 
 

sql/item_sum.h

@@ -704,11 +704,7 @@ class Item_sum_num :public Item_sum
   Item_sum_num(THD *thd, Item_sum_num *item):
     Item_sum(thd, item),is_evaluated(item->is_evaluated) {}
   bool fix_fields(THD *, Item **);
-  longlong val_int()
-  {
-    DBUG_ASSERT(fixed == 1);
-    return (longlong) rint(val_real());             /* Real as default */
-  }
+  longlong val_int() { return val_int_from_real();  /* Real as default */ }
   String *val_str(String*str);
   my_decimal *val_decimal(my_decimal *);
   void reset_field();
@@ -843,7 +839,7 @@ class Item_sum_avg :public Item_sum_sum
   bool add();
   double val_real();
   // In SPs we might force the "wrong" type with select into a declare variable
-  longlong val_int() { return (longlong) rint(val_real()); }
+  longlong val_int() { return val_int_from_real(); }
   my_decimal *val_decimal(my_decimal *);
   String *val_str(String *str);
   void reset_field();
@@ -1093,6 +1089,7 @@ class Item_sum_field :public Item
     decimals= item->decimals;
     max_length= item->max_length;
     unsigned_flag= item->unsigned_flag;
+    fixed= true;
   }
   table_map used_tables() const { return (table_map) 1L; }
   Field *get_tmp_table_field() { DBUG_ASSERT(0); return NULL; }
@@ -1127,7 +1124,7 @@ class Item_avg_field_double :public Item_avg_field
   { }
   enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; }
   enum Item_result result_type () const { return REAL_RESULT; }
-  longlong val_int() { return (longlong) rint(val_real()); }
+  longlong val_int() { return val_int_from_real(); }
   my_decimal *val_decimal(my_decimal *dec) { return val_decimal_from_real(dec); }
   String *val_str(String *str) { return val_string_from_real(str); }
   double val_real();
@@ -1162,8 +1159,7 @@ class Item_variance_field :public Item_sum_field
   { }
   enum Type type() const {return FIELD_VARIANCE_ITEM; }
   double val_real();
-  longlong val_int()
-  { /* can't be fix_fields()ed */ return (longlong) rint(val_real()); }
+  longlong val_int() { return val_int_from_real(); }
   String *val_str(String *str)
   { return val_string_from_real(str); }
   my_decimal *val_decimal(my_decimal *dec_buf)
@@ -1246,11 +1242,7 @@ class Item_sum_udf_float :public Item_udf_sum
     Item_udf_sum(thd, udf_arg, list) {}
   Item_sum_udf_float(THD *thd, Item_sum_udf_float *item)
     :Item_udf_sum(thd, item) {}
-  longlong val_int()
-  {
-    DBUG_ASSERT(fixed == 1);
-    return (longlong) rint(Item_sum_udf_float::val_real());
-  }
+  longlong val_int() { return val_int_from_real(); }
   double val_real();
   String *val_str(String*str);
   my_decimal *val_decimal(my_decimal *);