MDEV-33534 UBSAN: Negation of -X cannot be represented in type 'long …

…long int'; cast to an unsigned type to negate this value to itself in my_double_round from sql/item_func.cc|

The negation in this line:
  ulonglong abs_dec= dec_negative ? -dec : dec;
did not take into account that 'dec' can be the smallest possible
signed negative value -9223372036854775808. Its negation is
an operation with an undefined behavior.

Fixing the code to use Longlong_hybrid, which implements a safe
method to get an absolute value.

mysql-test/main/func_math.result

@@ -3617,5 +3617,20 @@ SELECT CRC32(ExtractValue('<a><b/></a>', '/a/b')) AS f;
 f
 0
 #
+# MDEV-33534 UBSAN: Negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_double_round from sql/item_func.cc|
+#
+SELECT TRUNCATE(EXP(-1.e-2),-1.e+30) AS c1;
+c1
+0
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) AS c1;
+c1
+0
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+c1
+0
+SELECT(ASIN(-1)+ LN(-1)) % (ATAN(-1) MOD FLOOR(1)) * (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+c1
+NULL
+#
 # End of 10.5 tests
 #

mysql-test/main/func_math.test

@@ -1929,6 +1929,16 @@ DROP TABLE t2, t1;
 SELECT CRC32(ExtractValue('<a><b/></a>', '/a/b')) AS f;
 
 
+--echo #
+--echo # MDEV-33534 UBSAN: Negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_double_round from sql/item_func.cc|
+--echo #
+
+SELECT TRUNCATE(EXP(-1.e-2),-1.e+30) AS c1;
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) AS c1;
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+SELECT(ASIN(-1)+ LN(-1)) % (ATAN(-1) MOD FLOOR(1)) * (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

sql/item_func.cc

@@ -2638,12 +2638,12 @@ void Item_func_round::fix_arg_hex_hybrid()
 }
 
 
-double my_double_round(double value, longlong dec, bool dec_unsigned,
+double my_double_round(double value, longlong dec_value, bool dec_unsigned,
                        bool truncate)
 {
   double tmp;
-  bool dec_negative= (dec < 0) && !dec_unsigned;
-  ulonglong abs_dec= dec_negative ? -dec : dec;
+  const Longlong_hybrid dec(dec_value, dec_unsigned);
+  const ulonglong abs_dec= dec.abs();
   /*
     tmp2 is here to avoid return the value with 80 bit precision
     This will fix that the test round(0.1,1) = round(0.1,1) is true
@@ -2658,22 +2658,24 @@ double my_double_round(double value, longlong dec, bool dec_unsigned,
   volatile double value_div_tmp= value / tmp;
   volatile double value_mul_tmp= value * tmp;
 
-  if (!dec_negative && std::isinf(tmp)) // "dec" is too large positive number
+  if (!dec.neg() && std::isinf(tmp)) // "dec" is a too large positive number
     return value;
 
-  if (dec_negative && std::isinf(tmp))
-    tmp2= 0.0;
-  else if (!dec_negative && std::isinf(value_mul_tmp))
+  if (dec.neg() && std::isinf(tmp)) // "dec" is a too small negative number
+    return 0.0;
+
+  // Handle "dec" with a reasonably small absolute value
+  if (!dec.neg() && std::isinf(value_mul_tmp))
     tmp2= value;
   else if (truncate)
   {
     if (value >= 0.0)
-      tmp2= dec < 0 ? floor(value_div_tmp) * tmp : floor(value_mul_tmp) / tmp;
+      tmp2= dec.neg() ? floor(value_div_tmp) * tmp : floor(value_mul_tmp) / tmp;
     else
-      tmp2= dec < 0 ? ceil(value_div_tmp) * tmp : ceil(value_mul_tmp) / tmp;
+      tmp2= dec.neg() ? ceil(value_div_tmp) * tmp : ceil(value_mul_tmp) / tmp;
   }
   else
-    tmp2=dec < 0 ? rint(value_div_tmp) * tmp : rint(value_mul_tmp) / tmp;
+    tmp2=dec.neg() ? rint(value_div_tmp) * tmp : rint(value_mul_tmp) / tmp;
 
   return tmp2;
 }