Bug#36563773 MYSQL SERVER 8.3.0 GLOBAL-BUFFER-OVERFLOW AT 'DECIMAL_BIN_SIZE_INLINE'

Limit the scale used when computing averages of decimal numbers.

Also verify that we do not set precision > DECIMAL_MAX_PRECISION

Change-Id: If32143ad7ed55f841e3973e78fa8e98cc245e775

Author: Tor Didriksen

sql/item.h

@@ -1569,6 +1569,7 @@ class Item : public Parse_tree_node {
   inline void set_data_type_decimal(uint8 precision, uint8 scale) {
     set_data_type(MYSQL_TYPE_NEWDECIMAL);
     collation.set_numeric();
+    assert(precision <= DECIMAL_MAX_PRECISION);
     decimals = scale;
     fix_char_length(my_decimal_precision_to_length_no_truncation(
         precision, scale, unsigned_flag));

sql/item_func.cc

@@ -2784,7 +2784,8 @@ bool Item_func_neg::resolve_type(THD *thd) {
         the negated number
       */
       unsigned_flag = false;
-      set_data_type_decimal(args[0]->decimal_precision(), 0);
+      set_data_type_decimal(
+          min<uint>(args[0]->decimal_precision(), DECIMAL_MAX_PRECISION), 0);
       hybrid_type = DECIMAL_RESULT;
       DBUG_PRINT("info", ("Type changed: DECIMAL_RESULT"));
     }
@@ -3351,6 +3352,7 @@ bool Item_func_int_val::resolve_type_inner(THD *) {
       // order of magnitude.
       int precision = args[0]->decimal_precision() - args[0]->decimals;
       if (args[0]->decimals != 0) ++precision;
+      precision = std::min(precision, DECIMAL_MAX_PRECISION);
       set_data_type_decimal(precision, 0);
       hybrid_type = DECIMAL_RESULT;
 
@@ -3508,6 +3510,7 @@ bool Item_func_round::resolve_type(THD *thd) {
         new_scale = val1;
       }
       if (precision == 0) precision = 1;
+      precision = min<uint>(precision, DECIMAL_MAX_PRECISION);
       set_data_type_decimal(precision, new_scale);
       hybrid_type = DECIMAL_RESULT;
       break;
@@ -6100,7 +6103,9 @@ bool Item_func_set_user_var::resolve_type(THD *thd) {
           args[0]->max_length;  // Preserves "length" of integer constants
       break;
     case MYSQL_TYPE_NEWDECIMAL:
-      set_data_type_decimal(args[0]->decimal_precision(), args[0]->decimals);
+      set_data_type_decimal(
+          min<uint>(args[0]->decimal_precision(), DECIMAL_MAX_PRECISION),
+          args[0]->decimals);
       break;
     case MYSQL_TYPE_DOUBLE:
       set_data_type_double();

sql/item_json_func.cc

@@ -3373,7 +3373,8 @@ static void set_data_type_from_cast_type(Item *item, Cast_target cast_type,
       item->set_data_type_datetime(decimals);
       return;
     case ITEM_CAST_DECIMAL:
-      item->set_data_type_decimal(length, decimals);
+      item->set_data_type_decimal(
+          std::min<unsigned>(length, DECIMAL_MAX_PRECISION), decimals);
       return;
     case ITEM_CAST_CHAR:
       // If no character set is specified, the JSON default character set is

sql/item_sum.cc

@@ -1929,7 +1929,8 @@ bool Item_sum_sum::resolve_type(THD *thd) {
     case DECIMAL_RESULT: {
       // SUM result cannot be longer than length(arg) + length(MAX_ROWS)
       const int precision =
-          args[0]->decimal_precision() + DECIMAL_LONGLONG_DIGITS;
+          min<uint>(args[0]->decimal_precision() + DECIMAL_LONGLONG_DIGITS,
+                    DECIMAL_MAX_PRECISION);
       set_data_type_decimal(precision, args[0]->decimals);
       curr_dec_buff = 0;
       my_decimal_set_zero(dec_buffs);
@@ -2264,13 +2265,14 @@ bool Item_sum_avg::resolve_type(THD *thd) {
   null_value = true;
   prec_increment = thd->variables.div_precincrement;
   if (hybrid_type == DECIMAL_RESULT) {
-    const int precision = args[0]->decimal_precision() + prec_increment;
+    const int precision = min<uint>(
+        args[0]->decimal_precision() + prec_increment, DECIMAL_MAX_PRECISION);
     int scale =
         min<uint>(args[0]->decimals + prec_increment, DECIMAL_MAX_SCALE);
     set_data_type_decimal(precision, scale);
     f_precision =
         min(precision + DECIMAL_LONGLONG_DIGITS, DECIMAL_MAX_PRECISION);
-    f_scale = args[0]->decimals;
+    f_scale = min<uint>(args[0]->decimals, f_precision);
     dec_bin_size = my_decimal_get_binary_size(f_precision, f_scale);
   } else {
     assert(hybrid_type == REAL_RESULT);