MDEV-11846: ERROR 1114 (HY000) table full when performing GROUP BY

The problem is there is an overflow for the key_file_length.
Added the maximum limit for the key_file_length

storage/maria/ma_create.c

@@ -660,11 +660,24 @@ int maria_create(const char *name, enum data_file_type datafile_type,
 
     if (length > max_key_length)
       max_key_length= length;
-    tot_length+= ((max_rows/(ulong) (((uint) maria_block_size -
+
+    if (tot_length == ULLONG_MAX)
+      continue;
+
+    ulonglong tot_length_part= (max_rows/(ulong) (((uint) maria_block_size -
                                       MAX_KEYPAGE_HEADER_SIZE -
                                       KEYPAGE_CHECKSUM_SIZE)/
-                                     (length*2))) *
-                  maria_block_size);
+                                     (length*2)));
+    if (tot_length_part >=  (ULLONG_MAX / maria_block_size +
+                            ULLONG_MAX % maria_block_size))
+      tot_length= ULLONG_MAX;
+    else
+    {
+      if (tot_length > ULLONG_MAX - tot_length_part * maria_block_size)
+        tot_length= ULLONG_MAX;
+      else
+        tot_length+= tot_length_part * maria_block_size;
+    }
   }
 
   unique_key_parts=0;
@@ -673,11 +686,24 @@ int maria_create(const char *name, enum data_file_type datafile_type,
     uniquedef->key=keys+i;
     unique_key_parts+=uniquedef->keysegs;
     share.state.key_root[keys+i]= HA_OFFSET_ERROR;
-    tot_length+= (max_rows/(ulong) (((uint) maria_block_size -
+
+    if (tot_length == ULLONG_MAX)
+      continue;
+    ulonglong tot_length_part= (max_rows/(ulong) (((uint) maria_block_size -
                                      MAX_KEYPAGE_HEADER_SIZE -
                                      KEYPAGE_CHECKSUM_SIZE) /
-                         ((MARIA_UNIQUE_HASH_LENGTH + pointer)*2)))*
-                         (ulong) maria_block_size;
+                         ((MARIA_UNIQUE_HASH_LENGTH + pointer)*2)));
+
+    if (tot_length_part >=  (ULLONG_MAX / maria_block_size +
+                            ULLONG_MAX % maria_block_size))
+      tot_length= ULLONG_MAX;
+    else
+    {
+      if (tot_length > ULLONG_MAX - tot_length_part * maria_block_size)
+        tot_length= ULLONG_MAX;
+      else
+        tot_length+= tot_length_part * maria_block_size;
+    }
   }
   keys+=uniques;				/* Each unique has 1 key */
   key_segs+=uniques;				/* Each unique has 1 key seg */
@@ -746,8 +772,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
     Get estimate for index file length (this may be wrong for FT keys)
     This is used for pointers to other key pages.
   */
-  tmp= (tot_length + maria_block_size * keys *
-	MARIA_INDEX_BLOCK_MARGIN) / maria_block_size;
+  tmp= (tot_length / maria_block_size + keys * MARIA_INDEX_BLOCK_MARGIN);
 
   /*
     use maximum of key_file_length we calculated and key_file_length value we