MDEV-9602 crash in st_key::actual_rec_per_key when group by constant

Problem was that cost_group_min_max() could not handle if group by was optimized away.

Author: montywi

mysql-test/r/group_by.result

@@ -2678,3 +2678,17 @@ NULL
 100098
 100099
 drop table t0,t1,t2;
+#
+# MDEV-9602 crash in st_key::actual_rec_per_key when group by constant
+#
+create table t1 (a date not null,unique (a)) engine=innodb;
+Warnings:
+Warning	1286	Unknown storage engine 'innodb'
+Warning	1266	Using storage engine MyISAM for table 't1'
+select distinct a from t1 group by 'a';
+a
+insert into t1 values("2001-02-02"),("2001-02-03");
+select distinct a from t1 group by 'a';
+a
+2001-02-02
+drop table t1;

mysql-test/t/group_by.test

@@ -1793,3 +1793,13 @@ from t1
 group by t1.b;
 
 drop table t0,t1,t2;
+
+--echo #
+--echo # MDEV-9602 crash in st_key::actual_rec_per_key when group by constant
+--echo #
+
+create table t1 (a date not null,unique (a)) engine=innodb;
+select distinct a from t1 group by 'a';
+insert into t1 values("2001-02-02"),("2001-02-03");
+select distinct a from t1 group by 'a';
+drop table t1;

sql/opt_range.cc

@@ -13105,7 +13105,17 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts,
   num_blocks= (ha_rows)(table_records / keys_per_block) + 1;
 
   /* Compute the number of keys in a group. */
-  keys_per_group= (ha_rows) index_info->actual_rec_per_key(group_key_parts - 1);
+  if (!group_key_parts)
+  {
+    /* Summary over the whole table */
+    keys_per_group= table_records;
+  }
+  else
+  {
+    keys_per_group= (ha_rows) index_info->actual_rec_per_key(group_key_parts -
+                                                             1);
+  }
+
   if (keys_per_group == 0) /* If there is no statistics try to guess */
     /* each group contains 10% of all records */
     keys_per_group= (table_records / 10) + 1;