MDEV-36118 Fix wrong result with MAX in loose index scan

Loose index scan currently only supports ASC key. When searching for
the next MAX value, the search starts from the rightmost range and
moves towards the left. If the key has "moved" as the result of a
successful ha_index_read_map() call when handling a previous range, we
check if the key is to the left of the current range, and if so, we
can skip to the next range.

The existing check on whether the loop has iterated at least once is
not sufficient, as an unsuccessful ha_index_read_map() often (always?)
does not "move" the key.

Author: mariadb-YuchenPei

mysql-test/main/group_min_max.result

@@ -4349,3 +4349,15 @@ drop table t1;
 #
 # End of 10.6 tests
 #
+#
+# MDEV-36118 Wrong result in loose index scan
+#
+CREATE TABLE t1 (a int, b int, KEY (a, b));
+insert into t1 values (1, 3), (1, 1);
+SELECT MAX(b) FROM t1 WHERE (b > 2 AND b < 4) OR (b = 5) GROUP BY a;
+MAX(b)
+3
+drop table t1;
+#
+# End of 10.11 tests
+#

mysql-test/main/group_min_max.test

@@ -2007,3 +2007,29 @@ drop table t1;
 --echo #
 --echo # End of 10.6 tests
 --echo #
+
+--echo #
+--echo # MDEV-36118 Wrong result in loose index scan
+--echo #
+
+CREATE TABLE t1 (a int, b int, KEY (a, b));
+insert into t1 values (1, 3), (1, 1);
+--source include/maybe_debug.inc
+if ($have_debug) {
+  --disable_query_log
+  set @old_debug=@@debug;
+  set debug="+d,force_group_by";
+  --enable_query_log
+}
+SELECT MAX(b) FROM t1 WHERE (b > 2 AND b < 4) OR (b = 5) GROUP BY a;
+if ($have_debug) {
+  --disable_query_log
+  set debug=@old_debug;
+  --enable_query_log
+}
+
+drop table t1;
+
+--echo #
+--echo # End of 10.11 tests
+--echo #

sql/opt_range.cc

@@ -16090,7 +16090,7 @@ int QUICK_GROUP_MIN_MAX_SELECT::next_max_in_range()
   ha_rkey_function find_flag;
   key_part_map keypart_map;
   QUICK_RANGE *cur_range;
-  int result;
+  int result= HA_ERR_KEY_NOT_FOUND;
 
   DBUG_ASSERT(min_max_ranges.elements > 0);
 
@@ -16099,10 +16099,11 @@ int QUICK_GROUP_MIN_MAX_SELECT::next_max_in_range()
     get_dynamic(&min_max_ranges, (uchar*)&cur_range, range_idx - 1);
 
     /*
-      If the current value for the min/max argument is smaller than the left
-      boundary of cur_range, there is no need to check this range.
+      If the key has already been "moved" by a successful call to
+      ha_index_read_map, and the current value for the max argument
+      comes before the range, there is no need to check this range.
     */
-    if (range_idx != min_max_ranges.elements &&
+    if (!result &&
         !(cur_range->flag & NO_MIN_RANGE) &&
         (key_cmp(min_max_arg_part, (const uchar*) cur_range->min_key,
                  min_max_arg_len) == -1))