MDEV-22935 Erroneous Aria Index / Optimizer behaviour

The problem was in the Aria part of the range optimizer,
maria_records_in_range(), which wrong concluded that there was no rows
in the range.

This error would happen in the unlikely case when searching for a range
on a partial key and there was a match for the first key part in the
upper part of the b-tree (node) and also a match in the underlying
node page.

In other words, for this bug to happen one have to use Aria, have a multi
part key with a lot of identical values for the first key part and do a
range search on the second part of the key.

Fixed by ensuring that we do not stop searching for partial keys found
on node.

Other things:
- Added some comments
- Changed a variable name to more clearly explain it's purpose.
- Fixed wrong cast in _ma_record_pos() that could cause problems on 32 bit
  systems.

Author: montywi

mysql-test/suite/maria/range.result

@@ -0,0 +1,6 @@
+#
+# MDEV-22935 Erroneous Aria Index / Optimizer behaviour
+#
+create table t1 (a char(255), b datetime, primary key(a,b)) engine=aria transactional=0 pack_keys=0;
+insert into t1 select concat("hello world hello world", truncate(seq/100,0)),from_unixtime(seq+1) from seq_1_to_20000;
+drop table t1;

mysql-test/suite/maria/range.test

@@ -0,0 +1,22 @@
+--source include/have_sequence.inc
+
+--echo #
+--echo # MDEV-22935 Erroneous Aria Index / Optimizer behaviour
+--echo #
+
+create table t1 (a char(255), b datetime, primary key(a,b)) engine=aria transactional=0 pack_keys=0;
+insert into t1 select concat("hello world hello world", truncate(seq/100,0)),from_unixtime(seq+1) from seq_1_to_20000;
+
+let $i= 200;
+--disable_query_log
+while ($i)
+{
+  let $tmp= `select count(*) from t1 where a="hello world hello world$i" and b <= from_unixtime($i*100+1)`;
+  if (`SELECT $tmp != 1`)
+  {
+    --echo "Found $tmp rows, expected 1, for value $i"
+  }  
+  dec $i;
+}
+--enable_query_log
+drop table t1;

storage/maria/ma_range.c

@@ -191,8 +191,8 @@ static ha_rows _ma_record_pos(MARIA_HA *info, const uchar *key_data,
                       info->s->state.key_root[inx], final_page);
   if (pos >= 0.0)
   {
-    DBUG_PRINT("exit",("pos: %ld",(ulong) (pos*info->state->records)));
-    DBUG_RETURN((ulong) (pos*info->state->records+0.5));
+    DBUG_PRINT("exit",("pos: %lld",(longlong) (pos*info->state->records)));
+    DBUG_RETURN((ha_rows) (pos*info->state->records+0.5));
   }
   DBUG_RETURN(HA_POS_ERROR);
 }
@@ -214,7 +214,7 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key,
 {
   int flag;
   uint keynr, UNINIT_VAR(max_keynr);
-  my_bool after_key;
+  my_bool last_key_on_page;
   uchar *keypos;
   double offset;
   MARIA_KEYDEF *keyinfo= key->keyinfo;
@@ -230,7 +230,7 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key,
     goto err;
   *final_page= pos;
   flag= (*keyinfo->bin_search)(key, &page, nextflag, &keypos,
-                               info->lastkey_buff, &after_key);
+                               info->lastkey_buff, &last_key_on_page);
   keynr= _ma_keynr(&page, keypos, &max_keynr);
 
   if (flag)
@@ -274,7 +274,7 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key,
         There may be identical keys in the tree. Try to match on of those.
         Matches keynr + [0-1]
       */
-      if ((offset= _ma_search_pos(info, key, SEARCH_FIND,
+      if ((offset= _ma_search_pos(info, key, nextflag,
                                   _ma_kpos(page.node,keypos),
                                   final_page)) < 0)
 	DBUG_RETURN(offset);			/* Read error */
@@ -290,9 +290,10 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key,
 
 
 /*
-  Get keynummer of current key and max number of keys in nod
+  Get keynumber of current key and max number of keys in nod
 
-  keynr >= 0 && key_nr <= max_key
+  @return key position on page (0 - (ret_max_key - 1))
+          ret_max_key contains how many keys there was on the page
 */
 
 static uint _ma_keynr(MARIA_PAGE *page, uchar *keypos, uint *ret_max_key)