MDEV-8415 utf8: compare broken bytes as "greater than any non-broken character"

Author: Alexander Barkov

mysql-test/r/ctype_utf8.result

@@ -5594,11 +5594,11 @@ a?z
 ab
 az
 aЀ
+aր
 SET @query=CONCAT('SELECT ch FROM t1 WHERE ch>''a', 0xD1,''' ORDER BY ch');
 PREPARE stmt FROM @query;
 EXECUTE stmt;
 ch
-aր
 z
 # 0xEA9A96 would be a good 3-byte character, 0xEA9A is an incomplete sequence
 SET @query=CONCAT('SELECT ch FROM t1 WHERE ch=''a', 0xEA9A,''' ORDER BY ch');
@@ -5802,11 +5802,11 @@ a?z
 ab
 az
 aЀ
+aր
 SET @query=CONCAT('SELECT ch FROM t1 WHERE ch>''a', 0xD1,''' ORDER BY ch');
 PREPARE stmt FROM @query;
 EXECUTE stmt;
 ch
-aր
 z
 # 0xEA9A96 would be a good 3-byte character, 0xEA9A is an incomplete sequence
 SET @query=CONCAT('SELECT ch FROM t1 WHERE ch=''a', 0xEA9A,''' ORDER BY ch');
@@ -6002,11 +6002,11 @@ a?z
 ab
 az
 aЀ
+aր
 SET @query=CONCAT('SELECT ch FROM t1 WHERE ch>''a', 0xD1,''' ORDER BY ch');
 PREPARE stmt FROM @query;
 EXECUTE stmt;
 ch
-aր
 z
 # 0xEA9A96 would be a good 3-byte character, 0xEA9A is an incomplete sequence
 SET @query=CONCAT('SELECT ch FROM t1 WHERE ch=''a', 0xEA9A,''' ORDER BY ch');

storage/myisam/myisamdef.h

@@ -394,7 +394,7 @@ struct st_myisam_info
 #define PACK_TYPE_SELECTED      1       /* Bits in field->pack_type */
 #define PACK_TYPE_SPACE_FIELDS  2
 #define PACK_TYPE_ZERO_FILL     4
-#define MI_FOUND_WRONG_KEY 32738        /* Impossible value from ha_key_cmp */
+#define MI_FOUND_WRONG_KEY 0x7FFFFFFF   /* Impossible value from ha_key_cmp */
 
 #define MI_MAX_KEY_BLOCK_SIZE   (MI_MAX_KEY_BLOCK_LENGTH/MI_MIN_KEY_BLOCK_LENGTH)
 #define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer,block_size) (((((key_length)+(data_pointer)+(key_pointer))*4+(key_pointer)+2)/(block_size)+1)*(block_size))