MDEV-8613 Full table scan for WHERE indexed_varchar_column <=> 'bad-character'

Author: Alexander Barkov

mysql-test/r/range.result

@@ -2263,5 +2263,17 @@ a	b
 70	NULL
 DROP TABLE t1;
 #
+# MDEV-8613 Full table scan for WHERE indexed_varchar_column <=> 'bad-character'
+#
+SET NAMES utf8;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8, KEY(a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('a'),('b'),('c'),('d'),('e');
+EXPLAIN SELECT * FROM t1 WHERE a<=>'😎';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+Warnings:
+Warning	1366	Incorrect string value: '\xF0\x9F\x98\x8E' for column 'a' at row 1
+DROP TABLE t1;
+#
 # End of 10.1 tests
 #

mysql-test/t/range.test

@@ -1795,6 +1795,15 @@ SELECT * FROM t1 WHERE 10<>a;
 DROP TABLE t1;
 
 
+--echo #
+--echo # MDEV-8613 Full table scan for WHERE indexed_varchar_column <=> 'bad-character'
+--echo #
+SET NAMES utf8;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8, KEY(a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('a'),('b'),('c'),('d'),('e');
+EXPLAIN SELECT * FROM t1 WHERE a<=>'😎';
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

sql/opt_range.cc

@@ -7845,7 +7845,7 @@ Item_bool_func::get_mm_leaf(RANGE_OPT_PARAM *param,
   err= value->save_in_field_no_warnings(field, 1);
   if (err == 2 && field->cmp_type() == STRING_RESULT)
   {
-    if (type == EQ_FUNC)
+    if (type == EQ_FUNC || type == EQUAL_FUNC)
     {
       tree= new (alloc) SEL_ARG(field, 0, 0);
       tree->type= SEL_ARG::IMPOSSIBLE;