MDEV-18898 SELECT using wrong index when using operator IN with mixed…

… types

These patches:

  # commit 74891ed
  #
  #  MDEV-11514, MDEV-11497, MDEV-11554, MDEV-11555 - IN and CASE type aggregation problems

  # commit 53499cd
  #
  # MDEV-31303 Key not used when IN clause has both signed and usigned values

earlier fixed MDEV-18898.

Adding only an MTR case.

	modified:   mysql-test/main/func_in.result
	modified:   mysql-test/main/func_in.test

mysql-test/main/func_in.result

@@ -1003,5 +1003,59 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	range	PRIMARY	PRIMARY	8	NULL	2	Using where; Using index
 DROP TABLE t1;
 #
+# MDEV-18898 SELECT using wrong index when using operator IN with mixed types
+#
+CREATE TEMPORARY TABLE t1 (
+id int(10) unsigned NOT NULL AUTO_INCREMENT,
+name varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+PRIMARY KEY (`id`),
+UNIQUE KEY `name` (`name`)
+);
+FOR i IN 1..255
+DO
+INSERT INTO t1 VALUES (i, MD5(i));
+END FOR
+$$
+#
+# Constants alone
+#
+ANALYZE SELECT id, name FROM t1 WHERE id = 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	const	PRIMARY	PRIMARY	4	const	1	NULL	100.00	NULL	
+ANALYZE SELECT id, name FROM t1 WHERE id = '2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	const	PRIMARY	PRIMARY	4	const	1	NULL	100.00	NULL	
+#
+# Two constants using IN
+#
+ANALYZE SELECT id, name FROM t1 WHERE id IN (1, 2);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+ANALYZE SELECT id, name FROM t1 WHERE id IN ('1', 2) /* Used a wrong index */;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+ANALYZE SELECT id, name FROM t1 WHERE id IN (1, '2') /* Used a wrong index */;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+ANALYZE SELECT id, name FROM t1 WHERE id IN ('1', '2');
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+#
+# Two constants using OR
+#
+ANALYZE SELECT id, name FROM t1 WHERE id = 1 OR id = 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+ANALYZE SELECT id, name FROM t1 WHERE id = '1' OR id = '2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+ANALYZE SELECT id, name FROM t1 WHERE id = 1 OR id = '2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+ANALYZE SELECT id, name FROM t1 WHERE id = '1' OR id = 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	2	2.00	100.00	100.00	Using index condition
+DROP TABLE t1;
+#
 # End of 10.5 tests
 #

mysql-test/main/func_in.test

@@ -777,6 +777,45 @@ EXPLAIN SELECT id FROM t1 WHERE id IN (9223372036854775807, 9223372036854775808)
 DROP TABLE t1;
 
 
+--echo #
+--echo # MDEV-18898 SELECT using wrong index when using operator IN with mixed types
+--echo #
+
+CREATE TEMPORARY TABLE t1 (
+  id int(10) unsigned NOT NULL AUTO_INCREMENT,
+  name varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name` (`name`)
+);
+DELIMITER $$;
+FOR i IN 1..255
+DO
+  INSERT INTO t1 VALUES (i, MD5(i));
+END FOR
+$$
+DELIMITER ;$$
+--echo #
+--echo # Constants alone
+--echo #
+ANALYZE SELECT id, name FROM t1 WHERE id = 1;
+ANALYZE SELECT id, name FROM t1 WHERE id = '2';
+--echo #
+--echo # Two constants using IN
+--echo #
+ANALYZE SELECT id, name FROM t1 WHERE id IN (1, 2);
+ANALYZE SELECT id, name FROM t1 WHERE id IN ('1', 2) /* Used a wrong index */;
+ANALYZE SELECT id, name FROM t1 WHERE id IN (1, '2') /* Used a wrong index */;
+ANALYZE SELECT id, name FROM t1 WHERE id IN ('1', '2');
+--echo #
+--echo # Two constants using OR
+--echo #
+ANALYZE SELECT id, name FROM t1 WHERE id = 1 OR id = 2;
+ANALYZE SELECT id, name FROM t1 WHERE id = '1' OR id = '2';
+ANALYZE SELECT id, name FROM t1 WHERE id = 1 OR id = '2';
+ANALYZE SELECT id, name FROM t1 WHERE id = '1' OR id = 2;
+DROP TABLE t1;
+
+
 --echo #
 --echo # End of 10.5 tests
 --echo #