Bug #25989915: LOOSE INDEX SCANS RETURNING WRONG RESULT

Problem:
--------
This problem occurs under the following conditions:
1) Table has a unique index.
2) Query checks whether a loose index scan is applicable to
this query. And mistakenly chooses it despite unique
indexes having no index extensions.

This is can give wrong results. A similar problem also
exists in Bug #26532061.

Solution:
---------
Index extensions are not applicable to UNIQUE indexes for
loose index scans.

So Field::is_part_of_actual_key should also consider the
HA_NOSAME flag.

mysql-test/r/group_min_max_innodb.result

@@ -559,3 +559,30 @@ c1
 2
 10
 DROP TABLE t1;
+#
+# Bug #25989915: LOOSE INDEX SCANS RETURNING WRONG RESULT
+#
+CREATE TABLE t1 (
+pk INT NOT NULL AUTO_INCREMENT,
+c1 varchar(100) DEFAULT NULL,
+c2 INT NOT NULL,
+PRIMARY KEY (pk),
+UNIQUE KEY ukey (c2,c1)
+);
+INSERT INTO t1(pk, c2) VALUES (100, 0), (101, 0), (102, 0), (103, 0);
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+EXPLAIN SELECT COUNT(DISTINCT(c2)) FROM t1 WHERE pk IN (102, 101);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	PRIMARY,ukey	ukey	107	NULL	4	Using where; Using index
+EXPLAIN SELECT COUNT(DISTINCT(c2)) FROM t1 WHERE pk IN (102, 100);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	PRIMARY,ukey	ukey	107	NULL	4	Using where; Using index
+SELECT COUNT(DISTINCT(c2)) FROM t1 WHERE pk IN (102, 101);
+COUNT(DISTINCT(c2))
+1
+SELECT COUNT(DISTINCT(c2)) FROM t1 WHERE pk IN (102, 100);
+COUNT(DISTINCT(c2))
+1
+DROP TABLE t1;

mysql-test/t/group_min_max_innodb.test

@@ -419,3 +419,29 @@ eval $query4;
 eval $query5;
 
 DROP TABLE t1;
+
+--echo #
+--echo # Bug #25989915: LOOSE INDEX SCANS RETURNING WRONG RESULT
+--echo #
+
+CREATE TABLE t1 (
+  pk INT NOT NULL AUTO_INCREMENT,
+  c1 varchar(100) DEFAULT NULL,
+  c2 INT NOT NULL,
+  PRIMARY KEY (pk),
+  UNIQUE KEY ukey (c2,c1)
+);
+
+INSERT INTO t1(pk, c2) VALUES (100, 0), (101, 0), (102, 0), (103, 0);
+ANALYZE TABLE t1;
+
+let query1= SELECT COUNT(DISTINCT(c2)) FROM t1 WHERE pk IN (102, 101);
+let query2= SELECT COUNT(DISTINCT(c2)) FROM t1 WHERE pk IN (102, 100);
+
+eval EXPLAIN $query1;
+eval EXPLAIN $query2;
+
+eval $query1;
+eval $query2;
+
+DROP TABLE t1;

sql/field.cc

@@ -10616,9 +10616,11 @@ Field_temporal::set_datetime_warning(Sql_condition::enum_warning_level level,
     make_truncated_value_warning(thd, level, val, ts_type, field_name);
 }
 
-bool Field::is_part_of_actual_key(THD *thd, uint cur_index)
+bool Field::is_part_of_actual_key(THD *thd, uint cur_index, KEY *cur_index_info)
 {
-  return thd->optimizer_switch_flag(OPTIMIZER_SWITCH_USE_INDEX_EXTENSIONS) ?
+  return
+    thd->optimizer_switch_flag(OPTIMIZER_SWITCH_USE_INDEX_EXTENSIONS) &&
+    !(cur_index_info->flags & HA_NOSAME) ?
     part_of_key.is_set(cur_index) :
     part_of_key_not_extended.is_set(cur_index);
 }

sql/field.h

@@ -1,7 +1,7 @@
 #ifndef FIELD_INCLUDED
 #define FIELD_INCLUDED
 
-/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1254,16 +1254,18 @@ class Field
 
   /**
     Check whether field is part of the index taking the index extensions flag
-    into account.
+    into account. Index extensions are also not applicable to UNIQUE indexes
+    for loose index scans.
 
     @param[in]     thd             THD object
     @param[in]     cur_index       Index of the key
+    @param[in]     cur_index_info  key_info object
 
     @retval true  Field is part of the key
     @retval false otherwise
 
   */
-  bool is_part_of_actual_key(THD *thd, uint cur_index);
+  bool is_part_of_actual_key(THD *thd, uint cur_index, KEY *cur_index_info);
 
   friend int cre_myisam(char * name, register TABLE *form, uint options,
 			ulonglong auto_increment_value);

sql/opt_range.cc

@@ -11619,7 +11619,7 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time)
           part of 'cur_index'
         */
         if (bitmap_is_set(table->read_set, cur_field->field_index) &&
-            !cur_field->is_part_of_actual_key(thd, cur_index))
+            !cur_field->is_part_of_actual_key(thd, cur_index, cur_index_info))
         {
           cause= "not_covering";
           goto next_index;                  // Field was not part of key