Fixed the bug mdev-7599.

At some conditions the function opt_sum_query() can apply MIN/MAX
optimizations to to Item_sum objects of a select These optimizations
becomes invalid if this select is the subquery of an IN subquery
predicate that is converted to a EXISTS subquery. Thus in this case
the MIX/MAX optimizations that have been applied in opt_sum_query()
must be rolled back.
This bug appeared in 5.3 when the code for the cost base choice between
materialization and in-to-exists transformation of non-correlated
IN subqueries was introduced. Before this code in-to-exists
transformations were always performed before the call of opt_sum_query().

mysql-test/r/subselect_mat_cost_bugs.result

@@ -379,6 +379,7 @@ drop table t3, t4, t5;
 #
 # LP BUG#858038 The result of a query with NOT IN subquery depends on the state of the optimizer switch
 #
+set @optimizer_switch_save= @@optimizer_switch;
 create table t1 (c1 char(2) not null, c2 char(2));
 create table t2 (c3 char(2),          c4 char(2));
 insert into t1 values ('a1', 'b1');
@@ -400,6 +401,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 select * from t1 where c1 = 'a2' and (c1, c2) not in (select * from t2);
 c1	c2
 drop table t1, t2;
+set optimizer_switch= @optimizer_switch_save;
 #
 # MDEV-12673: cost-based choice between materialization and in-to-exists
 #
@@ -442,3 +444,44 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 2	DEPENDENT SUBQUERY	t3	const	PRIMARY	PRIMARY	4	const	1	
 2	DEPENDENT SUBQUERY	t2	index	NULL	i2	11	NULL	2	Using where; Using index
 DROP TABLE t1,t2,t3;
+#
+# MDEV-7599: in-to-exists chosen after min/max optimization
+#
+set @optimizer_switch_save= @@optimizer_switch;
+CREATE TABLE t1 (a INT, KEY(a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT, c INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1,6),(2,4), (8,9);
+SELECT * FROM t2 WHERE b != ALL (SELECT MIN(a) FROM t1, t2 WHERE t2.c = t2.b);
+b	c
+EXPLAIN EXTENDED SELECT * FROM t2 WHERE b != ALL (SELECT MIN(a) FROM t1, t2 WHERE t2.c = t2.b);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where
+2	MATERIALIZED	t1	index	NULL	a	5	NULL	2	100.00	Using index
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where; Using join buffer (flat, BNL join)
+Warnings:
+Note	1003	select `test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` where (not(<expr_cache><`test`.`t2`.`b`>(<in_optimizer>(`test`.`t2`.`b`,`test`.`t2`.`b` in ( <materialize> (select min(`test`.`t1`.`a`) from `test`.`t1` join `test`.`t2` where (`test`.`t2`.`c` = `test`.`t2`.`b`) ), <primary_index_lookup>(`test`.`t2`.`b` in <temporary table> on distinct_key where ((`test`.`t2`.`b` = `<subquery2>`.`MIN(a)`))))))))
+set optimizer_switch= 'materialization=off';
+SELECT * FROM t2 WHERE b != ALL (SELECT MIN(a) FROM t1, t2 WHERE t2.c = t2.b);
+b	c
+EXPLAIN EXTENDED SELECT * FROM t2 WHERE b != ALL (SELECT MIN(a) FROM t1, t2 WHERE t2.c = t2.b);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where
+2	DEPENDENT SUBQUERY	t1	index	NULL	a	5	NULL	2	100.00	Using index
+2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where; Using join buffer (flat, BNL join)
+Warnings:
+Note	1003	select `test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` where (not(<expr_cache><`test`.`t2`.`b`>(<in_optimizer>(`test`.`t2`.`b`,<exists>(select min(`test`.`t1`.`a`) from `test`.`t1` join `test`.`t2` where (`test`.`t2`.`c` = `test`.`t2`.`b`) having trigcond((<cache>(`test`.`t2`.`b`) = <ref_null_helper>(min(`test`.`t1`.`a`)))))))))
+set optimizer_switch= @optimizer_switch_save;
+DROP TABLE t1,t2;
+CREATE TABLE t1 (f1 varchar(10)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('foo'),('bar');
+CREATE TABLE t2 (f2 varchar(10), key(f2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES ('baz'),('qux');
+CREATE TABLE t3 (f3 varchar(10)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES ('abc'),('def');
+SELECT * FROM t1
+WHERE f1 = ALL( SELECT MAX(t2a.f2)
+FROM t2 AS t2a INNER JOIN t2 t2b INNER JOIN t3
+ON (f3 = t2b.f2) );
+f1
+DROP TABLE t1,t2,t3;

mysql-test/t/subselect_mat_cost_bugs.test

@@ -406,6 +406,8 @@ drop table t3, t4, t5;
 --echo # LP BUG#858038 The result of a query with NOT IN subquery depends on the state of the optimizer switch
 --echo #
 
+set @optimizer_switch_save= @@optimizer_switch;
+
 create table t1 (c1 char(2) not null, c2 char(2));
 create table t2 (c3 char(2),          c4 char(2));
 
@@ -425,6 +427,8 @@ select * from t1 where c1 = 'a2' and (c1, c2) not in (select * from t2);
 
 drop table t1, t2;
 
+set optimizer_switch= @optimizer_switch_save;
+
 --echo #
 --echo # MDEV-12673: cost-based choice between materialization and in-to-exists
 --echo #
@@ -463,3 +467,43 @@ SELECT * FROM t1 WHERE i1 NOT IN (
 );
 
 DROP TABLE t1,t2,t3;
+
+--echo #
+--echo # MDEV-7599: in-to-exists chosen after min/max optimization
+--echo #
+
+set @optimizer_switch_save= @@optimizer_switch;
+
+CREATE TABLE t1 (a INT, KEY(a)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+
+CREATE TABLE t2 (b INT, c INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1,6),(2,4), (8,9);
+
+let $q=
+SELECT * FROM t2 WHERE b != ALL (SELECT MIN(a) FROM t1, t2 WHERE t2.c = t2.b);
+
+eval $q;
+eval EXPLAIN EXTENDED $q;
+set optimizer_switch= 'materialization=off';
+eval $q;
+eval EXPLAIN EXTENDED $q;
+set optimizer_switch= @optimizer_switch_save;
+
+DROP TABLE t1,t2;
+
+CREATE TABLE t1 (f1 varchar(10)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('foo'),('bar');
+
+CREATE TABLE t2 (f2 varchar(10), key(f2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES ('baz'),('qux');
+
+CREATE TABLE t3 (f3 varchar(10)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES ('abc'),('def');
+
+SELECT * FROM t1
+  WHERE f1 = ALL( SELECT MAX(t2a.f2)
+                  FROM t2 AS t2a INNER JOIN t2 t2b INNER JOIN t3
+                  ON (f3 = t2b.f2) );
+
+DROP TABLE t1,t2,t3;

sql/item_subselect.cc

@@ -2493,6 +2493,27 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg)
   DBUG_ENTER("Item_in_subselect::inject_in_to_exists_cond");
   DBUG_ASSERT(thd == join_arg->thd);
 
+  if (select_lex->min_max_opt_list.elements)
+  {
+    /*
+      MIN/MAX optimizations have been applied to Item_sum objects
+      of the subquery this subquery predicate in opt_sum_query().
+      Injection of new condition invalidates this optimizations.
+      Thus those optimizations must be rolled back.
+    */
+    List_iterator_fast<Item_sum> it(select_lex->min_max_opt_list);
+    Item_sum *item;
+    while ((item= it++))
+    {
+      item->clear();
+      item->reset_forced_const();
+    }
+    if (where_item)
+      where_item->update_used_tables();
+    if (having_item)
+      having_item->update_used_tables();
+  }
+
   if (where_item)
   {
     List<Item> *and_args= NULL;

sql/item_sum.h

@@ -454,6 +454,7 @@ class Item_sum :public Item_result_field
     used_tables_cache= 0; 
     forced_const= TRUE; 
   }
+  void reset_forced_const() { forced_const= FALSE; }
   virtual bool const_item() const { return forced_const; }
   virtual bool const_during_execution() const { return false; }
   virtual void print(String *str, enum_query_type query_type);

sql/opt_sum.cc

@@ -253,6 +253,8 @@ int opt_sum_query(THD *thd,
   int error= 0;
   DBUG_ENTER("opt_sum_query");
 
+  thd->lex->current_select->min_max_opt_list.empty();
+
   if (conds)
     where_tables= conds->used_tables();
 
@@ -444,7 +446,14 @@ int opt_sum_query(THD *thd,
           item_sum->aggregator_clear();
         }
         else
+        {
           item_sum->reset_and_add();
+          /*
+            Save a reference to the item for possible rollback
+            of the min/max optimizations for this select
+          */
+	  thd->lex->current_select->min_max_opt_list.push_back(item_sum);
+        }
         item_sum->make_const();
         recalc_const_item= 1;
         break;

sql/sql_lex.cc

@@ -1875,6 +1875,7 @@ void st_select_lex::init_query()
   leaf_tables_prep.empty();
   leaf_tables.empty();
   item_list.empty();
+  min_max_opt_list.empty();
   join= 0;
   having= prep_having= where= prep_where= 0;
   olap= UNSPECIFIED_OLAP_TYPE;

sql/sql_lex.h

@@ -766,6 +766,11 @@ class st_select_lex: public st_select_lex_node
   */
   List<Item_func_match> *ftfunc_list;
   List<Item_func_match> ftfunc_list_alloc;
+  /*
+    The list of items to which MIN/MAX optimizations of opt_sum_query()
+    have been applied. Used to rollback those optimizations if it's needed.
+  */
+  List<Item_sum> min_max_opt_list;
   JOIN *join; /* after JOIN::prepare it is pointer to corresponding JOIN */
   List<TABLE_LIST> top_join_list; /* join list of the top level          */
   List<TABLE_LIST> *join_list;    /* list for the currently parsed join  */