Bug#29897573: A FULL TABLE AGGREGATE QUERY IN DERIVED TABLE DOES NOT OFFLOAD

Bug#29994087: SIG11 AT MATERIALIZEITERATOR::INIT() IN SQL/COMPOSITE_ITERATORS.CC

Some queries that contained subqueries were not offloaded to a
secondary storage engine. The optimizer tried to evaluate the
subqueries during optimization, and this failed because the secondary
engine could not return results until execution. This is fixed by
preventing evaluation of subqueries during optimization when using a
secondary storage engine.

Add a new query option that secondary storage engines can set to
prevent subqueries from being evaluated during optimization. The
option is checked before evaluating subqueries in the optimizer.

Setting of query options for secondary storage engines is removed from
open_secondary_engine_tables(). Instead, the engines can set the
options they need in their implementation of
handlerton::prepare_secondary_engine().

Item_bool_func2::update_used_tables() didn't properly accumulate
properties from the ESCAPE clause of a LIKE expression, so that it was
not detected that a LIKE expression contained a subquery in some
cases. A specialization of the function is added in Item_func_like to
make sure properties of the ESCAPE clause are accumulated in the LIKE
expression.

The test secondary_engine.no_constant_tables is removed, as it no
longer exercises the code that it was supposed to test. The code is
covered by tests in other test suites.

Change-Id: I945df19452cbca0d5f5621f0bb68351009c9e79c

Author: kahatlen

mysql-test/suite/secondary_engine/r/no_constant_tables.result

@@ -1127,3 +1127,13 @@ id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered
 Warnings:
 Note	1003	/* select#1 */ select `tt1`.`x` AS `x`,`tt1`.`y` AS `y`,`tt1`.`z` AS `z`,`tt1`.`c` AS `c`,`tt1`.`d` AS `d`,`tt2`.`x` AS `x`,`tt2`.`y` AS `y`,`tt2`.`z` AS `z`,`tt2`.`c` AS `c`,`tt2`.`d` AS `d` from (/* select#2 */ select `test`.`t1`.`x` AS `x`,`test`.`t1`.`y` AS `y`,`test`.`t1`.`z` AS `z`,`test`.`t1`.`c` AS `c`,`test`.`t1`.`d` AS `d` from `test`.`t1` where (`test`.`t1`.`x` < 7) limit 20) `tt1` join (/* select#3 */ select `test`.`t1`.`x` AS `x`,`test`.`t1`.`y` AS `y`,`test`.`t1`.`z` AS `z`,`test`.`t1`.`c` AS `c`,`test`.`t1`.`d` AS `d` from `test`.`t1` where (`test`.`t1`.`x` < `test`.`t1`.`y`) limit 20) `tt2` where ((`tt2`.`x` = `tt1`.`x`) and (`tt1`.`x` > 5))
 DROP TABLE t1;
+#
+# Bug#29994087: SIG11 AT MATERIALIZEITERATOR::INIT() IN
+#               SQL/COMPOSITE_ITERATORS.CC
+#
+CREATE TABLE t (a INT, b INT) SECONDARY_ENGINE MOCK;
+ALTER TABLE t SECONDARY_LOAD;
+SELECT 1 FROM t
+WHERE (1, 2) IN (SELECT 3, 4 UNION SELECT 5, 6) OR b <= 10;
+1
+DROP TABLE t;

mysql-test/suite/secondary_engine/r/query_preparation.result

@@ -739,6 +739,16 @@ WHERE tt1.x> 5 and tt1.x=tt2.x;
 
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#29994087: SIG11 AT MATERIALIZEITERATOR::INIT() IN
+--echo #               SQL/COMPOSITE_ITERATORS.CC
+--echo #
+CREATE TABLE t (a INT, b INT) SECONDARY_ENGINE MOCK;
+ALTER TABLE t SECONDARY_LOAD;
+SELECT 1 FROM t
+WHERE (1, 2) IN (SELECT 3, 4 UNION SELECT 5, 6) OR b <= 10;
+DROP TABLE t;
+
 --disable_query_log
 UNINSTALL PLUGIN mock;
 --enable_query_log

mysql-test/suite/secondary_engine/t/no_constant_tables.test

@@ -5097,7 +5097,7 @@ bool Item_cond::fix_fields(THD *thd, Item **ref) {
       Do this optimization if fix_fields is allowed to change the condition
       and if this is the first execution.
       Check if the const item does not contain param's, SP args etc.  We also
-      cannot optimize conditions if its a view. The condition has to be a
+      cannot optimize conditions if it's a view. The condition has to be a
       top_level_item to get optimized as they can have only two return values,
       true or false. A non-top_level_item can have true, false and NULL return.
       Fulltext funcs cannot be removed as ftfunc_list stores the list
@@ -5821,8 +5821,16 @@ longlong Item_func_like::val_int() {
 */
 
 Item_func::optimize_type Item_func_like::select_optimize(const THD *thd) {
+  /*
+    Can be called both during preparation (from prune_partitions()) and
+    optimization. Check if the pattern can be evaluated in the current phase.
+  */
   if (!args[1]->may_evaluate_const(thd)) return OPTIMIZE_NONE;
 
+  // Don't evaluate the pattern if evaluation during optimization is disabled.
+  if (!evaluate_during_optimization(args[1], thd->lex->current_select()))
+    return OPTIMIZE_NONE;
+
   String *res2 = args[1]->val_str(&cmp.value2);
   if (!res2) return OPTIMIZE_NONE;
 
@@ -5961,6 +5969,13 @@ bool Item_func_like::eval_escape_clause(THD *thd) {
   return false;
 }
 
+void Item_func_like::update_used_tables() {
+  Item_bool_func2::update_used_tables();
+  escape_item->update_used_tables();
+  used_tables_cache |= escape_item->used_tables();
+  add_accum_properties(escape_item);
+}
+
 bool Item_func_xor::itemize(Parse_context *pc, Item **res) {
   if (skip_itemize(res)) return false;
   if (super::itemize(pc, res)) return true;

mysql-test/suite/secondary_engine/t/query_preparation.test

@@ -2193,6 +2193,8 @@ class Item_func_like final : public Item_bool_func2 {
                              const MY_BITMAP *fields_to_ignore,
                              double rows_in_table) override;
 
+  void update_used_tables() override;
+
  private:
   /**
     The method updates covering keys depending on the

sql/item_cmpfunc.cc

@@ -7244,6 +7244,17 @@ static bool save_value_and_handle_conversion(SEL_ROOT **tree, Item *value,
     return true;
   }
 
+  /*
+    Don't evaluate subqueries during optimization if they are disabled. This
+    function can be called during execution when doing dynamic range access, and
+    we only want to disable subquery evaluation during optimization, so check if
+    we're in the optimization phase by calling SELECT_LEX_UNIT::is_optimized().
+  */
+  const SELECT_LEX *const select = thd->lex->current_select();
+  if (!select->master_unit()->is_optimized() &&
+      !evaluate_during_optimization(value, select))
+    return true;
+
   // For comparison purposes allow invalid dates like 2000-01-32
   const sql_mode_t orig_sql_mode = thd->variables.sql_mode;
   thd->variables.sql_mode |= MODE_INVALID_DATES;

sql/item_cmpfunc.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -104,7 +104,7 @@
   Is set in slave SQL thread when there was an
   error on master, which, when is not reproducible
   on slave (i.e. the query succeeds on slave),
-  is not terminal to the state of repliation,
+  is not terminal to the state of replication,
   and should be ignored. The slave SQL thread,
   however, needs to rollback the effects of the
   succeeded statement to keep replication consistent.
@@ -113,7 +113,7 @@
 
 /*
   Dont report errors for individual rows,
-  But just report error on commit (or read ofcourse)
+  But just report error on commit (or read, of course)
   Note! Reserved for use in MySQL Cluster
 */
 #define OPTION_ALLOW_BATCH (1ULL << 36)  // THD, intern (slave)
@@ -122,4 +122,11 @@
 
 // Is set while thread is updating the data dictionary tables.
 #define OPTION_DD_UPDATE_CONTEXT (1ULL << 38)  // intern
-#endif                                         /* QUERY_OPTIONS_INCLUDED */
+
+/**
+  If this option is set, subqueries should not be evaluated during
+  optimization, even if they are known to produce a constant result.
+*/
+#define OPTION_NO_SUBQUERY_DURING_OPTIMIZATION (1ULL << 39)  // intern
+
+#endif /* QUERY_OPTIONS_INCLUDED */

sql/opt_range.cc

@@ -6438,7 +6438,6 @@ static bool open_secondary_engine_tables(THD *thd, uint flags) {
 
   auto hton = plugin_data<const handlerton *>(secondary_engine_plugin);
   sql_cmd->use_secondary_storage_engine(hton);
-  lex->add_statement_options(OPTION_NO_CONST_TABLES);
 
   // Replace the TABLE objects in the TABLE_LIST with secondary tables.
   Open_table_context ot_ctx(thd, flags | MYSQL_OPEN_SECONDARY_ENGINE);

sql/query_options.h

@@ -3455,7 +3455,13 @@ struct LEX : public Query_tables_list {
   ulonglong m_statement_options{0};
 
  public:
-  /// @return a bit set of options set for this statement
+  /**
+    Gets the options that have been set for this statement. The options are
+    propagated to the SELECT_LEX objects and should usually be read with
+    #SELECT_LEX::active_options().
+
+    @return a bit set of options set for this statement
+  */
   ulonglong statement_options() { return m_statement_options; }
   /**
     Add options to values of m_statement_options. options is an ORed

sql/sql_base.cc

@@ -61,6 +61,7 @@
 #include "sql/enum_query_type.h"
 #include "sql/error_handler.h"  // Functional_index_error_handler
 #include "sql/handler.h"
+#include "sql/item.h"
 #include "sql/item_cmpfunc.h"
 #include "sql/item_func.h"
 #include "sql/item_row.h"
@@ -3556,6 +3557,11 @@ static bool check_simple_equality(THD *thd, Item *left_item, Item *right_item,
       const_item = left_item;
     }
 
+    // Don't evaluate subqueries if they are disabled during optimization.
+    if (const_item != nullptr &&
+        !evaluate_during_optimization(const_item, thd->lex->current_select()))
+      return false;
+
     /*
       If the constant expression contains a reference to the field
       (for example, a = (a IS NULL)), we don't want to replace the
@@ -3650,7 +3656,7 @@ static bool check_row_equality(THD *thd, Item *left_row, Item_row *right_row,
         return true;
       if (!is_converted) thd->lex->current_select()->cond_count++;
     } else {
-      if (check_simple_equality(thd, left_item, right_item, 0, cond_equal,
+      if (check_simple_equality(thd, left_item, right_item, nullptr, cond_equal,
                                 &is_converted))
         return true;
       thd->lex->current_select()->cond_count++;
@@ -9108,7 +9114,8 @@ static bool make_join_select(JOIN *join, Item *cond) {
   }
   DBUG_EXECUTE("where",
                print_where(thd, const_cond, "constants", QT_ORDINARY););
-  if (const_cond != NULL) {
+  if (const_cond != nullptr &&
+      evaluate_during_optimization(const_cond, join->select_lex)) {
     const bool const_cond_result = const_cond->val_int() != 0;
     if (thd->is_error()) return true;
 
@@ -9695,7 +9702,8 @@ ORDER *JOIN::remove_const(ORDER *first_order, Item *cond, bool change_list,
         (primary_tables > 1 && rollup.state == ROLLUP::STATE_INITED &&
          select_lex->outer_join))
       *simple_order = 0;  // Must do a temp table to sort
-    else if (!(order_tables & not_const_tables)) {
+    else if ((order_tables & not_const_tables) == 0 &&
+             evaluate_during_optimization(order->item[0], select_lex)) {
       if (order->item[0]->has_subquery()) {
         if (!thd->lex->is_explain()) {
           Opt_trace_array trace_subselect(trace, "subselect_evaluation");
@@ -9863,15 +9871,26 @@ bool optimize_cond(THD *thd, Item **cond, COND_EQUAL **cond_equal,
   return false;
 }
 
+/**
+  Checks if a condition can be evaluated during constant folding. It can be
+  evaluated if it is constant during execution and not expensive to evaluate. If
+  it contains a subquery, it should not be evaluated if the option
+  OPTION_NO_SUBQUERY_DURING_OPTIMIZATION is active.
+*/
+static bool can_evaluate_condition(THD *thd, Item *condition) {
+  return condition->const_for_execution() && !condition->is_expensive() &&
+         evaluate_during_optimization(condition, thd->lex->current_select());
+}
+
 /**
   Calls fold_condition. If that made the condition constant for execution,
   simplify and fold again. @see fold_condition() for arguments.
 */
 static bool fold_condition_exec(THD *thd, Item *cond, Item **retcond,
                                 Item::cond_result *cond_value) {
   if (fold_condition(thd, cond, retcond, cond_value)) return true;
-  if (*retcond != nullptr && (*retcond)->const_for_execution() &&
-      !(*retcond)->is_expensive())  // simplify further maybe
+  if (*retcond != nullptr &&
+      can_evaluate_condition(thd, *retcond))  // simplify further maybe
     return remove_eq_conds(thd, *retcond, retcond, cond_value);
   return false;
 }
@@ -9974,13 +9993,13 @@ bool remove_eq_conds(THD *thd, Item *cond, Item **retcond,
       *retcond = item;
       return false;
     }
-  } else if (cond->const_for_execution() && !cond->is_expensive()) {
+  } else if (can_evaluate_condition(thd, cond)) {
     bool value;
     if (eval_const_cond(thd, cond, &value)) return true;
     *cond_value = value ? Item::COND_TRUE : Item::COND_FALSE;
     *retcond = NULL;
     return false;
-  } else {  // boolan compare function
+  } else {  // Boolean compare function
     *cond_value = cond->eq_cmp_result();
     if (*cond_value == Item::COND_OK) {
       return fold_condition_exec(thd, cond, retcond, cond_value);
@@ -10877,3 +10896,19 @@ static uint32 get_key_length_tmp_table(Item *item) {
 
   return len;
 }
+
+bool evaluate_during_optimization(const Item *item, const SELECT_LEX *select) {
+  /*
+    Should only be called on items that are const_for_execution(), as those
+    items are the only ones that are allowed to be evaluated during optimization
+    in the first place.
+
+    Additionally, allow items that only access tables in JOIN::const_table_map.
+    This should not be necessary, but the const_for_execution() property is not
+    always updated correctly by update_used_tables() for certain subqueries.
+  */
+  DBUG_ASSERT(item->const_for_execution() ||
+              (item->used_tables() & ~select->join->const_table_map) == 0);
+  return !item->has_subquery() || (select->active_options() &
+                                   OPTION_NO_SUBQUERY_DURING_OPTIMIZATION) == 0;
+}

sql/sql_lex.h

@@ -1316,4 +1316,16 @@ class Candidate_table_order {
 
 extern const char *antijoin_null_cond;
 
+/**
+  Checks if an Item, which is constant for execution, can be evaluated during
+  optimization. It cannot be evaluated if it contains a subquery and the
+  OPTION_NO_SUBQUERY_DURING_OPTIMIZATION query option is active.
+
+  @param item    the Item to check
+  @param select  the query block that contains the Item
+  @return false if this Item contains a subquery and subqueries cannot be
+  evaluated during optimization, or true otherwise
+*/
+bool evaluate_during_optimization(const Item *item, const SELECT_LEX *select);
+
 #endif /* SQL_OPTIMIZER_INCLUDED */

sql/sql_optimizer.cc

@@ -6469,10 +6469,12 @@ bool TABLE_LIST::is_mergeable() const {
          derived->is_mergeable();
 }
 
-///  @returns true if materializable table contains one or zero rows
 bool TABLE_LIST::materializable_is_const() const {
   DBUG_ASSERT(uses_materialization());
-  return derived_unit()->query_result()->estimated_rowcount <= 1;
+  const SELECT_LEX_UNIT *unit = derived_unit();
+  return unit->query_result()->estimated_rowcount <= 1 &&
+         (unit->first_select()->active_options() &
+          OPTION_NO_SUBQUERY_DURING_OPTIMIZATION) == 0;
 }
 
 /**

sql/sql_optimizer.h

@@ -2725,7 +2725,11 @@ struct TABLE_LIST {
   bool is_mergeable() const;
 
   /**
-    @returns true if materializable table contains one or zero rows.
+    Checks if this is a table that contains zero rows or one row, and that can
+    be materialized during optimization.
+
+    @returns true if materializable table contains one or zero rows, and
+    materialization during optimization is permitted
 
     Returning true implies that the table is materialized during optimization,
     so it need not be optimized during execution.

sql/table.cc

@@ -235,6 +235,12 @@ static bool PrepareSecondaryEngine(THD *thd, LEX *lex) {
   auto context = new (thd->mem_root) Mock_execution_context;
   if (context == nullptr) return true;
   lex->set_secondary_engine_execution_context(context);
+
+  // Disable use of constant tables and evaluation of subqueries during
+  // optimization.
+  lex->add_statement_options(OPTION_NO_CONST_TABLES |
+                             OPTION_NO_SUBQUERY_DURING_OPTIMIZATION);
+
   return false;
 }