LIMIT ROWS EXAMINED prematurely triggers during optimization

1. LIMIT ROWS EXAMINED clause allows to abort a query if a certain limit
of rows involved in processing is exceeded. This limitation should be only
enforced during the execution phase and not the optimization.

2. Unions are often executed using so-called "fake_select_lex" which
collects the final result from union parts. During that finalization
LIMIT ROWS EXAMINED must be ignored to avoid producing a potentially
incomplete result. There is a workaround at `st_select_lex_unit::exec()`
which deactivates the limit before fake_select_lex processing, and
re-activates it when the processing is finished. However, this re-activation
does not take into account whether the limit was active before the start
of fake_select_lex processing.

3. `st_select_lex_unit::exec()` can be invoked during the optimization
phase for evaluation of constant conditions with unions. At that time
the limit trigger is not activated. Given that the re-activation
mentioned above does not respect the previous state of the limit trigger,
a premature activation of the limit may happen.

This commit fixes that behavior by storing the state of the trigger
before its deactivation at `st_select_lex_unit::exec()` and re-activating
it only if the limit was active before.

4. This commit also removes the call to `thd->lex->set_rows_examined()`
from `st_select_lex_unit::exec_recursive()` which was probably copied
from `st_select_lex_unit::exec()`. But in the context of
 `exec_recursive()` the call does not make sense as there is no
previous deactivation of the limit as at `exec()`.

Reviewed by: Dave Gosselin, Sergei Petrunia

Author: mariadb-OlegSmirnov

mysql-test/main/limit_rows_examined.result

@@ -938,3 +938,14 @@ Warnings:
 Warning	1931	Query execution was interrupted. The query exceeded LIMIT ROWS EXAMINED 100. The query result may be incomplete
 DROP TABLE t1, t2;
 # End of 10.5 tests
+#
+# MDEV-22241: Assertion `0' failed in Protocol::end_statement after query with LIMIT ROWS EXAMINED
+#
+CREATE TABLE t1 (a int);
+INSERT INTO t1 VALUES (1);
+SELECT 1 FROM t1
+WHERE (1 IN (SELECT 8 UNION SELECT 5)) OR t1.a = 2
+LIMIT ROWS EXAMINED 1;
+1
+DROP TABLE t1;
+# End of 10.11 tests

mysql-test/main/limit_rows_examined.test

@@ -658,3 +658,18 @@ SELECT COUNT(*) FROM t1 JOIN t2 ON (b = a) UNION DISTINCT SELECT COUNT(*) FROM t
 DROP TABLE t1, t2;
 
 --echo # End of 10.5 tests
+
+--echo #
+--echo # MDEV-22241: Assertion `0' failed in Protocol::end_statement after query with LIMIT ROWS EXAMINED
+--echo #
+
+CREATE TABLE t1 (a int);
+INSERT INTO t1 VALUES (1);
+
+SELECT 1 FROM t1
+WHERE (1 IN (SELECT 8 UNION SELECT 5)) OR t1.a = 2
+LIMIT ROWS EXAMINED 1;
+
+DROP TABLE t1;
+
+--echo # End of 10.11 tests

sql/sql_lex.h

@@ -3604,8 +3604,6 @@ struct LEX: public Query_tables_list
   {
     if (limit_rows_examined)
       limit_rows_examined_cnt= limit_rows_examined->val_uint();
-    else
-      limit_rows_examined_cnt= ULONGLONG_MAX; // Unreachable value
   }
 
   /**
@@ -3617,7 +3615,7 @@ struct LEX: public Query_tables_list
   */
   bool deactivate_limit_rows_examined()
   {
-    bool was_activated= (limit_rows_examined_cnt == ULONGLONG_MAX);
+    bool was_activated= (limit_rows_examined_cnt != ULONGLONG_MAX);
     limit_rows_examined_cnt= ULONGLONG_MAX; // Unreachable value
     return was_activated;
   }

sql/sql_union.cc

@@ -2340,14 +2340,15 @@ bool st_select_lex_unit::exec()
 
   DBUG_EXECUTE_IF("show_explain_probe_union_read", 
                    dbug_serve_apcs(thd, 1););
+  bool limit_rows_was_activated;
   {
     List<Item_func_match> empty_list;
     empty_list.empty();
     /*
-      Deactivate LIMIT ROWS EXAMINED in order to produce the possibly incomplete
-      result of the UNION without interruption due to exceeding the limit.
+      Deactivate LIMIT ROWS EXAMINED to avoid producing potentially incomplete
+      result of the UNION due to exceeding of the limit.
     */
-    thd->lex->deactivate_limit_rows_examined();
+    limit_rows_was_activated= thd->lex->deactivate_limit_rows_examined();
 
     // Check if EOM
     if (fake_select_lex != NULL && likely(!thd->is_fatal_error))
@@ -2447,7 +2448,8 @@ bool st_select_lex_unit::exec()
   }
   thd->lex->current_select= lex_select_save;
 err:
-  thd->lex->activate_limit_rows_examined();
+  if (limit_rows_was_activated)
+    thd->lex->activate_limit_rows_examined();
   if (likely(!saved_error))
     thd->inc_examined_row_count(examined_rows);
   DBUG_RETURN(saved_error);
@@ -2588,7 +2590,6 @@ bool st_select_lex_unit::exec_recursive()
 
   thd->lex->current_select= lex_select_save;
 err:
-  thd->lex->activate_limit_rows_examined();
   DBUG_RETURN(saved_error);    
 }