MDEV-37052: JSON_TABLE stack overflow handling errors

The recursive nature of add_table_function_dependencies
resolution meant that the detection of a stack overrun
would continue to recursively call itself.
Its quite possible that a user SQL could get multiple
ER_STACK_OVERRUN_NEED_MORE errors.

Additionaly the results of the stack overrrun check
result was incorrectly assigned to a table_map result.

Its only because of the "if error" check after
add_table_function_dependencies is called, that would
detected the stack overrun error, prevented a
potential corruped tablemap is from being processed.

Corrected add_table_function_dependencies to stop and
return on the detection of a stack overrun error.

The add_extra_deps call also was true on a stack overrun.

Author: grooverdan

sql/json_table.cc

@@ -1428,22 +1428,29 @@ static bool add_extra_deps(List<TABLE_LIST> *join_list, table_map deps)
   @param  join_list    List of tables to process. Initial invocation should
                        supply the JOIN's top-level table list.
   @param  nest_tables  Bitmap of all tables in the join list.
+  @param  error        Pointer to value which is set to true on stack overrun
+                       error.
 
   @return Bitmap of all outside references that tables in join_list have,
-    or 0 on out of stack error.
+    or 0 on out of stack overrun error (in addition to *error= true).
 */
 
 table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
-                                          table_map nest_tables)
+                                          table_map nest_tables,
+					  bool *error)
 {
   TABLE_LIST *table;
   table_map res= 0;
   List_iterator<TABLE_LIST> li(*join_list);
 
   DBUG_EXECUTE_IF("json_check_min_stack_requirement",
-                  if (dbug_json_check_min_stack_requirement()) return 0;);
+                  if (dbug_json_check_min_stack_requirement())
+		    { *error= true; return 0; });
   if ((res=check_stack_overrun(current_thd, STACK_MIN_SIZE , NULL)))
-    return res;
+  {
+    *error= true;
+    return 0;
+  }
 
   // Recursively compute extra dependencies
   while ((table= li++))
@@ -1452,7 +1459,9 @@ table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
     if ((nested_join= table->nested_join))
     {
       res |= add_table_function_dependencies(&nested_join->join_list,
-                                             nested_join->used_tables);
+                                             nested_join->used_tables, error);
+      if (*error)
+	return 0;
     }
     else if (table->table_function)
     {
@@ -1465,7 +1474,10 @@ table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
   if (res)
   {
     if (add_extra_deps(join_list,  res))
+    {
+      *error= true;
       return 0;
+    }
   }
 
   return res;

sql/json_table.h

@@ -284,7 +284,7 @@ bool push_table_function_arg_context(LEX *lex, MEM_ROOT *alloc);
 TABLE *create_table_for_function(THD *thd, TABLE_LIST *sql_table);
 
 table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
-                                          table_map nest_tables);
+                                          table_map nest_tables, bool *error);
 
 #endif /* JSON_TABLE_INCLUDED */
 

sql/sql_select.cc

@@ -2263,6 +2263,7 @@ JOIN::optimize_inner()
   SELECT_LEX *sel= select_lex;
   if (sel->first_cond_optimization)
   {
+    bool error= false;
     /*
       The following code will allocate the new items in a permanent
       MEMROOT for prepared statements and stored procedures.
@@ -2280,7 +2281,7 @@ JOIN::optimize_inner()
     /* Convert all outer joins to inner joins if possible */
     conds= simplify_joins(this, join_list, conds, TRUE, FALSE);
 
-    add_table_function_dependencies(join_list, table_map(-1));
+    add_table_function_dependencies(join_list, table_map(-1), &error);
 
     if (thd->is_error() ||
         (!select_lex->leaf_tables_saved && select_lex->save_leaf_tables(thd)))