WL#10358: Implement table value constructors: VALUES

Implement the standard syntax for table value constructors (TVC).
The SQL standard lets us use VALUES syntax, i.e. TVC, wherever a SELECT
query specification may be used. This allows for statements such as
VALUES (1, 'one'), (2, 'two'), which previously only could be emulated
with UNIONS.

Note: The current MySQL server has a conflicting non-standard feature:
the VALUES() function of ON DUPLICATE KEY UPDATE update expressions,
which is a showstopper for the standard TVC syntax. However, the
standard also accepts a verbose form of TVC: VALUES ROW(), ROW() ..,
which is used instead.

Also extend <query specification> to allow an <explicit table> clause as
valid syntax. The syntax TABLE <table name> is equivalent to SELECT *
FROM <table name>.

The special syntax for INSERT statements, INSERT .. VALUES, remains as-is.
In addition, INSERT statements with VALUES that use the new verbose syntax
are rewritten to follow the same execution path as the old syntax.

Reviewed by: Roy Lyseng <roy.lyseng@oracle.com>

Author: roylyseng

mysql-test/suite/funcs_1/r/storedproc.result

@@ -5605,7 +5605,7 @@ CREATE PROCEDURE sp1()
 table:BEGIN
 SELECT @x;
 END//
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'table:BEGIN
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ':BEGIN
 SELECT @x;
 END' at line 2
 DROP PROCEDURE IF EXISTS sp1;
@@ -5779,7 +5779,7 @@ CREATE PROCEDURE sp1()
 values:BEGIN
 SELECT @x;
 END//
-ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'values:BEGIN
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ':BEGIN
 SELECT @x;
 END' at line 2
 DROP PROCEDURE IF EXISTS sp1;

share/errmsg-utf8.txt

@@ -9203,6 +9203,12 @@ ER_CONSTRAINT_NOT_FOUND
 ER_ALTER_CONSTRAINT_ENFORCEMENT_NOT_SUPPORTED
   eng "Altering constraint enforcement is not supported for the constraint '%-.192s'. Enforcement state alter is not supported for the PRIMARY, UNIQUE and FOREIGN KEY type constraints."
 
+ER_TABLE_VALUE_CONSTRUCTOR_MUST_HAVE_COLUMNS
+  eng "Each row of a VALUES clause must have at least one column, unless when used as source in an INSERT statement."
+
+ER_TABLE_VALUE_CONSTRUCTOR_CANNOT_HAVE_DEFAULT
+  eng "A VALUES clause cannot use DEFAULT values, unless used as a source in an INSERT statement."
+
 #
 #  End of 8.0 error messages (server-to-client).
 #  Do NOT add messages intended for the error log above!

sql/basic_row_iterators.h

@@ -38,6 +38,7 @@
 #include "my_inttypes.h"
 #include "sql/mem_root_array.h"
 #include "sql/row_iterator.h"
+#include "sql/sql_list.h"
 
 class Filesort_info;
 class Item;
@@ -517,4 +518,53 @@ class FollowTailIterator final : public TableRowIterator {
   ha_rows *m_stored_rows = nullptr;
 };
 
+/**
+  TableValueConstructor is the iterator for the table value constructor case of
+  a query_primary (i.e. queries of the form VALUES row_list; e.g. VALUES ROW(1,
+  10), ROW(2, 20)).
+
+  The iterator is passed the field list of its parent JOIN object, which may
+  contain Item_values_column objects that are created during
+  SELECT_LEX::prepare_values(). This is required so that Read() can replace the
+  currently selected row by simply changing the references of Item_values_column
+  objects to the next row.
+
+  The iterator must output multiple rows without being materialized, and does
+  not scan any tables. The indirection of Item_values_column is required, as the
+  executor outputs what is contained in join->fields (either directly, or
+  indirectly through ConvertItemsToCopy), and is thus responsible for ensuring
+  that join->fields contains the correct next row.
+ */
+class TableValueConstructorIterator final : public RowIterator {
+ public:
+  TableValueConstructorIterator(THD *thd, ha_rows *examined_rows,
+                                const List<List<Item>> &row_value_list,
+                                List<Item> *join_fields);
+
+  bool Init() override;
+  int Read() override;
+
+  std::vector<std::string> DebugString() const override {
+    return {"Rows fetched before execution"};
+  }
+
+  void SetNullRowFlag(bool) override { DBUG_ASSERT(false); }
+
+  void UnlockRow() override {}
+
+ private:
+  ha_rows *const m_examined_rows{nullptr};
+
+  /// Contains the row values that are part of a VALUES clause. Read() will
+  /// modify contained Item objects during execution by calls to is_null() and
+  /// the required val function to extract its value.
+  const List<List<Item>> &m_row_value_list;
+  List_STL_Iterator<const List<Item>> m_row_it;
+
+  /// References to the row we currently want to output. When multiple rows must
+  /// be output, this contains Item_values_column objects. In this case, each
+  /// call to Read() will replace its current reference with the next row.
+  List<Item> *const m_output_refs;
+};
+
 #endif  // SQL_BASIC_ROW_ITERATORS_H_

sql/item.cc

@@ -9519,6 +9519,111 @@ bool Item_type_holder::get_time(MYSQL_TIME *) {
   return true;
 }
 
+type_conversion_status Item_values_column::save_in_field_inner(
+    Field *to, bool no_conversions) {
+  type_conversion_status res;
+  res = m_value_ref->save_in_field(to, no_conversions);
+  null_value = m_value_ref->null_value;
+  return res;
+}
+
+Item_values_column::Item_values_column(THD *thd, Item *ref) : super(thd, ref) {
+  fixed = true;
+}
+
+/* purecov: begin deadcode */
+
+bool Item_values_column::eq(const Item *item, bool binary_cmp) const {
+  DBUG_ASSERT(false);
+  const Item *it = const_cast<Item *>(item)->real_item();
+  return m_value_ref && m_value_ref->eq(it, binary_cmp);
+}
+
+/* purecov: end */
+
+double Item_values_column::val_real() {
+  DBUG_ASSERT(fixed);
+  double tmp = m_value_ref->val_real();
+  null_value = m_value_ref->null_value;
+  return tmp;
+}
+
+longlong Item_values_column::val_int() {
+  DBUG_ASSERT(fixed);
+  longlong tmp = m_value_ref->val_int();
+  null_value = m_value_ref->null_value;
+  return tmp;
+}
+
+/* purecov: begin deadcode */
+
+my_decimal *Item_values_column::val_decimal(my_decimal *decimal_value) {
+  DBUG_ASSERT(false);
+  DBUG_ASSERT(fixed);
+  my_decimal *val = m_value_ref->val_decimal(decimal_value);
+  null_value = m_value_ref->null_value;
+  return val;
+}
+
+bool Item_values_column::val_bool() {
+  DBUG_ASSERT(false);
+  DBUG_ASSERT(fixed);
+  bool tmp = m_value_ref->val_bool();
+  null_value = m_value_ref->null_value;
+  return tmp;
+}
+
+bool Item_values_column::val_json(Json_wrapper *result) {
+  DBUG_ASSERT(false);
+  DBUG_ASSERT(fixed);
+  bool ok = m_value_ref->val_json(result);
+  null_value = m_value_ref->null_value;
+  return ok;
+}
+
+/* purecov: end */
+
+String *Item_values_column::val_str(String *tmp) {
+  DBUG_ASSERT(fixed);
+  tmp = m_value_ref->val_str(tmp);
+  null_value = m_value_ref->null_value;
+  return tmp;
+}
+
+bool Item_values_column::is_null() {
+  DBUG_ASSERT(fixed);
+  /*
+    Item_values_column is dualistic in nature: It represents both a set
+    of values, and, during evaluation, an individual value in this set.
+    This assert will ensure that we only check nullability of individual
+    values, since a set of values is never NULL. Note that setting
+    RAND_TABLE_BIT in the constructor prevents this function from being called
+    during resolving.
+  */
+  DBUG_ASSERT(m_value_ref != nullptr);
+  bool tmp = m_value_ref->is_null();
+  null_value = m_value_ref->null_value;
+  return tmp;
+}
+
+bool Item_values_column::get_date(MYSQL_TIME *ltime,
+                                  my_time_flags_t fuzzydate) {
+  DBUG_ASSERT(fixed);
+  bool result = m_value_ref->get_date(ltime, fuzzydate);
+  null_value = m_value_ref->null_value;
+  return result;
+}
+
+bool Item_values_column::get_time(MYSQL_TIME *ltime) {
+  DBUG_ASSERT(fixed);
+  DBUG_ASSERT(m_value_ref != nullptr);
+  return m_value_ref->get_time(ltime);
+}
+
+void Item_values_column::add_used_tables(Item *value) {
+  m_aggregated_used_tables |= value->used_tables();
+}
+
 void Item_result_field::cleanup() {
   DBUG_TRACE;
   Item::cleanup();

sql/item.h

@@ -728,7 +728,8 @@ class Item : public Parse_tree_node {
     XPATH_NODESET_CMP,
     VIEW_FIXER_ITEM,
     FIELD_BIT_ITEM,
-    NULL_RESULT_ITEM
+    NULL_RESULT_ITEM,
+    VALUES_COLUMN_ITEM
   };
 
   enum cond_result { COND_UNDEF, COND_OK, COND_TRUE, COND_FALSE };
@@ -6124,6 +6125,57 @@ class Item_type_holder final : public Item_aggregate_type {
   bool get_time(MYSQL_TIME *) override;
 };
 
+/**
+  Reference item that encapsulates both the type and the contained items of a
+  single column of a VALUES ROW query expression.
+
+  During execution, the item that will be output for the current iteration is
+  contained in m_value_ref. The type of the column and the referenced item may
+  differ in cases where a column of a VALUES clause contains different types
+  across different rows, and must therefore do type conversions to their common
+  denominator (e.g. a column containing both 10 and "10", of which the types
+  will be aggregated into VARCHAR).
+
+  See the class comment for TableValueConstructorIterator for info on how
+  Item_values_column is used as an indirection to iterate over the rows of a
+  table value constructor (i.e. VALUES ROW expressions).
+*/
+class Item_values_column final : public Item_aggregate_type {
+  typedef Item_aggregate_type super;
+
+ private:
+  Item *m_value_ref{nullptr};
+  /*
+    Even if a table value constructor contains only constant values, we
+    still need to identify individual rows within it. Set RAND_TABLE_BIT
+    to ensure that all rows are scanned, and that the whole VALUES clause
+    is never substituted with a const value or row.
+  */
+  table_map m_aggregated_used_tables{RAND_TABLE_BIT};
+
+  type_conversion_status save_in_field_inner(Field *field,
+                                             bool no_conversions) override;
+
+ public:
+  Item_values_column(THD *thd, Item *ref);
+
+  bool eq(const Item *item, bool binary_cmp) const override;
+  double val_real() override;
+  longlong val_int() override;
+  my_decimal *val_decimal(my_decimal *) override;
+  bool val_bool() override;
+  String *val_str(String *tmp) override;
+  bool val_json(Json_wrapper *result) override;
+  bool is_null() override;
+  bool get_date(MYSQL_TIME *ltime, my_time_flags_t fuzzydate) override;
+  bool get_time(MYSQL_TIME *ltime) override;
+
+  enum Type type() const override { return VALUES_COLUMN_ITEM; }
+  void set_value(Item *new_value) { m_value_ref = new_value; }
+  table_map used_tables() const override { return m_aggregated_used_tables; }
+  void add_used_tables(Item *value);
+};
+
 /// A class that represents a constant JSON value.
 class Item_json final : public Item_basic_constant {
   unique_ptr_destroy_only<Json_wrapper> m_value;

sql/item_cmpfunc.cc

@@ -5727,7 +5727,7 @@ void Item_is_not_null_test::update_used_tables() {
   set_accum_properties(args[0]);
   used_tables_cache |= args[0]->used_tables();
 
-  if (used_tables_cache == initial_pseudo_tables)
+  if (used_tables_cache == initial_pseudo_tables && args[0]->const_item())
     /* Remember if the value is always NULL or never NULL */
     cached_value = !args[0]->is_null();
 }

sql/item_subselect.cc

@@ -1966,7 +1966,7 @@ Item_in_subselect::single_value_in_to_exists_transformer(THD *thd,
     */
     Item *orig_item = select->item_list.head()->real_item();
 
-    if (select->table_list.elements || select->where_cond()) {
+    if (!select->source_table_is_one_row() || select->where_cond()) {
       bool tmp;
       Item_bool_func *item = func->create(m_injected_left_expr, orig_item);
       /*

sql/parse_tree_nodes.cc

@@ -811,6 +811,33 @@ Sql_cmd *PT_insert::make_cmd(THD *thd) {
 
   Parse_context pc(thd, lex->current_select());
 
+  // Currently there are two syntaxes (old and new, respectively) for INSERT
+  // .. VALUES statements:
+  //
+  //  - INSERT .. VALUES (), () ..
+  //  - INSERT .. VALUES ROW(), ROW() ..
+  //
+  // The latter is a table value constructor, i.e. it has an subquery
+  // expression, while the former is the standard VALUES syntax. When the
+  // non-standard VALUES() function (primarily used in ON DUPLICATE KEY UPDATE
+  // update expressions) is deprecated in the future, the old syntax can be used
+  // as a table value constructor as well.
+  //
+  // However, until such a change is made, we convert INSERT statements with
+  // table value constructors into PT_insert objects that are equal to the old
+  // syntax, as to enforce consistency by making sure they both follow the same
+  // execution path.
+  //
+  // Note that this removes the constness of both row_value_list and
+  // insert_query_expression, which should both be restored when deprecating
+  // VALUES as mentioned above.
+  if (has_select() && insert_query_expression->is_table_value_constructor()) {
+    row_value_list = insert_query_expression->get_row_value_list();
+    DBUG_ASSERT(row_value_list != nullptr);
+
+    insert_query_expression = nullptr;
+  }
+
   if (is_replace) {
     lex->sql_command = has_select() ? SQLCOM_REPLACE_SELECT : SQLCOM_REPLACE;
     lex->duplicates = DUP_REPLACE;
@@ -1026,6 +1053,21 @@ bool PT_query_specification::contextualize(Parse_context *pc) {
   return false;
 }
 
+bool PT_table_value_constructor::contextualize(Parse_context *pc) {
+  if (row_value_list->contextualize(pc)) return true;
+
+  pc->select->is_table_value_constructor = true;
+  pc->select->row_value_list = &row_value_list->get_many_values();
+
+  // Some queries, such as CREATE TABLE with SELECT, require item_list to
+  // contain items to call SELECT_LEX::prepare.
+  for (Item &item : *pc->select->row_value_list->head()) {
+    pc->select->item_list.push_back(&item);
+  }
+
+  return false;
+}
+
 bool PT_query_expression::contextualize_order_and_limit(Parse_context *pc) {
   /*
     Quick reject test. We don't need to do anything if there are no limit