Bug #30473261: CONVERT THE INDEX SUBQUERY ENGINES INTO USING THE ITERATOR EXECUTOR [patch 3/10, hashed refs]

Make RefIterator support hashed refs.

Most index lookups (ref accesses) follow a pattern where the index' parts are
matched 1:1 with a set of items, e.g. an index on (a,b) could have a lookup
a=3,b=t1.other_b.

However, as a special case, temporary tables may need indexes over a set of
fields that are so large that they cannot be part of a unique index (e.g.
blobs, or a large set of fields). In this case, the index is over a single
column (a non-unique numeric hash of all the fields involved), and the
TABLE_REF construction takes a rather different form, with each field being
hashed into this column. We need to handle this correctly in RefIterator,
and also print this correctly in EXPLAIN, even though we no longer have a 1:1
match between parts of the lookup key (TABLE_REF) and fields in the index
(key_parts in KEY).

This is needed for the next patch in the series. Stricly speaking, it only
uses hashed refs in (forward-scanning) RefIterator, but for orthogonality,
we add it to all paths where it is easy to do so.

Change-Id: If8442551d29ac76914e94470169a233812052638

Author: Steinar H. Gunderson

Diff for: sql/item_subselect.cc

@@ -3346,6 +3346,10 @@ bool subselect_hash_sj_engine::setup(THD *thd, List<Item> *tmp_columns) {
       !(tab->ref().items = (Item **)thd->alloc(sizeof(Item *) * tmp_key_parts)))
     return true;
 
+  if (tmp_table->hash_field) {
+    tab->ref().keypart_hash = &hash;
+  }
+
   uchar *cur_ref_buff = tab->ref().key_buff;
 
   /*
@@ -3399,11 +3403,11 @@ bool subselect_hash_sj_engine::setup(THD *thd, List<Item> *tmp_columns) {
       return true;
     }
 
-    if (tmp_table->hash_field)
+    if (tmp_table->hash_field) {
       tab->ref().key_copy[part_no] = new (thd->mem_root) store_key_hash_item(
           thd, field, cur_ref_buff, nullptr, field->pack_length(),
           tab->ref().items[part_no], &hash);
-    else
+    } else {
       tab->ref().key_copy[part_no] = new (thd->mem_root) store_key_item(
           thd, field,
           /* TODO:
@@ -3414,6 +3418,7 @@ bool subselect_hash_sj_engine::setup(THD *thd, List<Item> *tmp_columns) {
            */
           cur_ref_buff + (nullable ? 1 : 0), nullable ? cur_ref_buff : nullptr,
           key_parts[part_no].length, tab->ref().items[part_no]);
+    }
     if (nullable &&  // nullable column in tmp table,
                      // and UNKNOWN should not be interpreted as FALSE
         !item_in->abort_on_null) {

Diff for: sql/sql_executor.cc

@@ -132,8 +132,10 @@
 #include "template_utils.h"
 #include "thr_lock.h"
 
+using std::make_pair;
 using std::max;
 using std::min;
+using std::pair;
 using std::string;
 using std::unique_ptr;
 using std::vector;
@@ -167,6 +169,8 @@ static bool alloc_group_fields(JOIN *join, ORDER *group);
 static void SetCostOnTableIterator(const Cost_model_server &cost_model,
                                    const POSITION *pos, bool is_after_filter,
                                    RowIterator *iterator);
+static inline pair<uchar *, key_part_map> FindKeyBufferAndMap(
+    const TABLE_REF *ref);
 
 /**
    Evaluates HAVING condition
@@ -190,6 +194,21 @@ static constexpr size_t MAX_RECORD_BUFFER_SIZE = 128 * 1024;  // 128KB
 string RefToString(const TABLE_REF &ref, const KEY *key, bool include_nulls) {
   string ret;
 
+  if (ref.keypart_hash != nullptr) {
+    DBUG_ASSERT(!include_nulls);
+    ret = key->key_part[0].field->field_name;
+    ret += "=hash(";
+    for (unsigned key_part_idx = 0; key_part_idx < ref.key_parts;
+         ++key_part_idx) {
+      if (key_part_idx != 0) {
+        ret += ", ";
+      }
+      ret += ItemToString(ref.items[key_part_idx]);
+    }
+    ret += ")";
+    return ret;
+  }
+
   const uchar *key_buff = ref.key_buff;
 
   for (unsigned key_part_idx = 0; key_part_idx < ref.key_parts;
@@ -4845,9 +4864,10 @@ int EQRefIterator::Read() {
       return -1;
     }
 
+    pair<uchar *, key_part_map> key_buff_and_map = FindKeyBufferAndMap(m_ref);
     int error = table()->file->ha_index_read_map(
-        table()->record[0], m_ref->key_buff,
-        make_prev_keypart_map(m_ref->key_parts), HA_READ_KEY_EXACT);
+        table()->record[0], key_buff_and_map.first, key_buff_and_map.second,
+        HA_READ_KEY_EXACT);
     if (error) {
       return HandleError(error);
     }
@@ -5021,9 +5041,11 @@ int RefIterator<false>::Read() {  // Forward read.
       table()->set_no_row();
       return -1;
     }
+
+    pair<uchar *, key_part_map> key_buff_and_map = FindKeyBufferAndMap(m_ref);
     int error = table()->file->ha_index_read_map(
-        table()->record[0], m_ref->key_buff,
-        make_prev_keypart_map(m_ref->key_parts), HA_READ_KEY_EXACT);
+        table()->record[0], key_buff_and_map.first, key_buff_and_map.second,
+        HA_READ_KEY_EXACT);
     if (error) {
       return HandleError(error);
     }
@@ -5046,6 +5068,8 @@ int RefIterator<false>::Read() {  // Forward read.
 */
 template <>
 int RefIterator<true>::Read() {  // Reverse read.
+  assert(m_ref->keypart_hash == nullptr);
+
   if (m_first_record_since_init) {
     m_first_record_since_init = false;
 
@@ -5427,15 +5451,17 @@ int RefOrNullIterator::Read() {
     }
   }
 
+  pair<uchar *, key_part_map> key_buff_and_map = FindKeyBufferAndMap(m_ref);
+
   int error;
   if (m_reading_first_row) {
     m_reading_first_row = false;
     error = table()->file->ha_index_read_map(
-        table()->record[0], m_ref->key_buff,
-        make_prev_keypart_map(m_ref->key_parts), HA_READ_KEY_EXACT);
+        table()->record[0], key_buff_and_map.first, key_buff_and_map.second,
+        HA_READ_KEY_EXACT);
   } else {
     error = table()->file->ha_index_next_same(
-        table()->record[0], m_ref->key_buff, m_ref->key_length);
+        table()->record[0], key_buff_and_map.first, key_buff_and_map.second);
   }
 
   if (error == 0) {
@@ -9220,3 +9246,12 @@ int TableValueConstructorIterator::Read() {
   ++*m_examined_rows;
   return 0;
 }
+
+static inline pair<uchar *, key_part_map> FindKeyBufferAndMap(
+    const TABLE_REF *ref) {
+  if (ref->keypart_hash != nullptr) {
+    return make_pair(pointer_cast<uchar *>(ref->keypart_hash), key_part_map{1});
+  } else {
+    return make_pair(ref->key_buff, make_prev_keypart_map(ref->key_parts));
+  }
+}

Diff for: sql/sql_opt_exec_shared.h

@@ -111,6 +111,13 @@ struct TABLE_REF {
   */
   bool disable_cache;
 
+  /*
+    If non-nullptr, all the fields are hashed together through functions
+    in store_key (with the result being put into this field), as opposed to
+    being matched against individual fields in the associated KEY's key parts.
+   */
+  ulonglong *keypart_hash = nullptr;
+
   TABLE_REF()
       : key_err(true),
         key_parts(0),