MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on

For the case when  the SJM scan table is the first table in the join order,
then if we want to do the sorting on the SJM scan table, then we need to
make sure that we unpack the values to base table fields in two cases:
    1) Reading the SJM table and writing the sort-keys inside the sort-buffer
    2) Reading the sorted data from the sort file

Author: Varun Gupta

mysql-test/main/order_by.result

@@ -3417,7 +3417,7 @@ WHERE books.library_id = 8663 AND
 books.scheduled_for_removal=0 )
 ORDER BY wings.id;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	2	100.00	Using temporary; Using filesort
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	2	100.00	Using filesort
 1	PRIMARY	wings	eq_ref	PRIMARY	PRIMARY	4	test.books.wings_id	1	100.00	
 2	MATERIALIZED	books	ref	library_idx	library_idx	4	const	2	100.00	Using where
 Warnings:
@@ -4023,4 +4023,140 @@ COUNT(DISTINCT a)
 34
 SET @@tmp_memory_table_size= @save_tmp_memory_table_size;
 DROP TABLE t1;
+#
+# MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on
+#
+CREATE TABLE t1 (a INT, b int, primary key(a));
+CREATE TABLE t2 (a INT, b INT);
+INSERT INTO t1 (a,b) VALUES (58,1),(96,2),(273,3),(23,4),(231,5),(525,6),
+(2354,7),(321421,3),(535,2),(4535,3);
+INSERT INTO t2 (a,b) VALUES (58,3),(96,3),(273,3);
+# Join order should have the SJM scan table as the first table for both
+# the queries with GROUP BY and ORDER BY clause.
+EXPLAIN SELECT t1.a
+FROM t1
+WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+ORDER BY t1.a DESC;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	Using filesort
+1	PRIMARY	t1	eq_ref	PRIMARY	PRIMARY	4	test.t2.a	1	Using index
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+EXPLAIN FORMAT=JSON SELECT t1.a
+FROM t1
+WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+ORDER BY t1.a DESC;
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "read_sorted_file": {
+      "filesort": {
+        "sort_key": "t1.a desc",
+        "table": {
+          "table_name": "<subquery2>",
+          "access_type": "ALL",
+          "possible_keys": ["distinct_key"],
+          "rows": 3,
+          "filtered": 100,
+          "materialized": {
+            "unique": 1,
+            "query_block": {
+              "select_id": 2,
+              "table": {
+                "table_name": "t2",
+                "access_type": "ALL",
+                "rows": 3,
+                "filtered": 100,
+                "attached_condition": "t2.b = 3 and t2.a is not null"
+              }
+            }
+          }
+        }
+      }
+    },
+    "table": {
+      "table_name": "t1",
+      "access_type": "eq_ref",
+      "possible_keys": ["PRIMARY"],
+      "key": "PRIMARY",
+      "key_length": "4",
+      "used_key_parts": ["a"],
+      "ref": ["test.t2.a"],
+      "rows": 1,
+      "filtered": 100,
+      "using_index": true
+    }
+  }
+}
+SELECT t1.a
+FROM t1
+WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+ORDER BY t1.a DESC;
+a
+273
+96
+58
+EXPLAIN SELECT t1.a, group_concat(t1.b)
+FROM t1
+WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+GROUP BY t1.a DESC;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	3	Using filesort
+1	PRIMARY	t1	eq_ref	PRIMARY	PRIMARY	4	test.t2.a	1	
+2	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	3	Using where
+EXPLAIN FORMAT=JSON SELECT t1.a, group_concat(t1.b)
+FROM t1
+WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+GROUP BY t1.a DESC;
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "read_sorted_file": {
+      "filesort": {
+        "sort_key": "t1.a desc",
+        "table": {
+          "table_name": "<subquery2>",
+          "access_type": "ALL",
+          "possible_keys": ["distinct_key"],
+          "rows": 3,
+          "filtered": 100,
+          "materialized": {
+            "unique": 1,
+            "query_block": {
+              "select_id": 2,
+              "table": {
+                "table_name": "t2",
+                "access_type": "ALL",
+                "rows": 3,
+                "filtered": 100,
+                "attached_condition": "t2.b = 3 and t2.a is not null"
+              }
+            }
+          }
+        }
+      }
+    },
+    "table": {
+      "table_name": "t1",
+      "access_type": "eq_ref",
+      "possible_keys": ["PRIMARY"],
+      "key": "PRIMARY",
+      "key_length": "4",
+      "used_key_parts": ["a"],
+      "ref": ["test.t2.a"],
+      "rows": 1,
+      "filtered": 100
+    }
+  }
+}
+SELECT t1.a, group_concat(t1.b)
+FROM t1
+WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+GROUP BY t1.a DESC;
+a	group_concat(t1.b)
+273	3
+96	2
+58	1
+DROP TABLE t1, t2;
 # End of 10.5 tests

mysql-test/main/order_by.test

@@ -2500,5 +2500,37 @@ SELECT COUNT(DISTINCT a) FROM t1;
 SET @@tmp_memory_table_size= @save_tmp_memory_table_size;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on
+--echo #
+
+CREATE TABLE t1 (a INT, b int, primary key(a));
+CREATE TABLE t2 (a INT, b INT);
+
+INSERT INTO t1 (a,b) VALUES (58,1),(96,2),(273,3),(23,4),(231,5),(525,6),
+                            (2354,7),(321421,3),(535,2),(4535,3);
+INSERT INTO t2 (a,b) VALUES (58,3),(96,3),(273,3);
+
+--echo # Join order should have the SJM scan table as the first table for both
+--echo # the queries with GROUP BY and ORDER BY clause.
+
+let $query= SELECT t1.a
+            FROM t1
+            WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+            ORDER BY t1.a DESC;
+
+eval EXPLAIN $query;
+eval EXPLAIN FORMAT=JSON $query;
+eval $query;
+
+let $query= SELECT t1.a, group_concat(t1.b)
+            FROM t1
+            WHERE t1.a IN (SELECT a FROM t2 WHERE b=3)
+            GROUP BY t1.a DESC;
+
+eval EXPLAIN $query;
+eval EXPLAIN FORMAT=JSON $query;
+eval $query;
+DROP TABLE t1, t2;
 
 --echo # End of 10.5 tests

sql/filesort.cc

@@ -251,6 +251,9 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort,
 
   param.init_for_filesort(sort_len, table, max_rows, filesort->sort_positions);
 
+  param.set_all_read_bits= filesort->set_all_read_bits;
+  param.unpack= filesort->unpack;
+
   sort->addon_fields=  param.addon_fields;
   sort->sort_keys= param.sort_keys;
 
@@ -883,14 +886,20 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select,
       goto err;
   }
 
+  if (param->set_all_read_bits)
+    sort_form->column_bitmaps_set(save_read_set, save_write_set);
   DEBUG_SYNC(thd, "after_index_merge_phase1");
 
   for (;;)
   {
     if (quick_select)
       error= select->quick->get_next();
     else					/* Not quick-select */
+    {
       error= file->ha_rnd_next(sort_form->record[0]);
+      if (param->unpack)
+        param->unpack(sort_form);
+    }
     if (unlikely(error))
       break;
     file->position(sort_form->record[0]);

sql/filesort.h

@@ -62,6 +62,13 @@ class Filesort: public Sql_alloc
   Filesort_tracker *tracker;
   Sort_keys *sort_keys;
 
+  /*
+    TRUE means all the fields of table of whose bitmap read_set is set
+         need to be read while reading records in the sort buffer.
+    FALSE otherwise
+  */
+  bool set_all_read_bits;
+
   Filesort(ORDER *order_arg, ha_rows limit_arg, bool sort_positions_arg,
            SQL_SELECT *select_arg):
     order(order_arg),
@@ -71,14 +78,18 @@ class Filesort: public Sql_alloc
     own_select(false), 
     using_pq(false),
     sort_positions(sort_positions_arg),
-    sort_keys(NULL)
+    sort_keys(NULL),
+    set_all_read_bits(FALSE),
+    unpack(NULL)
   {
     DBUG_ASSERT(order);
   };
 
   ~Filesort() { cleanup(); }
   /* Prepare ORDER BY list for sorting. */
   Sort_keys* make_sortorder(THD *thd, JOIN *join, table_map first_table_bit);
+  /* Unpack temp table columns to base table columns*/
+  void (*unpack)(TABLE *);
 
 private:
   void cleanup();

sql/opt_subselect.cc

@@ -4285,11 +4285,11 @@ bool setup_sj_materialization_part2(JOIN_TAB *sjm_tab)
     sjm_tab->type= JT_ALL;
 
     /* Initialize full scan */
-    sjm_tab->read_first_record= join_read_record_no_init;
+    sjm_tab->read_first_record= join_init_read_record;
     sjm_tab->read_record.copy_field= sjm->copy_field;
     sjm_tab->read_record.copy_field_end= sjm->copy_field +
                                          sjm->sjm_table_cols.elements;
-    sjm_tab->read_record.read_record_func= rr_sequential_and_unpack;
+    sjm_tab->read_record.read_record_func= read_record_func_for_rr_and_unpack;
   }
 
   sjm_tab->bush_children->end[-1].next_select= end_sj_materialize;
@@ -7132,3 +7132,16 @@ bool Item_in_subselect::pushdown_cond_for_in_subquery(THD *thd, Item *cond)
   thd->lex->current_select= save_curr_select;
   DBUG_RETURN(FALSE);
 }
+
+/*
+  @brief
+    Check if a table is a SJM Scan table
+
+  @retval
+    TRUE     SJM scan table
+    FALSE    Otherwise
+*/
+bool TABLE_LIST::is_sjm_scan_table()
+{
+  return is_active_sjm() && sj_mat_info->is_sj_scan;
+}

sql/records.cc

@@ -822,3 +822,32 @@ inline void SORT_INFO::unpack_addon_fields(uchar *buff)
       field->unpack(field->ptr, buff + addonf->offset, buff_end, 0);
   }
 }
+
+
+/*
+  @brief
+    Read and unpack next record from a table
+
+  @details
+    The function first reads the next record from the table.
+    If a success then it unpacks the values to the base table fields.
+    This is used by SJM scan table to unpack the values of the materialized
+    table to the base table fields
+
+  @retval
+    0   Record successfully read.
+  @retval
+    -1   There is no record to be read anymore.
+    >0   Error
+*/
+int read_record_func_for_rr_and_unpack(READ_RECORD *info)
+{
+  int error;
+  if ((error= info->read_record_func_and_unpack_calls(info)))
+    return error;
+
+  for (Copy_field *cp= info->copy_field; cp != info->copy_field_end; cp++)
+    (*cp->do_copy)(cp);
+
+  return error;
+}

sql/records.h

@@ -55,6 +55,7 @@ struct READ_RECORD
   TABLE *table;                                 /* Head-form */
   Unlock_row_func unlock_row;
   Read_func read_record_func;
+  Read_func read_record_func_and_unpack_calls;
   THD *thd;
   SQL_SELECT *select;
   uint ref_length, reclength, rec_cache_size, error_offset;