WL#13811 Iterator based EXPLAIN FORMAT=JSON [1/2]

FORMAT=TREE uses ExplainIterator() to generate the EXPLAIN tree output.
With hypergraph optimizer enabled and FORMAT=JSON, follow this same
ExplainIterator() path to generate the new iterator-based JSON output.

Given an AccessPath, convert it into an intermediate Json structure.
This structure can be then used to print it into either json format or
tree format.

ExplainAccessPath() now returns a Json object. This object is passed
to ExplainJsonToString() to convert it into a string.
Explain_format_tree and Explain_format_json have their overridden
ExplainJsonToString() functions.

For supporting the new JSON format, use the same Explain_format_JSON
structure. But have a new enum member m_version which will indicate
whether the JSON format is iterator-based or not.

Materialize and other similar plans need to show their table iterator
plans as their parent plans. So generate a subplan tree for the table
iterator plan and push the main Materialize plan under the leaf node
of the subplan tree.
Note: Earlier, we used to print only the first line of the table
iterator subplan.

Support EXPLAIN ANALYZE. Use iterator->GetProfiler() to collect the
actual costs.

Show top level re-written query in the new JSON format.

SHA Subplan token related changes are done in Explain_format_tree
function. It's a bit different than the original. The subtokens are
generated *while* the tree is traversed for explain output. Earlier,
the whole subplan tree was scanned at each tree node to generate the
token for that node (GetForceSubplanToken was called for each node). A
side effect of this commit is that the list of subplan tokens that is
returned back has a different order (child to parent), but that does
not matter. So an mtr test needed to be changed.

For this commit, the JSON format displays only cost fields. Many more
fields will be included in the next commit.

In --hypergraph mode, many mtr tests had to skip FORMAT=JSON
statements. For any test that has only EXPLAIN statements, the test
itself is skipped.

Change-Id: Ibf5f6c267ebd9c65d1c1e428227c96e88d98ac85

Author: Amit Khandekar

mysql-test/include/explain_for_connection.inc

@@ -28,6 +28,7 @@ SET DEBUG_SYNC= 'RESET';
 # Grab output of regular explain
 --replace_regex $replace
 --output $MYSQL_TMP_DIR/EN
+--skip_if_hypergraph  # Depends on the query plan.
 eval EXPLAIN $fmt $query;
 
 
@@ -43,6 +44,7 @@ SET DEBUG_SYNC= 'now WAIT_FOR ready_for_explain';
 SET DEBUG_SYNC= 'after_explain_other SIGNAL explained';
 --replace_regex $replace
 --output $MYSQL_TMP_DIR/EO
+--skip_if_hypergraph  # Depends on the query plan.
 eval EXPLAIN $fmt FOR CONNECTION $QID;
 
 connection cq;

mysql-test/include/explain_for_connection_small.inc

@@ -1,3 +1,5 @@
+--source include/not_hypergraph.inc  # Non-traditional formats differ in hypergraph.
+
   --source include/have_debug.inc
   --source include/have_debug_sync.inc
   --source include/not_windows.inc

mysql-test/include/index_merge_ror.inc

@@ -135,6 +135,7 @@ select key1,key2 from t1 where key1=100 and key2=100;
 explain select key1,key2,key3,key4,filler1 from t1 where key1=100 and key2=100 or key3=100 and key4=100;
 # Mask cost values since it varies between runs how much of the indexes
 # are cached in InnoDB's buffer pool
+--skip_if_hypergraph  # Depends on the query plan.
 --replace_regex /(_cost": )"[0-9.]+"/\1"#"/
 explain format=json select key1,key2,key3,key4,filler1 from t1 where key1=100 and key2=100 or key3=100 and key4=100;
 select key1,key2,key3,key4,filler1 from t1 where key1=100 and key2=100 or key3=100 and key4=100;

mysql-test/include/order_by.inc

@@ -242,6 +242,7 @@ ANALYZE TABLE t1;
 explain select * from t1 where (a = 1 and b is null and c = 'b') or (a > 2) order by a desc;
 select * from t1 where (a = 1 and b is null and c = 'b') or (a > 2) order by a desc;
 explain select * from t1 where a >= 1 and a < 3 order by a desc;
+--skip_if_hypergraph  # Depends on the query plan.
 explain format=json select * from t1 where a >= 1 and a < 3 order by a desc;
 --partially_sorted_result 1
 select * from t1 where a >= 1 and a < 3 order by a desc;

mysql-test/include/subquery_sj.inc

@@ -417,6 +417,7 @@ update t22 set c = '2005-12-08 15:58:27' where a = 255;
 ANALYZE TABLE t1, t11, t12, t21, t22;
 explain select t21.* from t21,t22 where t21.a = t22.a and 
 t22.a in (select t12.a from t11, t12 where t11.a in(255,256) and t11.a = t12.a and t11.c is null) and t22.c is null order by t21.a;
+--skip_if_hypergraph  # Depends on the query plan.
 explain format=json select * from t1 where a in (select a from t11);
 select t21.* from t21,t22 where t21.a = t22.a and 
 t22.a in (select t12.a from t11, t12 where t11.a in(255,256) and t11.a = t12.a and t11.c is null) and t22.c is null order by t21.a;
@@ -454,6 +455,7 @@ select * from t1 where 2 in (select a from t0);
 #
 let $query=select * from (select a from t0) x where a in (select a from t1);
 --eval explain $query
+--skip_if_hypergraph  # Depends on the query plan.
 --eval explain format=json $query
 drop table t0, t1;
 
@@ -579,6 +581,7 @@ insert into t3 select A.a + 10*B.a, A.a + 10*B.a, 'filler' from t1 A, t1 B, t1 C
 ANALYZE TABLE t1, t2, t3;
 explain select * from t1, t3 where t3.a in (select a from t2) and (t3.a < 10 or t3.a >30) and t1.a =3;
 --replace_regex /"rows_produced_per_join": 792/"rows_produced_per_join": 793/
+--skip_if_hypergraph  # Depends on the query plan.
 explain format=json select * from t1, t3 where t3.a in (select a from t2) and (t3.a < 10 or t3.a >30) and t1.a =3;
 
 #
@@ -3140,6 +3143,7 @@ ANALYZE TABLE t1, t2, t3;
 --replace_column 11 10
 --eval EXPLAIN $query
 --replace_regex /"filtered": "50.00"/"filtered": "10.00"/ /"read_cost": "0.12"/"read_cost": "0.13"/ /"prefix_cost": "11.69"/"prefix_cost": "11.70"/
+--skip_if_hypergraph  # Depends on the query plan.
 --eval EXPLAIN FORMAT=JSON $query
 
 DROP TABLE t1,t2,t3;
@@ -5017,6 +5021,7 @@ WHERE g1 NOT IN
 );
 ANALYZE TABLE t1, t2, t3;
 eval explain $query;
+--skip_if_hypergraph  # Depends on the query plan.
 eval explain format=json $query;
 eval $query;
 
@@ -6422,6 +6427,7 @@ SELECT t2.vc, t2.ik AS t2_ik, t1.ik AS t1_ik
 FROM t2 LEFT JOIN t1 ON t2.vc=t1.vc
 WHERE t2.vc IN (SELECT vc FROM t2 AS t3);
 ANALYZE TABLE t1, t2;
+--skip_if_hypergraph  # Depends on the query plan.
 eval explain format=json $query;
 --sorted_result
 eval $query;
@@ -6502,6 +6508,7 @@ WHERE (col_varchar_nokey, col_varchar_nokey) IN
     FROM t2 AS derived2
    );
 
+--skip_if_hypergraph  # Depends on the query plan.
 eval explain format=json $query;
 --sorted_result
 eval $query;
@@ -6539,6 +6546,7 @@ WHERE (col_int_nokey, col_int_nokey) IN
     FROM t2
    );
 
+--skip_if_hypergraph  # Depends on the query plan.
 eval explain format=json $query;
 --sorted_result
 eval $query;
@@ -6622,6 +6630,7 @@ WHERE (alias2.col_varchar_key, alias2.col_varchar_key) IN
         FROM t1 AS sq2_alias1, t1 AS sq2_alias2
         WHERE sq2_alias2.col_int_key < 2);
 --replace_regex /"eval_cost": "0.13"/"eval_cost": "0.12"/
+--skip_if_hypergraph  # Depends on the query plan.
 eval explain format=json $query;
 --sorted_result
 eval $query;

mysql-test/include/window_bitwise_ops.inc

@@ -62,6 +62,7 @@ SELECT HEX(i), HEX(li), HEX(b), HEX(lb),
 --echo our dummy double d SUM which forces non-optimized mode
 --echo iff windowing_use_high_precision==TRUE
 ANALYZE TABLE t;
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON
 SELECT HEX(i), HEX(li), HEX(b), HEX(lb),
        HEX(BIT_OR(i) OVER w), HEX(BIT_OR(li) OVER w),

mysql-test/include/window_std_var.inc

@@ -28,6 +28,7 @@ SELECT i, j, VARIANCE(i+j) OVER (ORDER BY j DESC ROWS UNBOUNDED PRECEDING) varia
 
 --echo Check that we eliminate redundant sorting in ORDER BY even with wfs
 SELECT i, j, STD(i+j) OVER (ROWS UNBOUNDED PRECEDING) std FROM t ORDER BY NULL DESC;
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON SELECT i, j, std(i+j) OVER (ROWS UNBOUNDED PRECEDING) STD FROM t ORDER BY NULL DESC;
 --echo With LIMIT
 SELECT i, j, STD(i+j) OVER (ROWS UNBOUNDED PRECEDING) STD FROM t ORDER BY std DESC LIMIT 3;

mysql-test/r/explain_tree.result

@@ -729,7 +729,7 @@ ERROR 42000: This version of MySQL doesn't yet support 'EXPLAIN ANALYZE FOR CONN
 EXPLAIN ANALYZE FORMAT=TRADITIONAL SELECT 1;
 ERROR 42000: This version of MySQL doesn't yet support 'FORMAT=TRADITIONAL with EXPLAIN ANALYZE'
 EXPLAIN ANALYZE FORMAT=JSON SELECT 1;
-ERROR 42000: This version of MySQL doesn't yet support 'FORMAT=JSON with EXPLAIN ANALYZE'
+ERROR 42000: This version of MySQL doesn't yet support 'EXPLAIN ANALYZE with JSON format'
 EXPLAIN ANALYZE FORMAT=TREE SELECT 1;
 EXPLAIN
 -> Rows fetched before execution  (cost=0.00..0.00 rows=1) (actual time=N.NNN..N.NNN rows=1 loops=1)
@@ -870,10 +870,11 @@ EXPLAIN
 -> Select #2 (subquery in projection; dependent)
     -> Limit: 1 row(s)  (cost=0.45..0.45 rows=1)
         -> Filter: <if>(outer_field_is_not_null, <is_not_null_test>(d.a), true)  (cost=0.45..0.45 rows=1)
-            -> Alternative plans for IN subquery: Index lookup unless a IS NULL [other sub-iterators not shown]
+            -> Alternative plans for IN subquery: Index lookup unless a IS NULL
                 -> Covering index lookup on d using <auto_key0> (a=<cache>(t1.a) or NULL)
                     -> Materialize  (cost=0.45..0.45 rows=1)
                         -> Table scan on t1  (cost=0.35 rows=1)
+                -> Table scan on d
 
 Warnings:
 Warning	1287	Setting user variables within expressions is deprecated and will be removed in a future release. Consider alternatives: 'SET variable=expression, ...', or 'SELECT expression(s) INTO variables(s)'.
@@ -1160,8 +1161,8 @@ EXPLAIN
             -> Aggregate: max(x1.m1)  (cost=2.50..2.50 rows=1) (never executed)
                 -> Table scan on x1  (cost=2.50..2.50 rows=0) (never executed)
                     -> Materialize CTE x1  (cost=2.50..2.50 rows=0) (never executed)
-                        -> Table scan on <temporary> (never executed)
-                            -> Aggregate using temporary table (never executed)
+                        -> Table scan on <temporary>  (never executed)
+                            -> Aggregate using temporary table  (never executed)
                                 -> Table scan on t1  (cost=0.35 rows=1) (never executed)
 
 DROP TABLE t1;

mysql-test/r/subplan_tokens.result

@@ -25,7 +25,7 @@ EXPLAIN
     -> [0xeed2c0bd3e39ba93] Table scan on t1  (cost=0.08..0.25 rows=3)
 
 To force this plan, use:
-SET DEBUG='+d,subplan_tokens,force_subplan_0xd3135ef8a92cb9b8,force_subplan_0xeed2c0bd3e39ba93';
+SET DEBUG='+d,subplan_tokens,force_subplan_0xeed2c0bd3e39ba93,force_subplan_0xd3135ef8a92cb9b8';
 
 SET DEBUG='+d,subplan_tokens,'
   'force_subplan_0x38d170e70c04e92c,'
@@ -40,7 +40,7 @@ EXPLAIN
             -> [0x079e429c703ec298] Table scan on t2  (cost=0.08..0.25 rows=3)
 
 To force this plan, use:
-SET DEBUG='+d,subplan_tokens,force_subplan_0x1836646581797f8f,force_subplan_0x38d170e70c04e92c,force_subplan_0xeed2c0bd3e39ba93,force_subplan_0x079e429c703ec298';
+SET DEBUG='+d,subplan_tokens,force_subplan_0xeed2c0bd3e39ba93,force_subplan_0x079e429c703ec298,force_subplan_0x38d170e70c04e92c,force_subplan_0x1836646581797f8f';
 
 SET DEBUG='-d,subplan_tokens';
 EXPLAIN FORMAT=tree SELECT * FROM t1 JOIN t1 AS t2 ON t1.a=t2.a ORDER BY t1.a;

mysql-test/suite/innodb/include/explain_then_rollback_and_commit.inc

@@ -1,3 +1,4 @@
+--skip_if_hypergraph  # Depends on the query plan.
 --eval EXPLAIN FORMAT=JSON $sql_to_do
 
 BEGIN;

mysql-test/suite/innodb/t/blob_partial_update.test

@@ -23,10 +23,12 @@ INSERT INTO tab VALUES
 ANALYZE TABLE tab;
 --echo # Sven
 select json_extract(j1, '$[0]') from tab;
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'VISWA');
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Viswa');
 --echo # Show Viswa
 select json_extract(j1, '$[0]') from tab;
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Nath');
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Nath');
 --echo # Show Nath
@@ -36,6 +38,7 @@ select json_extract(j1, '$[0]') from tab;
 
 ALTER TABLE tab MODIFY j1 LONGTEXT ;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setlongtext');
 
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setlongtext');
@@ -50,6 +53,7 @@ SELECT JSON_STORAGE_FREE(j1) FROM tab;
 --echo # Show Setlongtext
 SELECT json_extract(j1, '$[0]') FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Replongtext');
 
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Replongtext');
@@ -67,6 +71,7 @@ SELECT json_extract(j1, '$[0]') from tab;
 --echo When column modify from LONGTEXT to JSON
 ALTER TABLE tab MODIFY j1 JSON ;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setjson');
 
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setjson');
@@ -76,6 +81,7 @@ ANALYZE TABLE tab;
 --echo # Show Setjson
 SELECT  json_extract(j1, '$[0]') FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repjson');
 
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repjson');
@@ -90,6 +96,7 @@ ALTER TABLE tab ROW_FORMAT = COMPRESSED ;
 
 SHOW CREATE TABLE tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setcomp');
 
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setcomp');
@@ -99,6 +106,7 @@ ANALYZE TABLE tab;
 --echo # Show Setcomp
 SELECT  json_extract(j1, '$[0]') FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repcomp');
 
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repcomp');
@@ -113,6 +121,7 @@ ANALYZE TABLE tab;
 
 SHOW CREATE TABLE tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setdyna');
 
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setdyna');
@@ -122,6 +131,7 @@ ANALYZE TABLE tab;
 --echo # Show Setdyna
 SELECT  json_extract(j1, '$[0]') FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repdyna');
 
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repdyna');
@@ -136,6 +146,7 @@ ALTER TABLE tab ROW_FORMAT = REDUNDANT ;
 
 SHOW CREATE TABLE tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setredu');
 
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setredu');
@@ -145,6 +156,7 @@ ANALYZE TABLE tab;
 --echo # Show Setredu
 SELECT  json_extract(j1, '$[0]') FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repredu');
 
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repredu');
@@ -159,6 +171,7 @@ SHOW CREATE TABLE tab;
 
 ANALYZE TABLE tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setcomp');
 
 UPDATE tab SET j1 = JSON_SET(j1, '$[0]', 'Setcomp');
@@ -168,6 +181,7 @@ ANALYZE TABLE tab;
 --echo # Show Setcomp
 SELECT  json_extract(j1, '$[0]') FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repcomp');
 
 UPDATE tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repcomp');
@@ -178,6 +192,7 @@ SELECT json_extract(j1, '$[0]') FROM tab;
 --echo # Create a view on JSON column
 CREATE VIEW view_tab AS SELECT * FROM tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE view_tab SET j1 = JSON_SET(j1, '$[0]', 'Setview');
 
 UPDATE view_tab SET j1 = JSON_SET(j1, '$[0]', 'Setview');
@@ -192,6 +207,7 @@ SELECT JSON_STORAGE_FREE(j1) FROM view_tab;
 --echo # Show Setview
 SELECT  json_extract(j1, '$[0]') FROM view_tab;
 
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE view_tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repview');
 
 UPDATE view_tab SET j1 = JSON_REPLACE(j1, '$[0]', 'Repview');
@@ -296,6 +312,7 @@ INSERT INTO tab2 VALUES(1, '{"a":"a","b":"b"}', DEFAULT);
 
 ANALYZE TABLE tab2;
 
+--skip_if_hypergraph  # Depends on the query plan.
 --error ER_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN
 EXPLAIN FORMAT=JSON UPDATE tab2 SET c2 = JSON_SET(c2, '$.a', 'x');
 
@@ -319,6 +336,7 @@ INSERT INTO tab3 VALUES(3, '{"Chetan": [300,400],"Sasri": [500,600]}','{"Dhoni":
 SELECT * FROM tab3 order by a;
 
 --echo Check with JSON_SET
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab3 SET j1 = JSON_SET(j1, '$.a', 'x'),
 j2 = JSON_SET(j2, '$.a', 'y'),
 j1 = JSON_SET(j1, '$.b', 'z');
@@ -330,6 +348,7 @@ j1 = JSON_SET(j1, '$.b', 'z');
 SELECT * FROM tab3 order by a;
 
 --echo Check with JSON_REPLACE
+--skip_if_hypergraph  # Depends on the query plan.
 EXPLAIN FORMAT=JSON UPDATE tab3 SET j1 = JSON_REPLACE(j1,'$[1].A1', 'partRe') ;
 
 UPDATE tab3 SET j1 = JSON_REPLACE(j1,'$[1].A1', 'PartRe') ;

mysql-test/suite/innodb/t/blob_partial_update_2.test

@@ -309,6 +309,7 @@ INSERT INTO tab2 VALUES(1, '{"a":"a","b":"b"}', DEFAULT);
 
 ANALYZE TABLE tab2;
 
+--skip_if_hypergraph  # Depends on the query plan.
 --error ER_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN
 EXPLAIN FORMAT=JSON UPDATE tab2 SET c2 = JSON_SET(c2, '$.a', 'x');
 

mysql-test/suite/innodb_fts/t/opt.test

@@ -1028,6 +1028,7 @@ DROP TABLE t1;
 CREATE TABLE t1(f1 CHAR(1) CHARSET latin1, FULLTEXT(f1)) ENGINE=INNODB;
 EXPLAIN SELECT 1 FROM t1 WHERE 1.238585e+308 <= MATCH(f1) AGAINST ('1' IN BOOLEAN MODE);
 
+--skip_if_hypergraph  # Depends on query plan.
 EXPLAIN FORMAT = JSON SELECT 1 FROM t1 WHERE 1.238585e+308 <= MATCH(f1) AGAINST ('1' IN BOOLEAN MODE);
 
 DROP TABLE t1;