Merge MDEV-26519: JSON_HB histograms into 10.8

include/json_lib.h

@@ -373,6 +373,9 @@ int json_find_paths_first(json_engine_t *je, json_find_paths_t *state,
 int json_find_paths_next(json_engine_t *je, json_find_paths_t *state);
 
 
+#define JSON_ERROR_OUT_OF_SPACE  (-1)
+#define JSON_ERROR_ILLEGAL_SYMBOL (-2)
+
 /*
   Converst JSON string constant into ordinary string constant
   which can involve unpacking json escapes and changing character set.
@@ -385,10 +388,13 @@ int json_unescape(CHARSET_INFO *json_cs,
                   uchar *res, uchar *res_end);
 
 /*
-  Converst ordinary string constant into JSON string constant.
-  which can involve appropriate escaping and changing character set.
-  Returns negative integer in the case of an error,
-  the length of the result otherwise.
+  Convert a string constant into JSON string constant.
+  This can involve appropriate escaping and changing the character set.
+  Returns the length of the result on success,
+  on error returns a negative error code.
+  Some error codes:
+    JSON_ERROR_OUT_OF_SPACE    Not enough space in the provided buffer
+    JSON_ERROR_ILLEGAL_SYMBOL  Source symbol cannot be represented in JSON
 */
 int json_escape(CHARSET_INFO *str_cs, const uchar *str, const uchar *str_end,
                 CHARSET_INFO *json_cs, uchar *json, uchar *json_end);

libmysqld/CMakeLists.txt

@@ -138,6 +138,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc
            ../sql/opt_trace.cc
            ../sql/xa.cc
            ../sql/json_table.cc
+           ../sql/opt_histogram_json.cc
            ${GEN_SOURCES}
            ${MYSYS_LIBWRAP_SOURCE}
 )

mysql-test/include/default_mysqld.cnf

@@ -118,6 +118,7 @@ binlog-direct-non-transactional-updates
 
 default-storage-engine=myisam
 use_stat_tables=preferably
+histogram_type=json_hb
 
 loose-ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem
 loose-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem

mysql-test/include/histogram_replaces.inc

@@ -0,0 +1 @@
+--source include/json_hb_histogram.inc

mysql-test/include/json_hb_histogram.inc

@@ -0,0 +1,3 @@
+# Remove non-deterministic parts of JSON_HB histogram
+
+--replace_regex /("(collected_at|collected_by)": )"[^"]*"/\1"REPLACED"/

mysql-test/main/derived_cond_pushdown.result

@@ -18387,7 +18387,7 @@ explain extended select t2.a,t2.b,t2.c,t.c as t_c,t.max,t.min
 from t2, t3, (select c, max(b) max, min(b) min from t4 group by c) t
 where t2.b < 40 and t2.a=t3.a and t3.c=t.c;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	90	63.28	Using where
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	90	60.00	Using where
 1	PRIMARY	t3	ref	idx_a	idx_a	5	test.t2.a	1	100.00	Using where
 1	PRIMARY	<derived2>	ref	key0	key0	128	test.t3.c	10	100.00	
 2	DERIVED	t4	ALL	idx_c	NULL	NULL	NULL	160	100.00	Using temporary; Using filesort
@@ -18406,7 +18406,7 @@ EXPLAIN
           "table_name": "t2",
           "access_type": "ALL",
           "rows": 90,
-          "filtered": 63.28125,
+          "filtered": 60,
           "attached_condition": "t2.b < 40 and t2.a is not null"
         }
       },
@@ -18910,7 +18910,7 @@ explain extended select *
 from t2, t3, (select c, b, sum(b) over (partition by c) from t4 ) t
 where t2.b < 40 and t2.a=t3.a and t3.c=t.c;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	90	63.28	Using where
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	90	60.00	Using where
 1	PRIMARY	t3	ref	idx_a	idx_a	5	test.t2.a	1	100.00	Using where
 1	PRIMARY	<derived2>	ref	key0	key0	128	test.t3.c	10	100.00	
 2	DERIVED	t4	ALL	idx_c	NULL	NULL	NULL	160	100.00	Using temporary
@@ -18929,7 +18929,7 @@ EXPLAIN
           "table_name": "t2",
           "access_type": "ALL",
           "rows": 90,
-          "filtered": 63.28125,
+          "filtered": 60,
           "attached_condition": "t2.b < 40 and t2.a is not null"
         }
       },

mysql-test/main/group_min_max.result

@@ -2080,27 +2080,27 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 explain extended select a1,a2,min(b),max(b) from t1
 where (a1 = 'b' or a1 = 'd' or a1 = 'a' or a1 = 'c') and (a2 > 'a') and (c > 'a111') group by a1,a2;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	range	idx_t1_0,idx_t1_1,idx_t1_2	idx_t1_1	130	NULL	276	99.22	Using where; Using index
+1	SIMPLE	t1	range	idx_t1_0,idx_t1_1,idx_t1_2	idx_t1_1	130	NULL	276	98.44	Using where; Using index
 Warnings:
 Note	1003	select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2`,min(`test`.`t1`.`b`) AS `min(b)`,max(`test`.`t1`.`b`) AS `max(b)` from `test`.`t1` where (`test`.`t1`.`a1` = 'b' or `test`.`t1`.`a1` = 'd' or `test`.`t1`.`a1` = 'a' or `test`.`t1`.`a1` = 'c') and `test`.`t1`.`a2` > 'a' and `test`.`t1`.`c` > 'a111' group by `test`.`t1`.`a1`,`test`.`t1`.`a2`
 explain extended select a1,a2,b,min(c),max(c) from t1
 where (a1 = 'b' or a1 = 'd' or a1 = 'a' or a1 = 'c') and (a2 > 'a') and (d > 'xy2') group by a1,a2,b;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	ALL	idx_t1_0,idx_t1_1,idx_t1_2	NULL	NULL	NULL	512	40.43	Using where; Using temporary; Using filesort
+1	SIMPLE	t1	ALL	idx_t1_0,idx_t1_1,idx_t1_2	NULL	NULL	NULL	512	26.95	Using where; Using temporary; Using filesort
 Warnings:
 Note	1003	select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2`,`test`.`t1`.`b` AS `b`,min(`test`.`t1`.`c`) AS `min(c)`,max(`test`.`t1`.`c`) AS `max(c)` from `test`.`t1` where (`test`.`t1`.`a1` = 'b' or `test`.`t1`.`a1` = 'd' or `test`.`t1`.`a1` = 'a' or `test`.`t1`.`a1` = 'c') and `test`.`t1`.`a2` > 'a' and `test`.`t1`.`d` > 'xy2' group by `test`.`t1`.`a1`,`test`.`t1`.`a2`,`test`.`t1`.`b`
 explain extended select a1,a2,b,c from t1
 where (a1 = 'b' or a1 = 'd' or a1 = 'a' or a1 = 'c') and (a2 > 'a') and (d > 'xy2') group by a1,a2,b,c;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	ALL	idx_t1_0,idx_t1_1,idx_t1_2	NULL	NULL	NULL	512	40.43	Using where; Using temporary; Using filesort
+1	SIMPLE	t1	ALL	idx_t1_0,idx_t1_1,idx_t1_2	NULL	NULL	NULL	512	26.95	Using where; Using temporary; Using filesort
 Warnings:
 Note	1003	select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c` from `test`.`t1` where (`test`.`t1`.`a1` = 'b' or `test`.`t1`.`a1` = 'd' or `test`.`t1`.`a1` = 'a' or `test`.`t1`.`a1` = 'c') and `test`.`t1`.`a2` > 'a' and `test`.`t1`.`d` > 'xy2' group by `test`.`t1`.`a1`,`test`.`t1`.`a2`,`test`.`t1`.`b`,`test`.`t1`.`c`
 explain select a1,a2,b,max(c),min(c) from t2 where (a2 = 'a') and (b = 'b') or (b < 'b') group by a1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t2	index	NULL	idx_t2_1	163	NULL	548	Using where; Using index
 explain extended select a1,a2,b from t1 where (a1 = 'b' or a1 = 'd' or a1 = 'a' or a1 = 'c') and (a2 > 'a') and (c > 'a111') group by a1,a2,b;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	range	idx_t1_0,idx_t1_1,idx_t1_2	idx_t1_1	130	NULL	276	99.22	Using where; Using index
+1	SIMPLE	t1	range	idx_t1_0,idx_t1_1,idx_t1_2	idx_t1_1	130	NULL	276	98.44	Using where; Using index
 Warnings:
 Note	1003	select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2`,`test`.`t1`.`b` AS `b` from `test`.`t1` where (`test`.`t1`.`a1` = 'b' or `test`.`t1`.`a1` = 'd' or `test`.`t1`.`a1` = 'a' or `test`.`t1`.`a1` = 'c') and `test`.`t1`.`a2` > 'a' and `test`.`t1`.`c` > 'a111' group by `test`.`t1`.`a1`,`test`.`t1`.`a2`,`test`.`t1`.`b`
 explain select a1,a2,min(b),c from t2 where (a2 = 'a') and (c = 'a111') group by a1;

mysql-test/main/join_cache.result

@@ -6220,7 +6220,7 @@ EXPLAIN
             "key_length": "10",
             "used_key_parts": ["kp1", "kp2"],
             "rows": 836,
-            "filtered": 76.43428802,
+            "filtered": 76,
             "index_condition": "b.kp2 <= 10",
             "attached_condition": "b.kp2 <= 10 and b.col1 + 1 < 33333"
           },

mysql-test/main/join_outer.result

@@ -2795,15 +2795,15 @@ test.t3	analyze	status	OK
 explain extended select * from t1 left join t3 on t1.a=t3.b and t3.a<5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	10	100.00	
-1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	1000	1.96	Using where
+1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	1000	0.50	Using where
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t1` left join `test`.`t3` on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5) where 1
 # t3.filtered must less than 100%, too:
 explain extended select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	10	100.00	
 1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	100.00	
-1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	1000	1.96	Using where
+1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	1000	0.50	Using where
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t2`.`a` AS `a` from `test`.`t1` left join (`test`.`t3` join `test`.`t2`) on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5) where 1
 drop table t1,t2,t3;

mysql-test/main/join_outer_jcl6.result

@@ -2802,15 +2802,15 @@ test.t3	analyze	status	OK
 explain extended select * from t1 left join t3 on t1.a=t3.b and t3.a<5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	10	100.00	
-1	SIMPLE	t3	hash_ALL	NULL	#hash#$hj	5	test.t1.a	1000	1.96	Using where; Using join buffer (flat, BNLH join)
+1	SIMPLE	t3	hash_ALL	NULL	#hash#$hj	5	test.t1.a	1000	0.50	Using where; Using join buffer (flat, BNLH join)
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t1` left join `test`.`t3` on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5 and `test`.`t1`.`a` is not null) where 1
 # t3.filtered must less than 100%, too:
 explain extended select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	10	100.00	
 1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where; Using join buffer (flat, BNL join)
-1	SIMPLE	t3	hash_ALL	NULL	#hash#$hj	5	test.t1.a	1000	1.96	Using where; Using join buffer (incremental, BNLH join)
+1	SIMPLE	t3	hash_ALL	NULL	#hash#$hj	5	test.t1.a	1000	0.50	Using where; Using join buffer (incremental, BNLH join)
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t2`.`a` AS `a` from `test`.`t1` left join (`test`.`t3` join `test`.`t2`) on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5 and `test`.`t1`.`a` is not null) where 1
 drop table t1,t2,t3;

mysql-test/main/mysqld--help.result

@@ -368,7 +368,8 @@ The following specify which files/extra groups are read (specified before remain
  Specifies type of the histograms created by ANALYZE.
  Possible values are: SINGLE_PREC_HB - single precision
  height-balanced, DOUBLE_PREC_HB - double precision
- height-balanced.
+ height-balanced, JSON_HB - height-balanced, stored as
+ JSON.
  --host-cache-size=# How many host names should be cached to avoid resolving.
  (Automatically configured unless set explicitly)
  --idle-readonly-transaction-timeout=# 

mysql-test/main/opt_trace.result

@@ -2091,12 +2091,12 @@ explain  select * from t1 where a=1 and b=2 order by c limit 1	{
                   {
                     "column_name": "a",
                     "ranges": ["1 <= a <= 1"],
-                    "selectivity_from_histogram": 0.1796875
+                    "selectivity_from_histogram": 0.181
                   },
                   {
                     "column_name": "b",
                     "ranges": ["2 <= b <= 2"],
-                    "selectivity_from_histogram": 0.015625
+                    "selectivity_from_histogram": 0.021
                   }
                 ],
                 "cond_selectivity": 0.021
@@ -8083,20 +8083,20 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans'))
 
                     {
                         "access_type": "scan",
-                        "resulting_rows": 5.9375,
-                        "cost": 2.829589844,
+                        "resulting_rows": 5,
+                        "cost": 3.017089844,
                         "chosen": true
                     }
                 ],
                 "chosen_access_method": 
                 {
                     "type": "scan",
-                    "records": 5.9375,
-                    "cost": 2.829589844,
+                    "records": 5,
+                    "cost": 3.017089844,
                     "uses_join_buffering": false
                 }
             },
-            "rows_for_plan": 5.9375,
+            "rows_for_plan": 5,
             "cost_for_plan": 4.017089844,
             "rest_of_plan": 
             [
@@ -8114,22 +8114,22 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans'))
 
                             {
                                 "access_type": "scan",
-                                "resulting_rows": 804.6875,
-                                "cost": 256.8548584,
+                                "resulting_rows": 800,
+                                "cost": 220.9863281,
                                 "chosen": true
                             }
                         ],
                         "chosen_access_method": 
                         {
                             "type": "scan",
-                            "records": 804.6875,
-                            "cost": 256.8548584,
+                            "records": 800,
+                            "cost": 220.9863281,
                             "uses_join_buffering": false
                         }
                     },
-                    "rows_for_plan": 4777.832031,
-                    "cost_for_plan": 1216.438354,
-                    "estimated_join_cardinality": 4777.832031
+                    "rows_for_plan": 4000,
+                    "cost_for_plan": 1025.003418,
+                    "estimated_join_cardinality": 4000
                 }
             ]
         },
@@ -8146,20 +8146,20 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans'))
 
                     {
                         "access_type": "scan",
-                        "resulting_rows": 804.6875,
-                        "cost": 43.25976562,
+                        "resulting_rows": 800,
+                        "cost": 44.19726562,
                         "chosen": true
                     }
                 ],
                 "chosen_access_method": 
                 {
                     "type": "scan",
-                    "records": 804.6875,
-                    "cost": 43.25976562,
+                    "records": 800,
+                    "cost": 44.19726562,
                     "uses_join_buffering": false
                 }
             },
-            "rows_for_plan": 804.6875,
+            "rows_for_plan": 800,
             "cost_for_plan": 204.1972656,
             "pruned_by_heuristic": true
         }
@@ -8230,8 +8230,8 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans'))
 
                             {
                                 "access_type": "scan",
-                                "resulting_rows": 804.6875,
-                                "cost": 43.25976562,
+                                "resulting_rows": 800,
+                                "cost": 44.19726562,
                                 "chosen": false
                             }
                         ],
@@ -8245,8 +8245,8 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans'))
                     },
                     "rows_for_plan": 10,
                     "cost_for_plan": 26.02294779,
-                    "selectivity": 0.8046875,
-                    "estimated_join_cardinality": 8.046875
+                    "selectivity": 0.8,
+                    "estimated_join_cardinality": 8
                 }
             ]
         },
@@ -8263,20 +8263,20 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans'))
 
                     {
                         "access_type": "scan",
-                        "resulting_rows": 804.6875,
-                        "cost": 43.25976562,
+                        "resulting_rows": 800,
+                        "cost": 44.19726562,
                         "chosen": true
                     }
                 ],
                 "chosen_access_method": 
                 {
                     "type": "scan",
-                    "records": 804.6875,
-                    "cost": 43.25976562,
+                    "records": 800,
+                    "cost": 44.19726562,
                     "uses_join_buffering": false
                 }
             },
-            "rows_for_plan": 804.6875,
+            "rows_for_plan": 800,
             "cost_for_plan": 204.1972656,
             "pruned_by_cost": true
         }
@@ -8564,7 +8564,7 @@ test.t1	analyze	status	Engine-independent statistics collected
 test.t1	analyze	status	OK
 EXPLAIN EXTENDED SELECT * from t1 WHERE a between 1 and 5 and b <= 5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	0.22	Using where
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	0.25	Using where
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` between 1 and 5 and `test`.`t1`.`b` <= 5
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
@@ -8579,7 +8579,7 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns'))
             [
                 "1 <= a <= 5"
             ],
-            "selectivity_from_histogram": 0.046875
+            "selectivity_from_histogram": 0.05
         },
 
         {
@@ -8588,13 +8588,13 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns'))
             [
                 "NULL < b <= 5"
             ],
-            "selectivity_from_histogram": 0.046875
+            "selectivity_from_histogram": 0.05
         }
     ]
 ]
 EXPLAIN EXTENDED SELECT * from t1 WHERE a != 5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	100.00	Using where
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	99.00	Using where
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`a` <> 5
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
@@ -8610,13 +8610,13 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns'))
                 "NULL < a < 5",
                 "5 < a"
             ],
-            "selectivity_from_histogram": 1
+            "selectivity_from_histogram": 0.99
         }
     ]
 ]
 EXPLAIN EXTENDED SELECT * from t1 WHERE b >= 10 and b < 25;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	15.62	Using where
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	15.00	Using where
 Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where `test`.`t1`.`b` >= 10 and `test`.`t1`.`b` < 25
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
@@ -8631,7 +8631,7 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.selectivity_for_columns'))
             [
                 "10 <= b < 25"
             ],
-            "selectivity_from_histogram": 0.15625
+            "selectivity_from_histogram": 0.15
         }
     ]
 ]