Bug#34787357 Hypergraph: row estimates for field=non_field_term ignores indexes and histogram [2/2]

This patch concerns selectivity estimates for the Hypergraph optimizer, for
predicates like:

table.field <op> <expression>

where <expression> is something else than a field, and independent of
'table'. If <op> is '=' the selectivity of these predicates was previously
estimated as 1.0 or 0.1, depending on the specific predicate.

But if we have statistics (index or histogram) on the number of
distinct values for 'field', and make the assumption that the value of
<expression> is uniformly distributed over the values of 'field', we
can make a better estimate as:

selectivity = 1.0 / <number of distinct values>

This patch makes such estimates if a suitable histogram or index can
be found. The patch also makes a similar change when <op> is '<>'
(i.e. not equal).  Previously, we estimate the selectivity to be 1.0
or 0.9, depending on the specific predicate.  And now, if we have a
suitable index or histogram, we make the following estimate:
predicates like:

selectivity = 1.0 - 1.0 / <number of distinct values>

For the '<', '>', '<=' and '>=' operators, we previously estimated the
selectivity to 1.0 if <expression> was an (independent) subquery or
non-deterministic. Now we use the default value for other types of
expressions for these operators (i.e. 0.33).

Change-Id: I34c307b12216025bedf6e995ee89a58793f59aac

Author: Jan Wedvik

mysql-test/include/index_merge_ror_cpk.inc

@@ -77,7 +77,7 @@ analyze table t1;
 --echo # Verify that range scan on CPK is ROR
 --echo # (use index_intersection because it is impossible to check that for index union)
 --echo # Column 9, rows, can change depending on innodb-page-size.
---replace_regex $elide_costs
+--replace_regex $elide_costs_and_rows
 explain select * from t1 where pk1 = 1 and pk2 < 80  and key1=0;
 --echo # CPK scan + 1 ROR range scan is a special case
 --sorted_result

mysql-test/r/explain_json_hypergraph.result

@@ -9,11 +9,11 @@ Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
 EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE f1 = ( SELECT MIN(f1) FROM t1 AS i WHERE i.f1 > t1.f1 );
 EXPLAIN
--> Filter: (t1.f1 = (select #2))  (cost=3.1..3.1 rows=1)
+-> Filter: (t1.f1 = (select #2))  (cost=2.5..2.5 rows=1)
     -> Table scan on t1  (cost=0.0833..0.25 rows=3)
     -> Select #2 (subquery in condition; dependent)
-        -> Aggregate: min(i.f1)  (cost=0.85..0.85 rows=1)
-            -> Filter: (i.f1 > t1.f1)  (cost=0.183..0.55 rows=3)
+        -> Aggregate: min(i.f1)  (cost=0.65..0.65 rows=1)
+            -> Filter: (i.f1 > t1.f1)  (cost=0.55..0.55 rows=1)
                 -> Table scan on i  (cost=0.0833..0.25 rows=3)
 
 Warnings:
@@ -47,9 +47,9 @@ EXPLAIN
           "condition": "(i.f1 > t1.f1)",
           "operation": "Filter: (i.f1 > t1.f1)",
           "access_type": "filter",
-          "estimated_rows": 3.0,
+          "estimated_rows": 0.9998999834060669,
           "estimated_total_cost": 0.55,
-          "estimated_first_row_cost": 0.18333333333333335
+          "estimated_first_row_cost": 0.55
         }
       ],
       "heading": "Select #2 (subquery in condition; dependent)",
@@ -62,22 +62,22 @@ EXPLAIN
       "access_type": "aggregate",
       "estimated_rows": 1.0,
       "subquery_location": "condition",
-      "estimated_total_cost": 0.8500000000000001,
-      "estimated_first_row_cost": 0.8500000000000001
+      "estimated_total_cost": 0.6499899983406068,
+      "estimated_first_row_cost": 0.6499899983406068
     }
   ],
   "condition": "(t1.f1 = (select #2))",
   "operation": "Filter: (t1.f1 = (select #2))",
   "access_type": "filter",
   "estimated_rows": 1.0,
-  "estimated_total_cost": 3.1000000000000005,
-  "estimated_first_row_cost": 3.1000000000000005
+  "estimated_total_cost": 2.4999699950218206,
+  "estimated_first_row_cost": 2.4999699950218206
 }
 Warnings:
 Note	1276	Field or reference 'test.t1.f1' of SELECT #2 was resolved in SELECT #1
 EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE f1 > ( SELECT f1 FROM t1 LIMIT 1 );
 EXPLAIN
--> Filter: (t1.f1 > (select #2))  (cost=0.267..0.633 rows=3)
+-> Filter: (t1.f1 > (select #2))  (cost=0.633..0.633 rows=1)
     -> Table scan on t1  (cost=0.0833..0.25 rows=3)
     -> Select #2 (subquery in condition; run only once)
         -> Limit: 1 row(s)  (cost=0.0833..0.0833 rows=1)
@@ -123,9 +123,9 @@ EXPLAIN
   "condition": "(t1.f1 > (select #2))",
   "operation": "Filter: (t1.f1 > (select #2))",
   "access_type": "filter",
-  "estimated_rows": 3.0,
+  "estimated_rows": 0.9998999834060669,
   "estimated_total_cost": 0.6333333333333333,
-  "estimated_first_row_cost": 0.26666666666666666
+  "estimated_first_row_cost": 0.6333333333333333
 }
 drop table t1;
 #
@@ -800,7 +800,7 @@ EXPLAIN
         -> Table scan on x2  (rows=1)
     -> Select #2 (subquery in condition; dependent)
         -> Aggregate: min(x3.a)  (rows=1)
-            -> Filter: (x1.a = x3.a)  (rows=1)
+            -> Filter: (x1.a = x3.a)  (rows=0.1)
                 -> Table scan on x3  (rows=1)
 
 Warnings:
@@ -844,7 +844,7 @@ EXPLAIN
           "condition": "(x1.a = x3.a)",
           "operation": "Filter: (x1.a = x3.a)",
           "access_type": "filter",
-          "estimated_rows": 1.0,
+          "estimated_rows": 0.10000000149011612,
           "estimated_total_cost": 0.35,
           "estimated_first_row_cost": 0.35
         }
@@ -859,8 +859,8 @@ EXPLAIN
       "access_type": "aggregate",
       "estimated_rows": 1.0,
       "subquery_location": "condition",
-      "estimated_total_cost": 0.44999999999999996,
-      "estimated_first_row_cost": 0.44999999999999996
+      "estimated_total_cost": 0.36000000014901157,
+      "estimated_first_row_cost": 0.36000000014901157
     }
   ],
   "join_type": "inner join",
@@ -885,7 +885,7 @@ EXPLAIN
         -> Table scan on x2  (rows=1)
     -> Select #2 (subquery in extra conditions; dependent)
         -> Aggregate: min(x3.a)  (rows=1)
-            -> Filter: (x1.a = x3.a)  (rows=1)
+            -> Filter: (x1.a = x3.a)  (rows=0.1)
                 -> Table scan on x3  (rows=1)
 
 Warnings:
@@ -929,7 +929,7 @@ EXPLAIN
           "condition": "(x1.a = x3.a)",
           "operation": "Filter: (x1.a = x3.a)",
           "access_type": "filter",
-          "estimated_rows": 1.0,
+          "estimated_rows": 0.10000000149011612,
           "estimated_total_cost": 0.35,
           "estimated_first_row_cost": 0.35
         }
@@ -944,8 +944,8 @@ EXPLAIN
       "access_type": "aggregate",
       "estimated_rows": 1.0,
       "subquery_location": "extra conditions",
-      "estimated_total_cost": 0.44999999999999996,
-      "estimated_first_row_cost": 0.44999999999999996
+      "estimated_total_cost": 0.36000000014901157,
+      "estimated_first_row_cost": 0.36000000014901157
     }
   ],
   "join_type": "inner join",

mysql-test/r/explain_tree_hypergraph.result

@@ -11,32 +11,32 @@ test.t1	analyze	status	OK
 EXPLAIN FORMAT=TREE SELECT * FROM t1 x1
 WHERE x1.b = (SELECT MAX(b) FROM t1 x2 WHERE x2.a>x1.a);
 EXPLAIN
--> Filter: (x1.b = (select #2))  (cost=2.38..23.8 rows=10)
+-> Filter: (x1.b = (select #2))  (cost=1.71..17.1 rows=10)
     -> Table scan on x1  (cost=0.025..0.25 rows=10)
     -> Select #2 (subquery in condition; dependent)
-        -> Aggregate: max(x2.b)  (cost=2.25..2.25 rows=1)
-            -> Filter: (x2.a > x1.a)  (cost=0.125..1.25 rows=10)
+        -> Aggregate: max(x2.b)  (cost=1.58..1.58 rows=1)
+            -> Filter: (x2.a > x1.a)  (cost=0.375..1.25 rows=3.33)
                 -> Table scan on x2  (cost=0.025..0.25 rows=10)
 
 Warnings:
 Note	1276	Field or reference 'test.x1.a' of SELECT #2 was resolved in SELECT #1
 EXPLAIN FORMAT=TREE SELECT * FROM t1 x1
 WHERE x1.b IN (SELECT MAX(b) FROM t1 x2 WHERE x2.a>x1.a);
 EXPLAIN
--> Filter: <in_optimizer>(x1.b,<exists>(select #2))  (cost=2.48..24.8 rows=10)
+-> Filter: <in_optimizer>(x1.b,<exists>(select #2))  (cost=1.81..18.1 rows=10)
     -> Table scan on x1  (cost=0.025..0.25 rows=10)
     -> Select #2 (subquery in condition; dependent)
-        -> Filter: (<cache>(x1.b) = <ref_null_helper>(max(x2.b)))  (cost=2.35..2.35 rows=1)
-            -> Aggregate: max(x2.b)  (cost=2.25..2.25 rows=1)
-                -> Filter: (x2.a > x1.a)  (cost=0.125..1.25 rows=10)
+        -> Filter: (<cache>(x1.b) = <ref_null_helper>(max(x2.b)))  (cost=1.68..1.68 rows=1)
+            -> Aggregate: max(x2.b)  (cost=1.58..1.58 rows=1)
+                -> Filter: (x2.a > x1.a)  (cost=0.375..1.25 rows=3.33)
                     -> Table scan on x2  (cost=0.025..0.25 rows=10)
 
 Warnings:
 Note	1276	Field or reference 'test.x1.a' of SELECT #2 was resolved in SELECT #1
 EXPLAIN FORMAT=TREE SELECT * FROM t1 x1
 WHERE x1.b = (SELECT MAX(b) FROM t1 x2 WHERE x2.a>5);
 EXPLAIN
--> Filter: (x1.b = (select #2))  (cost=1.19..2.31 rows=10)
+-> Filter: (x1.b = (select #2))  (cost=2.31..2.31 rows=1)
     -> Table scan on x1  (cost=0.025..0.25 rows=10)
     -> Select #2 (subquery in condition; run only once)
         -> Aggregate: max(x2.b)  (cost=1.06..1.06 rows=1)
@@ -58,22 +58,22 @@ EXPLAIN
 EXPLAIN FORMAT=TREE SELECT * FROM t1 x1
 WHERE x1.b = (SELECT MAX(b) FROM t1 x2 WHERE x2.a>10*rand(0));
 EXPLAIN
--> Filter: (x1.b = (select #2))  (cost=2.38..23.8 rows=10)
+-> Filter: (x1.b = (select #2))  (cost=1.71..17.1 rows=10)
     -> Table scan on x1  (cost=0.025..0.25 rows=10)
     -> Select #2 (subquery in condition; uncacheable)
-        -> Aggregate: max(x2.b)  (cost=2.25..2.25 rows=1)
-            -> Filter: (x2.a > (10 * rand(0)))  (cost=0.125..1.25 rows=10)
+        -> Aggregate: max(x2.b)  (cost=1.58..1.58 rows=1)
+            -> Filter: (x2.a > (10 * rand(0)))  (cost=0.375..1.25 rows=3.33)
                 -> Table scan on x2  (cost=0.025..0.25 rows=10)
 
 EXPLAIN FORMAT=TREE SELECT * FROM t1 x1
 WHERE x1.b IN (SELECT MAX(b) FROM t1 x2 WHERE x2.a>10*rand(0));
 EXPLAIN
--> Filter: <in_optimizer>(x1.b,<exists>(select #2))  (cost=2.48..24.8 rows=10)
+-> Filter: <in_optimizer>(x1.b,<exists>(select #2))  (cost=1.81..18.1 rows=10)
     -> Table scan on x1  (cost=0.025..0.25 rows=10)
     -> Select #2 (subquery in condition; dependent)
-        -> Filter: (<cache>(x1.b) = <ref_null_helper>(max(x2.b)))  (cost=2.35..2.35 rows=1)
-            -> Aggregate: max(x2.b)  (cost=2.25..2.25 rows=1)
-                -> Filter: (x2.a > (10 * rand(0)))  (cost=0.125..1.25 rows=10)
+        -> Filter: (<cache>(x1.b) = <ref_null_helper>(max(x2.b)))  (cost=1.68..1.68 rows=1)
+            -> Aggregate: max(x2.b)  (cost=1.58..1.58 rows=1)
+                -> Filter: (x2.a > (10 * rand(0)))  (cost=0.375..1.25 rows=3.33)
                     -> Table scan on x2  (cost=0.025..0.25 rows=10)
 
 EXPLAIN FORMAT=TREE SELECT SUM(x1.a) s FROM t1 x1

mysql-test/r/hash_join_hypergraph.result

@@ -684,7 +684,7 @@ WHERE
 t1.col1 = t2.col1
 AND t1.col1 = FLOOR(RAND() / 2 + 2);
 EXPLAIN
--> Filter: (t1.col1 = floor(((rand() / 2) + 2)))  (rows=0.4)
+-> Filter: (t1.col1 = floor(((rand() / 2) + 2)))  (rows=0.04)
     -> Inner hash join (t1.col1 = t2.col1)  (rows=0.4)
         -> Table scan on t1  (rows=2)
         -> Hash
@@ -1021,10 +1021,10 @@ WHERE (NOT EXISTS
 ON alias4.col_varchar = alias3.col_varchar_key)
 WHERE alias3.pk >= table1.pk));
 EXPLAIN
--> Hash antijoin (no condition), extra conditions: (alias3.pk >= table1.pk)  (rows=1.07)
-    -> Nested loop inner join  (rows=1.6)
+-> Hash antijoin (no condition), extra conditions: (alias3.pk >= table1.pk)  (rows=1.33)
+    -> Nested loop inner join  (rows=2)
         -> Table scan on table2  (rows=1)
-        -> Filter: ((table2.pk = table1.pk) or (table1.col_varchar < 'D'))  (rows=1.6)
+        -> Filter: ((table2.pk = table1.pk) or (table1.col_varchar < 'D'))  (rows=2)
             -> Table scan on table1  (rows=4)
     -> Hash
         -> Inner hash join (alias4.col_varchar = alias3.col_varchar_key)  (rows=1)
@@ -2202,10 +2202,10 @@ WHERE t2.pk = 9
 EXPLAIN
 -> Table scan on t1  (rows=2)
 -> Select #2 (subquery in projection; dependent)
-    -> Nested loop inner join  (rows=0.01)
-        -> Nested loop left join  (rows=0.1)
-            -> Single-row covering index lookup on t2 using PRIMARY (pk=9)  (rows=0.1)
-            -> Filter: ((t3.pk = 9) and (t2.pk = 9))  (rows=0.01)
+    -> Nested loop inner join  (rows=0.1)
+        -> Nested loop left join  (rows=1)
+            -> Single-row covering index lookup on t2 using PRIMARY (pk=9)  (rows=1)
+            -> Filter: ((t3.pk = 9) and (t2.pk = 9))  (rows=0.1)
                 -> Table scan on t3  (rows=1)
         -> Filter: (t4.pk = 9)  (rows=0.1)
             -> Table scan on t4  (rows=1)

mysql-test/r/histograms.result

@@ -3980,4 +3980,27 @@ Note	1003	/* select#1 */ select `test`.`ten_thousand`.`x` AS `x` from `test`.`te
 DROP TABLE ten;
 DROP TABLE hundred;
 DROP TABLE ten_thousand;
+#
+# Bug#34787357 Hypergraph: row estimates for
+# field=non_field_term ignores indexes and histogram.
+#
+CREATE TABLE t1 (col1 INT);
+INSERT INTO t1 VALUES (1),(1),(2),(2),(3),(3),(NULL),(NULL);
+ANALYZE TABLE t1 UPDATE HISTOGRAM ON col1;
+Table	Op	Msg_type	Msg_text
+test.t1	histogram	status	Histogram statistics created for column 'col1'.
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+EXPLAIN SELECT * FROM t1 WHERE col1 = FLOOR(RAND(0));
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	8	25.00	Using where
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t1`.`col1` AS `col1` from `test`.`t1` where (`test`.`t1`.`col1` = floor(rand(0)))
+EXPLAIN SELECT * FROM t1 WHERE col1 <> FLOOR(RAND(0));
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	8	50.00	Using where
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t1`.`col1` AS `col1` from `test`.`t1` where (`test`.`t1`.`col1` <> floor(rand(0)))
+DROP TABLE t1;
 # restart: