MDEV-31864 Assertion `d >= 0' failed in COST_ADD with join_cache_leve…

…l > 2 and partitions

The problem was a partitioned table had 0 rows, which caused a division by
zero at hash_join_fanout()
Fix: Do not call hash_join_fanout with 0 rows.

mysql-test/main/selectivity.result

@@ -2084,3 +2084,32 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
 1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	2	
 drop table t1,t2;
+#
+# MDEV-31864 Assertion `d >= 0' failed in COST_ADD with join_cache_level
+# > 2 and partitions
+#
+SET join_cache_level= 3;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT, c INT);
+INSERT INTO t2 VALUES (3,4),(5,6);
+CREATE TABLE t3 (d INT, e INT) PARTITION BY RANGE COLUMNS (d) (p1 VALUES LESS THAN (1000), pn VALUES LESS THAN (MAXVALUE));
+ANALYZE TABLE t1, t2, t3 PERSISTENT FOR ALL;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+test.t2	analyze	status	Engine-independent statistics collected
+test.t2	analyze	status	OK
+test.t3	analyze	status	Engine-independent statistics collected
+test.t3	analyze	status	OK
+explain SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON t2.b = t3.e) ON t2.c = 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	0	Using where
+SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON t2.b = t3.e) ON t2.c = 1;
+a	b	c	d	e
+1	NULL	NULL	NULL	NULL
+2	NULL	NULL	NULL	NULL
+set join_cache_level= default;
+DROP TABLE t1, t2, t3;

mysql-test/main/selectivity.test

@@ -2,6 +2,7 @@
 --source include/have_stat_tables.inc
 --source include/have_sequence.inc
 --source include/default_charset.inc
+--source include/have_partition.inc
 
 --disable_warnings
 drop table if exists t0,t1,t2,t3;
@@ -1429,3 +1430,23 @@ truncate table t1;
 insert into t1 select * from t2;
 explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
 drop table t1,t2;
+
+--echo #
+--echo # MDEV-31864 Assertion `d >= 0' failed in COST_ADD with join_cache_level
+--echo # > 2 and partitions
+--echo #
+
+SET join_cache_level= 3;
+
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT, c INT);
+INSERT INTO t2 VALUES (3,4),(5,6);
+CREATE TABLE t3 (d INT, e INT) PARTITION BY RANGE COLUMNS (d) (p1 VALUES LESS THAN (1000), pn VALUES LESS THAN (MAXVALUE));
+ANALYZE TABLE t1, t2, t3 PERSISTENT FOR ALL;
+
+explain SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON t2.b = t3.e) ON t2.c = 1;
+SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON t2.b = t3.e) ON t2.c = 1;
+
+set join_cache_level= default;
+DROP TABLE t1, t2, t3;

mysql-test/main/selectivity_innodb.result

@@ -2091,6 +2091,35 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
 1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	2	
 drop table t1,t2;
+#
+# MDEV-31864 Assertion `d >= 0' failed in COST_ADD with join_cache_level
+# > 2 and partitions
+#
+SET join_cache_level= 3;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (b INT, c INT);
+INSERT INTO t2 VALUES (3,4),(5,6);
+CREATE TABLE t3 (d INT, e INT) PARTITION BY RANGE COLUMNS (d) (p1 VALUES LESS THAN (1000), pn VALUES LESS THAN (MAXVALUE));
+ANALYZE TABLE t1, t2, t3 PERSISTENT FOR ALL;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+test.t2	analyze	status	Engine-independent statistics collected
+test.t2	analyze	status	OK
+test.t3	analyze	status	Engine-independent statistics collected
+test.t3	analyze	status	OK
+explain SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON t2.b = t3.e) ON t2.c = 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	2	Using where
+SELECT * FROM t1 LEFT JOIN (t2 JOIN t3 ON t2.b = t3.e) ON t2.c = 1;
+a	b	c	d	e
+1	NULL	NULL	NULL	NULL
+2	NULL	NULL	NULL	NULL
+set join_cache_level= default;
+DROP TABLE t1, t2, t3;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 set @tmp_ust= @@use_stat_tables;
 set @tmp_oucs= @@optimizer_use_condition_selectivity;

sql/sql_select.cc

@@ -8145,6 +8145,7 @@ double hash_join_fanout(JOIN *join, JOIN_TAB *tab, table_map remaining_tables,
   bool found_not_usable_field= 0;
   bool found_usable_field __attribute__((unused))= 0;
   DBUG_ENTER("hash_join_cardinality");
+  DBUG_ASSERT(rnd_records > 0 && min_freq > 0);
 
   Json_writer_object trace_obj(thd, "hash_join_cardinality");
 
@@ -9128,7 +9129,8 @@ best_access_path(JOIN      *join,
     DBUG_ASSERT(hj_start_key);
 
     fanout= rnd_records;
-    if (optimizer_flag(thd, OPTIMIZER_SWITCH_HASH_JOIN_CARDINALITY))
+    if (optimizer_flag(thd, OPTIMIZER_SWITCH_HASH_JOIN_CARDINALITY) &&
+        rnd_records > 0)
     {
       /*
         Starting from this point, rnd_records should not be used anymore.