MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in ...

The problem was that make_join_select() called test_quick_select() outside
of best_access_path(). This could use indexes that where not taken into
account before and this caused changes to selectivity and 'records_out'.

Fixed by updating records_out if test_quick_select() was called.

mysql-test/main/selectivity.result

@@ -1998,3 +1998,16 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	B	ref	a	a	5	const	1	
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1,t2;
+#
+# MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in get_range_limit_read_cost
+# with LIMIT .. OFFSET
+#
+CREATE TABLE t1 (a INT, b VARCHAR(1), KEY(b), KEY(a)) engine=myisam;
+INSERT INTO t1 VALUES
+(3,'a'),(2,'g'),(5,'v'),(9,'n'),(6,'u'),
+(7,'s'),(0,'z'),(3,'z'),(NULL,'m'),(6,'r');
+CREATE TABLE t2 (pk INT PRIMARY KEY);
+INSERT INTO t2 VALUES (1),(2);
+SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
+pk
+DROP TABLE t1, t2;

mysql-test/main/selectivity.test

@@ -1359,3 +1359,20 @@ set optimizer_use_condition_selectivity=2;
 EXPLAIN  SELECT * FROM t1 A, t1 B WHERE A.a = B.a and A.id = 65;
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1,t2;
+
+--echo #
+--echo # MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in get_range_limit_read_cost
+--echo # with LIMIT .. OFFSET
+--echo #
+
+CREATE TABLE t1 (a INT, b VARCHAR(1), KEY(b), KEY(a)) engine=myisam;
+INSERT INTO t1 VALUES
+(3,'a'),(2,'g'),(5,'v'),(9,'n'),(6,'u'),
+(7,'s'),(0,'z'),(3,'z'),(NULL,'m'),(6,'r');
+
+CREATE TABLE t2 (pk INT PRIMARY KEY);
+INSERT INTO t2 VALUES (1),(2);
+
+SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
+
+DROP TABLE t1, t2;

mysql-test/main/selectivity_innodb.result

@@ -2005,6 +2005,19 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	B	ref	a	a	5	const	1	Using index
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1,t2;
+#
+# MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in get_range_limit_read_cost
+# with LIMIT .. OFFSET
+#
+CREATE TABLE t1 (a INT, b VARCHAR(1), KEY(b), KEY(a)) engine=myisam;
+INSERT INTO t1 VALUES
+(3,'a'),(2,'g'),(5,'v'),(9,'n'),(6,'u'),
+(7,'s'),(0,'z'),(3,'z'),(NULL,'m'),(6,'r');
+CREATE TABLE t2 (pk INT PRIMARY KEY);
+INSERT INTO t2 VALUES (1),(2);
+SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
+pk
+DROP TABLE t1, t2;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 set @tmp_ust= @@use_stat_tables;
 set @tmp_oucs= @@optimizer_use_condition_selectivity;

sql/sql_select.cc

@@ -13434,6 +13434,17 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
               set_if_smaller(join->best_positions[i].records_out,
                              join->best_positions[i].records_read);
             }
+            else
+            {
+              /*
+                sel->head->opt_range_condition_rows may have been updated to a smaller number than
+                before by a call to test_quick_select. This can happen even if the range optimizer
+                decided to not use the range (sel->quick was not set).
+              */
+              set_if_smaller(join->best_positions[i].records_out,
+                             rows2double(sel->head->opt_range_condition_rows));
+
+            }
 	  }
 	  else
 	  {