From 65da5645305ee3b5d49c571314b832059bace09c Mon Sep 17 00:00:00 2001
From: Monty <monty@mariadb.org>
Date: Mon, 9 Jan 2023 17:48:06 +0200
Subject: [PATCH] MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed
 in ...

The problem was that make_join_select() called test_quick_select() outside
of best_access_path(). This could use indexes that where not taken into
account before and this caused changes to selectivity and 'records_out'.

Fixed by updating records_out if test_quick_select() was called.
---
 mysql-test/main/selectivity.result        | 13 +++++++++++++
 mysql-test/main/selectivity.test          | 17 +++++++++++++++++
 mysql-test/main/selectivity_innodb.result | 13 +++++++++++++
 sql/sql_select.cc                         | 11 +++++++++++
 4 files changed, 54 insertions(+)

diff --git a/mysql-test/main/selectivity.result b/mysql-test/main/selectivity.result
index ef21df2cfcd99..e8ec0c30e807b 100644
--- a/mysql-test/main/selectivity.result
+++ b/mysql-test/main/selectivity.result
@@ -1998,3 +1998,16 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	B	ref	a	a	5	const	1	
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1,t2;
+#
+# MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in get_range_limit_read_cost
+# with LIMIT .. OFFSET
+#
+CREATE TABLE t1 (a INT, b VARCHAR(1), KEY(b), KEY(a)) engine=myisam;
+INSERT INTO t1 VALUES
+(3,'a'),(2,'g'),(5,'v'),(9,'n'),(6,'u'),
+(7,'s'),(0,'z'),(3,'z'),(NULL,'m'),(6,'r');
+CREATE TABLE t2 (pk INT PRIMARY KEY);
+INSERT INTO t2 VALUES (1),(2);
+SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
+pk
+DROP TABLE t1, t2;
diff --git a/mysql-test/main/selectivity.test b/mysql-test/main/selectivity.test
index a4fa09f01162c..d775f8e43702a 100644
--- a/mysql-test/main/selectivity.test
+++ b/mysql-test/main/selectivity.test
@@ -1359,3 +1359,20 @@ set optimizer_use_condition_selectivity=2;
 EXPLAIN  SELECT * FROM t1 A, t1 B WHERE A.a = B.a and A.id = 65;
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1,t2;
+
+--echo #
+--echo # MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in get_range_limit_read_cost
+--echo # with LIMIT .. OFFSET
+--echo #
+
+CREATE TABLE t1 (a INT, b VARCHAR(1), KEY(b), KEY(a)) engine=myisam;
+INSERT INTO t1 VALUES
+(3,'a'),(2,'g'),(5,'v'),(9,'n'),(6,'u'),
+(7,'s'),(0,'z'),(3,'z'),(NULL,'m'),(6,'r');
+
+CREATE TABLE t2 (pk INT PRIMARY KEY);
+INSERT INTO t2 VALUES (1),(2);
+
+SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
+
+DROP TABLE t1, t2;
diff --git a/mysql-test/main/selectivity_innodb.result b/mysql-test/main/selectivity_innodb.result
index 669d629fc0224..274ba62329d05 100644
--- a/mysql-test/main/selectivity_innodb.result
+++ b/mysql-test/main/selectivity_innodb.result
@@ -2005,6 +2005,19 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	B	ref	a	a	5	const	1	Using index
 set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
 drop table t1,t2;
+#
+# MDEV-30360 Assertion `cond_selectivity <= 1.000000001' failed in get_range_limit_read_cost
+# with LIMIT .. OFFSET
+#
+CREATE TABLE t1 (a INT, b VARCHAR(1), KEY(b), KEY(a)) engine=myisam;
+INSERT INTO t1 VALUES
+(3,'a'),(2,'g'),(5,'v'),(9,'n'),(6,'u'),
+(7,'s'),(0,'z'),(3,'z'),(NULL,'m'),(6,'r');
+CREATE TABLE t2 (pk INT PRIMARY KEY);
+INSERT INTO t2 VALUES (1),(2);
+SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
+pk
+DROP TABLE t1, t2;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 set @tmp_ust= @@use_stat_tables;
 set @tmp_oucs= @@optimizer_use_condition_selectivity;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index bf3935db613bf..bcbff6707bc69 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -13434,6 +13434,17 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
               set_if_smaller(join->best_positions[i].records_out,
                              join->best_positions[i].records_read);
             }
+            else
+            {
+              /*
+                sel->head->opt_range_condition_rows may have been updated to a smaller number than
+                before by a call to test_quick_select. This can happen even if the range optimizer
+                decided to not use the range (sel->quick was not set).
+              */
+              set_if_smaller(join->best_positions[i].records_out,
+                             rows2double(sel->head->opt_range_condition_rows));
+
+            }
 	  }
 	  else
 	  {