MDEV-19834 Selectivity of an equality condition discounted twice

When discounting selectivity of ref access, don't discount the
selectivity we've already discounted for range access.

The 10.1 version of the fix. Will need to adjust condition filtering
test results in 10.4

Author: spetrunia

mysql-test/r/selectivity.result

@@ -1635,3 +1635,37 @@ set @@use_stat_tables= @save_use_stat_tables;
 set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
 drop table t1;
 drop function f1;
+#
+# MDEV-19834 Selectivity of an equality condition discounted twice
+#
+set @@optimizer_use_condition_selectivity=4;
+set @@use_stat_tables='preferably';
+create table t1 (a int, b int, key (b), key (a));
+insert into t1
+select (rand(1)*1000)/10, (rand(1001)*1000)/50 from seq_1_to_1000;
+analyze table t1 ;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	Table is already up to date
+# Check what info the optimizer has about selectivities
+explain extended select * from t1 use index () where a in (17,51,5);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	1000	2.97	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` USE INDEX () where (`test`.`t1`.`a` in (17,51,5))
+explain extended select * from t1 use index () where b=2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	1000	4.76	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` USE INDEX () where (`test`.`t1`.`b` = 2)
+# Now, the equality is used for ref access, while the range condition
+# gives selectivity data
+explain extended select * from t1 where a in (17,51,5) and b=2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ref	b,a	b	5	const	58	2.90	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where ((`test`.`t1`.`b` = 2) and (`test`.`t1`.`a` in (17,51,5)))
+drop table t1;
+set use_stat_tables= @save_use_stat_tables;
+set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
+# End of 10.1 tests

mysql-test/t/selectivity.test

@@ -1,4 +1,5 @@
 --source include/have_stat_tables.inc
+--source include/have_sequence.inc
 
 --disable_warnings
 drop table if exists t0,t1,t2,t3;
@@ -1102,3 +1103,26 @@ set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectiv
 drop table t1;
 drop function f1;
 
+--echo #
+--echo # MDEV-19834 Selectivity of an equality condition discounted twice
+--echo #
+set @@optimizer_use_condition_selectivity=4;
+set @@use_stat_tables='preferably';
+create table t1 (a int, b int, key (b), key (a));
+insert into t1
+select (rand(1)*1000)/10, (rand(1001)*1000)/50 from seq_1_to_1000;
+analyze table t1 ;
+
+--echo # Check what info the optimizer has about selectivities
+explain extended select * from t1 use index () where a in (17,51,5);
+explain extended select * from t1 use index () where b=2;
+
+--echo # Now, the equality is used for ref access, while the range condition
+--echo # gives selectivity data
+explain extended select * from t1 where a in (17,51,5) and b=2;
+drop table t1;
+
+set use_stat_tables= @save_use_stat_tables;
+set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
+--echo # End of 10.1 tests
+

sql/sql_select.cc

@@ -7618,6 +7618,7 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
     KEYUSE *keyuse= pos->key;
     KEYUSE *prev_ref_keyuse= keyuse;
     uint key= keyuse->key;
+    bool used_range_selectivity= false;
 
     /*
       Check if we have a prefix of key=const that matches a quick select.
@@ -7643,6 +7644,7 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
           keyparts++;
         }
         sel /= (double)table->quick_rows[key] / (double) table->stat_records();
+        used_range_selectivity= true;
       }
     }
 
@@ -7678,13 +7680,14 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
           if (keyparts > keyuse->keypart)
 	  {
             /* Ok this is the keyuse that will be used for ref access */
-            uint fldno;
-            if (is_hash_join_key_no(key))
-	      fldno= keyuse->keypart;
-            else
-              fldno= table->key_info[key].key_part[keyparts-1].fieldnr - 1;
-            if (keyuse->val->const_item())
+            if (!used_range_selectivity && keyuse->val->const_item())
             { 
+              uint fldno;
+              if (is_hash_join_key_no(key))
+                fldno= keyuse->keypart;
+              else
+                fldno= table->key_info[key].key_part[keyparts-1].fieldnr - 1;
+
               if (table->field[fldno]->cond_selectivity > 0)
 	      {            
                 sel /= table->field[fldno]->cond_selectivity;