Port v5.1 Prefix Index Queries Optimization

Summary:
Feature: Prefix Index Queries Optimization

Optimize prefix index queries to skip cluster index lookup when possible.

Currently InnoDB will always fetch the clustered index (primary key
index) for all prefix columns in an index, even when the value of a
particular record is smaller than the prefix length.  This change
optimizes that case to use the record from the secondary index and avoid
the extra lookup.

Also adds two status vars that track how effective this is:

    innodb_secondary_index_triggered_cluster_reads:
	Times secondary index lookup triggered cluster lookup.

    innodb_secondary_index_triggered_cluster_reads_avoided:
	Times prefix optimization avoided triggering cluster lookup.

Test Plan:
Jenkins, all pass.

New test included, fails without this change, passes with it.

Random toggle added to stress tests, the small ones all pass,
and the nightly Jenkins stress tests should all pass too.

Reviewers: pivanof

Reviewed By: pivanof

CC: jtolmer, MarkCallaghan, flamingcow, jeremycole, andrew-ford, inaam-rana, liang.guo.752

Differential Revision: https://reviews.facebook.net/D15921
Differential Revision: https://reviews.facebook.net/D16455

mysql-test/r/fast_prefix_index_fetch_innodb.result

@@ -0,0 +1,102 @@
+drop table if exists prefixinno;
+set global innodb_prefix_index_cluster_optimization = ON;
+show variables like 'innodb_prefix_index_cluster_optimization';
+Variable_name	Value
+innodb_prefix_index_cluster_optimization	ON
+# Create a table with a large varchar field that we index the prefix
+# of and ensure we only trigger cluster lookups when we expect it.
+create table prefixinno (
+id int not null,
+fake_id int not null,
+bigfield varchar(4096),
+primary key(id),
+index bigfield_idx (bigfield(32)),
+index fake_id_bigfield_prefix (fake_id, bigfield(32))
+) engine=innodb;
+insert into prefixinno values (1, 1001, repeat('a', 1)),
+(8, 1008, repeat('b', 8)),
+(24, 1024, repeat('c', 24)),
+(31, 1031, repeat('d', 31)),
+(32, 1032, repeat('x', 32)),
+(33, 1033, repeat('y', 33)),
+(128, 1128, repeat('z', 128));
+select * from prefixinno;
+id	fake_id	bigfield
+1	1001	a
+8	1008	bbbbbbbb
+24	1024	cccccccccccccccccccccccc
+31	1031	ddddddddddddddddddddddddddddddd
+32	1032	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+33	1033	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+128	1128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+# Baseline sanity check: 0, 0.
+no-op query
+no-op query
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible for optimization.
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible for optimization, access via fake_id only.
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Not eligible for optimization, access via fake_id of big row.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Not eligible for optimization.
+id	bigfield
+32	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Not eligible for optimization.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible, should not increment lookup counter.
+id	bigfield
+8	bbbbbbbb
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible, should not increment lookup counter.
+id	bigfield
+24	cccccccccccccccccccccccc
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Should increment lookup counter.
+id	bigfield
+128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Disable optimization, confirm we still increment counter.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# make test suite happy by cleaning up our mess

mysql-test/suite/innodb_stress/include/innodb_stress.inc

@@ -4,6 +4,13 @@
 # Check the table against the replica.
 # Reinvoke the threads.
 
+
+--connection master
+--let $innodb_index_cluster_optimization_save_master = `SELECT @@innodb_prefix_index_cluster_optimization`
+
+--connection slave
+--let $innodb_index_cluster_optimization_save_slave = `SELECT @@innodb_prefix_index_cluster_optimization`
+
 # create the directory for temporary log files.
 --exec mkdir -p $MYSQL_TMP_DIR/load_generator
 
@@ -162,3 +169,15 @@ let $slave_sync_timeout = 7200;
 connection master;
 
 # --exec rm -rf $MYSQL_TMP_DIR/load_generator
+
+--connection slave
+# Restore the value of variable on slave
+--disable_query_log
+eval SET GLOBAL innodb_prefix_index_cluster_optimization = $innodb_index_cluster_optimization_save_slave;
+--enable_query_log
+
+--connection master
+# Restore the value of variable on master
+--disable_query_log
+eval SET GLOBAL innodb_prefix_index_cluster_optimization = $innodb_index_cluster_optimization_save_master;
+--enable_query_log

mysql-test/suite/innodb_stress/t/load_generator.py

@@ -277,9 +277,8 @@ def runme(self):
       self.loop_num += 1
 
       # Randomly toggle innodb_prefix_index_cluster_optimization 5% of the time
-      # nizamordulu: disabled for now until we port it to 5.6
-      #if self.rand.randint(0, 20) == 0:
-        #cur.execute("SET GLOBAL innodb_prefix_index_cluster_optimization=1-@@innodb_prefix_index_cluster_optimization")
+      if self.rand.randint(0, 20) == 0:
+        cur.execute("SET GLOBAL innodb_prefix_index_cluster_optimization=1-@@innodb_prefix_index_cluster_optimization")
 
       try:
         stmt = None

mysql-test/suite/sys_vars/r/innodb_prefix_index_cluster_optimization_basic.result

@@ -0,0 +1,122 @@
+SET @start_global_value = @@global.innodb_prefix_index_cluster_optimization;
+SELECT @start_global_value;
+@start_global_value
+0
+#
+# exists as global only
+#
+Valid values are 'ON' and 'OFF'
+select @@global.innodb_prefix_index_cluster_optimization in (0, 1);
+@@global.innodb_prefix_index_cluster_optimization in (0, 1)
+1
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0
+select @@session.innodb_prefix_index_cluster_optimization;
+ERROR HY000: Variable 'innodb_prefix_index_cluster_optimization' is a GLOBAL variable
+show global variables like 'innodb_prefix_index_cluster_optimization';
+Variable_name	Value
+innodb_prefix_index_cluster_optimization	OFF
+show session variables like 'innodb_prefix_index_cluster_optimization';
+Variable_name	Value
+innodb_prefix_index_cluster_optimization	OFF
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+#
+# show that it's writable
+#
+set global innodb_prefix_index_cluster_optimization = 'OFF';
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+set @@global.innodb_prefix_index_cluster_optimization = 'ON';
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set global innodb_prefix_index_cluster_optimization = 0;
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+set @@global.innodb_prefix_index_cluster_optimization = 1;
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set session innodb_prefix_index_cluster_optimization = 'OFF';
+ERROR HY000: Variable 'innodb_prefix_index_cluster_optimization' is a GLOBAL variable and should be set with SET GLOBAL
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set @@session.innodb_prefix_index_cluster_optimization = 'ON';
+ERROR HY000: Variable 'innodb_prefix_index_cluster_optimization' is a GLOBAL variable and should be set with SET GLOBAL
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+#
+# incorrect types
+#
+set global innodb_prefix_index_cluster_optimization = 1.1;
+ERROR 42000: Incorrect argument type to variable 'innodb_prefix_index_cluster_optimization'
+set global innodb_prefix_index_cluster_optimization = 1e1;
+ERROR 42000: Incorrect argument type to variable 'innodb_prefix_index_cluster_optimization'
+set global innodb_prefix_index_cluster_optimization = 2;
+ERROR 42000: Variable 'innodb_prefix_index_cluster_optimization' can't be set to the value of '2'
+NOTE: The following should fail with ER_WRONG_VALUE_FOR_VAR (BUG#50643)
+set global innodb_prefix_index_cluster_optimization = -3;
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set global innodb_prefix_index_cluster_optimization = 'AUTO';
+ERROR 42000: Variable 'innodb_prefix_index_cluster_optimization' can't be set to the value of 'AUTO'
+#
+# Cleanup
+#
+SET @@global.innodb_prefix_index_cluster_optimization = @start_global_value;
+SELECT @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0

mysql-test/suite/sys_vars/t/innodb_prefix_index_cluster_optimization_basic.test

@@ -0,0 +1,76 @@
+--source include/have_innodb.inc
+
+SET @start_global_value = @@global.innodb_prefix_index_cluster_optimization;
+SELECT @start_global_value;
+
+--echo #
+--echo # exists as global only
+--echo #
+
+--echo Valid values are 'ON' and 'OFF'
+select @@global.innodb_prefix_index_cluster_optimization in (0, 1);
+select @@global.innodb_prefix_index_cluster_optimization;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.innodb_prefix_index_cluster_optimization;
+show global variables like 'innodb_prefix_index_cluster_optimization';
+show session variables like 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--echo #
+--echo # show that it's writable
+--echo #
+
+set global innodb_prefix_index_cluster_optimization = 'OFF';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+set @@global.innodb_prefix_index_cluster_optimization = 'ON';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+set global innodb_prefix_index_cluster_optimization = 0;
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+set @@global.innodb_prefix_index_cluster_optimization = 1;
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--error ER_GLOBAL_VARIABLE
+set session innodb_prefix_index_cluster_optimization = 'OFF';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--error ER_GLOBAL_VARIABLE
+set @@session.innodb_prefix_index_cluster_optimization = 'ON';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--echo #
+--echo # incorrect types
+--echo #
+
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 1.1;
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 1e1;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 2;
+--echo NOTE: The following should fail with ER_WRONG_VALUE_FOR_VAR (BUG#50643)
+set global innodb_prefix_index_cluster_optimization = -3;
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 'AUTO';
+
+--echo #
+--echo # Cleanup
+--echo #
+
+SET @@global.innodb_prefix_index_cluster_optimization = @start_global_value;
+SELECT @@global.innodb_prefix_index_cluster_optimization;