MDEV-6929: Port Facebook Prefix Index Queries Optimization

Merge Facebook commit 154c579b828a60722a7d9477fc61868c07453d08
and e8f0052f9b112dc786bf9b957ed5b16a5749f7fd authored
by Steaphan Greene from https://github.com/facebook/mysql-5.6

Optimize prefix index queries to skip cluster index lookup when possible.

Currently InnoDB will always fetch the clustered index (primary key
index) for all prefix columns in an index, even when the value of a
particular record is smaller than the prefix length. This change
optimizes that case to use the record from the secondary index and avoid
the extra lookup.

Also adds two status vars that track how effective this is:

innodb_secondary_index_triggered_cluster_reads:
Times secondary index lookup triggered cluster lookup.

innodb_secondary_index_triggered_cluster_reads_avoided:
Times prefix optimization avoided triggering cluster lookup.

mysql-test/r/fast_prefix_index_fetch_innodb.result

@@ -0,0 +1,102 @@
+drop table if exists prefixinno;
+set global innodb_prefix_index_cluster_optimization = ON;
+show variables like 'innodb_prefix_index_cluster_optimization';
+Variable_name	Value
+innodb_prefix_index_cluster_optimization	ON
+# Create a table with a large varchar field that we index the prefix
+# of and ensure we only trigger cluster lookups when we expect it.
+create table prefixinno (
+id int not null,
+fake_id int not null,
+bigfield varchar(4096),
+primary key(id),
+index bigfield_idx (bigfield(32)),
+index fake_id_bigfield_prefix (fake_id, bigfield(32))
+) engine=innodb;
+insert into prefixinno values (1, 1001, repeat('a', 1)),
+(8, 1008, repeat('b', 8)),
+(24, 1024, repeat('c', 24)),
+(31, 1031, repeat('d', 31)),
+(32, 1032, repeat('x', 32)),
+(33, 1033, repeat('y', 33)),
+(128, 1128, repeat('z', 128));
+select * from prefixinno;
+id	fake_id	bigfield
+1	1001	a
+8	1008	bbbbbbbb
+24	1024	cccccccccccccccccccccccc
+31	1031	ddddddddddddddddddddddddddddddd
+32	1032	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+33	1033	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+128	1128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+# Baseline sanity check: 0, 0.
+no-op query
+no-op query
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible for optimization.
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible for optimization, access via fake_id only.
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Not eligible for optimization, access via fake_id of big row.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Not eligible for optimization.
+id	bigfield
+32	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Not eligible for optimization.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible, should not increment lookup counter.
+id	bigfield
+8	bbbbbbbb
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Eligible, should not increment lookup counter.
+id	bigfield
+24	cccccccccccccccccccccccc
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Should increment lookup counter.
+id	bigfield
+128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# Disable optimization, confirm we still increment counter.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+cluster_lookups_matched
+1
+cluster_lookups_avoided_matched
+1
+# make test suite happy by cleaning up our mess

mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result

@@ -40,6 +40,8 @@ buffer_pages_written	disabled
 buffer_index_pages_written	disabled
 buffer_non_index_pages_written	disabled
 buffer_pages_read	disabled
+buffer_index_sec_rec_cluster_reads	disabled
+buffer_index_sec_rec_cluster_reads_avoided	disabled
 buffer_data_reads	disabled
 buffer_data_written	disabled
 buffer_flush_batch_scanned	disabled

mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result

@@ -40,6 +40,8 @@ buffer_pages_written	disabled
 buffer_index_pages_written	disabled
 buffer_non_index_pages_written	disabled
 buffer_pages_read	disabled
+buffer_index_sec_rec_cluster_reads	disabled
+buffer_index_sec_rec_cluster_reads_avoided	disabled
 buffer_data_reads	disabled
 buffer_data_written	disabled
 buffer_flush_batch_scanned	disabled

mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result

@@ -40,6 +40,8 @@ buffer_pages_written	disabled
 buffer_index_pages_written	disabled
 buffer_non_index_pages_written	disabled
 buffer_pages_read	disabled
+buffer_index_sec_rec_cluster_reads	disabled
+buffer_index_sec_rec_cluster_reads_avoided	disabled
 buffer_data_reads	disabled
 buffer_data_written	disabled
 buffer_flush_batch_scanned	disabled

mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result

@@ -40,6 +40,8 @@ buffer_pages_written	disabled
 buffer_index_pages_written	disabled
 buffer_non_index_pages_written	disabled
 buffer_pages_read	disabled
+buffer_index_sec_rec_cluster_reads	disabled
+buffer_index_sec_rec_cluster_reads_avoided	disabled
 buffer_data_reads	disabled
 buffer_data_written	disabled
 buffer_flush_batch_scanned	disabled

mysql-test/suite/sys_vars/r/innodb_prefix_index_cluster_optimization_basic.result

@@ -0,0 +1,122 @@
+SET @start_global_value = @@global.innodb_prefix_index_cluster_optimization;
+SELECT @start_global_value;
+@start_global_value
+0
+#
+# exists as global only
+#
+Valid values are 'ON' and 'OFF'
+select @@global.innodb_prefix_index_cluster_optimization in (0, 1);
+@@global.innodb_prefix_index_cluster_optimization in (0, 1)
+1
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0
+select @@session.innodb_prefix_index_cluster_optimization;
+ERROR HY000: Variable 'innodb_prefix_index_cluster_optimization' is a GLOBAL variable
+show global variables like 'innodb_prefix_index_cluster_optimization';
+Variable_name	Value
+innodb_prefix_index_cluster_optimization	OFF
+show session variables like 'innodb_prefix_index_cluster_optimization';
+Variable_name	Value
+innodb_prefix_index_cluster_optimization	OFF
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+#
+# show that it's writable
+#
+set global innodb_prefix_index_cluster_optimization = 'OFF';
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+set @@global.innodb_prefix_index_cluster_optimization = 'ON';
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set global innodb_prefix_index_cluster_optimization = 0;
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	OFF
+set @@global.innodb_prefix_index_cluster_optimization = 1;
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set session innodb_prefix_index_cluster_optimization = 'OFF';
+ERROR HY000: Variable 'innodb_prefix_index_cluster_optimization' is a GLOBAL variable and should be set with SET GLOBAL
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set @@session.innodb_prefix_index_cluster_optimization = 'ON';
+ERROR HY000: Variable 'innodb_prefix_index_cluster_optimization' is a GLOBAL variable and should be set with SET GLOBAL
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+#
+# incorrect types
+#
+set global innodb_prefix_index_cluster_optimization = 1.1;
+ERROR 42000: Incorrect argument type to variable 'innodb_prefix_index_cluster_optimization'
+set global innodb_prefix_index_cluster_optimization = 1e1;
+ERROR 42000: Incorrect argument type to variable 'innodb_prefix_index_cluster_optimization'
+set global innodb_prefix_index_cluster_optimization = 2;
+ERROR 42000: Variable 'innodb_prefix_index_cluster_optimization' can't be set to the value of '2'
+set global innodb_prefix_index_cluster_optimization = -3;
+ERROR 42000: Variable 'innodb_prefix_index_cluster_optimization' can't be set to the value of '-3'
+select @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+1
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION	ON
+set global innodb_prefix_index_cluster_optimization = 'AUTO';
+ERROR 42000: Variable 'innodb_prefix_index_cluster_optimization' can't be set to the value of 'AUTO'
+#
+# Cleanup
+#
+SET @@global.innodb_prefix_index_cluster_optimization = @start_global_value;
+SELECT @@global.innodb_prefix_index_cluster_optimization;
+@@global.innodb_prefix_index_cluster_optimization
+0

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -1461,6 +1461,20 @@ NUMERIC_BLOCK_SIZE	0
 ENUM_VALUE_LIST	NULL
 READ_ONLY	YES
 COMMAND_LINE_ARGUMENT	OPTIONAL
+VARIABLE_NAME	INNODB_PREFIX_INDEX_CLUSTER_OPTIMIZATION
+SESSION_VALUE	NULL
+GLOBAL_VALUE	OFF
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	OFF
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Enable prefix optimization to sometimes avoid cluster index lookups.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	INNODB_PRINT_ALL_DEADLOCKS
 SESSION_VALUE	NULL
 GLOBAL_VALUE	OFF

mysql-test/suite/sys_vars/t/innodb_prefix_index_cluster_optimization_basic.test

@@ -0,0 +1,76 @@
+--source include/have_innodb.inc
+
+SET @start_global_value = @@global.innodb_prefix_index_cluster_optimization;
+SELECT @start_global_value;
+
+--echo #
+--echo # exists as global only
+--echo #
+
+--echo Valid values are 'ON' and 'OFF'
+select @@global.innodb_prefix_index_cluster_optimization in (0, 1);
+select @@global.innodb_prefix_index_cluster_optimization;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.innodb_prefix_index_cluster_optimization;
+show global variables like 'innodb_prefix_index_cluster_optimization';
+show session variables like 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--echo #
+--echo # show that it's writable
+--echo #
+
+set global innodb_prefix_index_cluster_optimization = 'OFF';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+set @@global.innodb_prefix_index_cluster_optimization = 'ON';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+set global innodb_prefix_index_cluster_optimization = 0;
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+set @@global.innodb_prefix_index_cluster_optimization = 1;
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--error ER_GLOBAL_VARIABLE
+set session innodb_prefix_index_cluster_optimization = 'OFF';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--error ER_GLOBAL_VARIABLE
+set @@session.innodb_prefix_index_cluster_optimization = 'ON';
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+
+--echo #
+--echo # incorrect types
+--echo #
+
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 1.1;
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 1e1;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 2;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = -3;
+select @@global.innodb_prefix_index_cluster_optimization;
+select * from information_schema.global_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+select * from information_schema.session_variables where variable_name = 'innodb_prefix_index_cluster_optimization';
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_prefix_index_cluster_optimization = 'AUTO';
+
+--echo #
+--echo # Cleanup
+--echo #
+
+SET @@global.innodb_prefix_index_cluster_optimization = @start_global_value;
+SELECT @@global.innodb_prefix_index_cluster_optimization;