Support covering secondary key lookups for varchars

Summary:
Addresses #303. Changes:
- Adds a status var called `ROCKSDB_COVERED_SECONDARY_KEY_LOOKUPS` which is incremented when we avoid reading a record via a keyread.
- Modifies `Rdb_key_def::pack_record` to store, for secondary indexes with varchar columns, a 2-byte bitmap in the unpack info after the header. This is the "covered bitmap", where each bit corresponds to a keypart. A bit is set if that key part contains the entire column value. Bits are only used currently for varchar columns, but could later be extended to blobs/text.
- Modifies `unpack_record` to reflect `pack_record` changes.
- Adds a new data tag for unpack_info called `RDB_UNPACK_COVERED_DATA_TAG`. This is used in place of `RDB_UNPACK_DATA_TAG` to indicate a covered bitmap is present in unpack_info.
- Backwards-compatible with older formats.
- Adds `Rdb_key_def::covers_lookup()`, which returns true if all of the columns requested by a query are covered by the secondary key. This is used in `read_row_from_secondary_key` and `secondary_index_read`.

Note: This doesn't support blobs/text yet, like the example in #303, since we currently skip unpacking blobs in unpack_record.

This feature will exist but be disabled in prod until further testing.
Closes #639

Differential Revision: D5286989

Pulled By: pbjc

fbshipit-source-id: f186cc8

mysql-test/suite/rocksdb/include/prefix_index_only_query_check.inc

@@ -0,0 +1,21 @@
+#
+# A helper include file for prefix index index-only query tests
+#
+# Parameters:
+# $prefix_index_check_title - title of the test
+# $prefix_index_check_query - test query
+# $prefix_index_check_read_avoided_delta - expected change of
+#   'rocksdb_covered_secondary_key_lookups' status variable
+#   value after running the query
+
+--let $show_count_statement = show status like 'rocksdb_covered_secondary_key_lookups'
+
+--echo # $prefix_index_check_title
+--let $base_count = query_get_value($show_count_statement, Value, 1)
+
+--eval $prefix_index_check_query
+
+--let $count = query_get_value($show_count_statement, Value, 1)
+--let $assert_text= $prefix_index_check_title: $prefix_index_check_read_avoided_delta rocksdb_covered_secondary_key_lookups
+--let $assert_cond= $count - $base_count = $prefix_index_check_read_avoided_delta
+--source include/assert.inc

mysql-test/suite/rocksdb/r/add_index_inplace.result

@@ -327,7 +327,7 @@ SET @prior_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check;
 SET @prior_rocksdb_merge_buf_size = @@rocksdb_merge_buf_size;
 SET global rocksdb_strict_collation_check = off;
 SET session rocksdb_merge_combine_read_size = 566;
-SET session rocksdb_merge_buf_size = 336;
+SET session rocksdb_merge_buf_size = 340;
 show variables like '%rocksdb_bulk_load%';
 Variable_name	Value
 rocksdb_bulk_load	OFF

mysql-test/suite/rocksdb/r/covered_unpack_info_format.result

@@ -0,0 +1,73 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (
+id INT,
+fake_id INT,
+bigfield VARCHAR(4096),
+PRIMARY KEY (id),
+KEY bf (bigfield(32)),
+KEY fid (fake_id, bigfield(32))
+) ENGINE=rocksdb;
+INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)),
+(8, 1008, REPEAT('b', 8)),
+(24, 1024, REPEAT('c', 24)),
+(31, 1031, REPEAT('d', 31)),
+(32, 1032, REPEAT('x', 32)),
+(33, 1033, REPEAT('y', 33)),
+(128, 1128, REPEAT('z', 128));
+SELECT * FROM t1;
+id	fake_id	bigfield
+1	1001	a
+8	1008	bbbbbbbb
+24	1024	cccccccccccccccccccccccc
+31	1031	ddddddddddddddddddddddddddddddd
+32	1032	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+33	1033	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+128	1128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+# Eligible for optimization, access via fake_id only
+SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1031;
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+include/assert.inc [Eligible for optimization, access via fake_id only: 2 rocksdb_covered_secondary_key_lookups]
+# Not eligible for optimization, access via fake_id of big row.
+SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1033;
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+include/assert.inc [Not eligible for optimization, access via fake_id of big row.: 0 rocksdb_covered_secondary_key_lookups]
+DROP TABLE t1;
+set session debug= '+d,MYROCKS_NO_COVERED_BITMAP_FORMAT';
+CREATE TABLE t1 (
+id INT,
+fake_id INT,
+bigfield VARCHAR(4096),
+PRIMARY KEY (id),
+KEY bf (bigfield(32)),
+KEY fid (fake_id, bigfield(32))
+) ENGINE=rocksdb;
+set session debug= '-d,MYROCKS_NO_COVERED_BITMAP_FORMAT';
+INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)),
+(8, 1008, REPEAT('b', 8)),
+(24, 1024, REPEAT('c', 24)),
+(31, 1031, REPEAT('d', 31)),
+(32, 1032, REPEAT('x', 32)),
+(33, 1033, REPEAT('y', 33)),
+(128, 1128, REPEAT('z', 128));
+SELECT * FROM t1;
+id	fake_id	bigfield
+1	1001	a
+8	1008	bbbbbbbb
+24	1024	cccccccccccccccccccccccc
+31	1031	ddddddddddddddddddddddddddddddd
+32	1032	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+33	1033	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+128	1128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+# No longer eligible for optimization since no covered bitmap was stored.
+SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1031;
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+include/assert.inc [No longer eligible for optimization since no covered bitmap was stored.: 0 rocksdb_covered_secondary_key_lookups]
+# Not eligible for optimization.
+SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1033;
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+include/assert.inc [Not eligible for optimization.: 0 rocksdb_covered_secondary_key_lookups]
+DROP TABLE t1;

mysql-test/suite/rocksdb/r/fast_prefix_index_fetch.result

@@ -0,0 +1,80 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (
+id INT,
+fake_id INT,
+bigfield VARCHAR(4096),
+PRIMARY KEY (id),
+KEY bf (bigfield(32)),
+KEY fid (fake_id, bigfield(32))
+) ENGINE=rocksdb;
+INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)),
+(8, 1008, REPEAT('b', 8)),
+(24, 1024, REPEAT('c', 24)),
+(31, 1031, REPEAT('d', 31)),
+(32, 1032, REPEAT('x', 32)),
+(33, 1033, REPEAT('y', 33)),
+(128, 1128, REPEAT('z', 128));
+SELECT * FROM t1;
+id	fake_id	bigfield
+1	1001	a
+8	1008	bbbbbbbb
+24	1024	cccccccccccccccccccccccc
+31	1031	ddddddddddddddddddddddddddddddd
+32	1032	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+33	1033	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+128	1128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+# Baseline sanity check
+no-op query
+no-op query
+include/assert.inc [Baseline sanity check: 0 rocksdb_covered_secondary_key_lookups]
+# Eligible for optimization.
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+include/assert.inc [Eligible for optimization.: 2 rocksdb_covered_secondary_key_lookups]
+# Eligible for optimization, access via fake_id only
+id	bigfield
+31	ddddddddddddddddddddddddddddddd
+include/assert.inc [Eligible for optimization, access via fake_id only: 2 rocksdb_covered_secondary_key_lookups]
+# Not eligible for optimization, access via fake_id of big row.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+include/assert.inc [Not eligible for optimization, access via fake_id of big row.: 0 rocksdb_covered_secondary_key_lookups]
+# Eligible for optimization.
+id	bigfield
+32	xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+include/assert.inc [Eligible for optimization.: 1 rocksdb_covered_secondary_key_lookups]
+# Not eligible for optimization.
+id	bigfield
+33	yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
+include/assert.inc [Not eligible for optimization.: 0 rocksdb_covered_secondary_key_lookups]
+# Eligible for optimization.
+id	bigfield
+8	bbbbbbbb
+include/assert.inc [Eligible for optimization.: 2 rocksdb_covered_secondary_key_lookups]
+# Eligible for optimization.
+id	bigfield
+24	cccccccccccccccccccccccc
+include/assert.inc [Eligible for optimization.: 2 rocksdb_covered_secondary_key_lookups]
+# Not eligible for optimization.
+id	bigfield
+128	zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+include/assert.inc [Not eligible for optimization.: 0 rocksdb_covered_secondary_key_lookups]
+#
+# Test that multi-byte charsets are handled correctly
+#
+# Charset record obviously shorter than the prefix
+a	b
+1	a
+include/assert.inc [Charset record obviously shorter than the prefix: 2 rocksdb_covered_secondary_key_lookups]
+# Charset record shorter than prefix
+a	b
+2	cc
+include/assert.inc [Charset record shorter than prefix: 2 rocksdb_covered_secondary_key_lookups]
+# Charset record with glyphs shorter than prefix
+a	b
+3	ŽŽ
+include/assert.inc [Charset record with glyphs shorter than prefix: 1 rocksdb_covered_secondary_key_lookups]
+# Charset record longer than prefix
+a	b
+4	žžžž
+include/assert.inc [Charset record longer than prefix: 0 rocksdb_covered_secondary_key_lookups]

mysql-test/suite/rocksdb/r/multi_varchar_sk_lookup.result

@@ -0,0 +1,37 @@
+DROP TABLE IF EXISTS T1;
+CREATE TABLE T1 (
+P1 VARCHAR(64), P2 VARCHAR(64), P3 VARCHAR(64), P4 VARCHAR(64),
+S1 VARCHAR(64), S2 VARCHAR(64), S3 VARCHAR(64), S4 VARCHAR(64),
+S5 VARCHAR(64), S6 VARCHAR(64), S7 VARCHAR(64), S8 VARCHAR(64),
+S9 VARCHAR(64), S10 VARCHAR(64), S11 VARCHAR(64), S12 VARCHAR(64),
+S13 VARCHAR(64), S14 VARCHAR(64), S15 VARCHAR(64), S16 VARCHAR(64),
+PRIMARY KEY (P1(8), P2(8), P3(8), P4(8)),
+KEY SK (S1(8), S2(8), S3(8), S4(8),
+S5(8), S6(8), S7(8), S8(8),
+S9(8), S10(8), S11(8), S12(8),
+S13(8), S14(8), S15(8), S16(8))
+) ENGINE=rocksdb;
+INSERT INTO T1 VALUES ('1', '2', '3', '4',
+'5', '6', '7', '8',
+'9', '10', '11', '12',
+'13', '14', '15', '16',
+'17', '18', '19', '20');
+SELECT * FROM T1;
+P1	P2	P3	P4	S1	S2	S3	S4	S5	S6	S7	S8	S9	S10	S11	S12	S13	S14	S15	S16
+1	2	3	4	5	6	7	8	9	10	11	12	13	14	15	16	17	18	19	20
+# Not eligible for optimization, shorter than prefix length.
+SELECT P4, S2 FROM T1 FORCE INDEX(SK) WHERE S1='5';
+P4	S2
+4	6
+include/assert.inc [Not eligible for optimization, shorter than prefix length.: 0 rocksdb_covered_secondary_key_lookups]
+DELETE FROM T1;
+INSERT INTO T1 VALUES ('100000000', '200000000', '300000000', '400000000',
+'500000000', '600000000', '700000000', '800000000',
+'900000000', '100000000', '110000000', '120000000',
+'130000000', '140000000', '150000000', '160000000',
+'170000000', '180000000', '190000000', '200000000');
+# Not eligible for optimization, longer than prefix length.
+SELECT P4, S2 FROM T1 FORCE INDEX(SK) WHERE S1='5';
+P4	S2
+include/assert.inc [Not eligible for optimization, longer than prefix length.: 0 rocksdb_covered_secondary_key_lookups]
+DROP TABLE T1;

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -1448,6 +1448,7 @@ rocksdb_memtable_total	#
 rocksdb_memtable_unflushed	#
 rocksdb_queries_point	#
 rocksdb_queries_range	#
+rocksdb_covered_secondary_key_lookups	#
 rocksdb_block_cache_add	#
 rocksdb_block_cache_data_hit	#
 rocksdb_block_cache_data_miss	#
@@ -1524,6 +1525,7 @@ ROCKSDB_MEMTABLE_TOTAL
 ROCKSDB_MEMTABLE_UNFLUSHED
 ROCKSDB_QUERIES_POINT
 ROCKSDB_QUERIES_RANGE
+ROCKSDB_COVERED_SECONDARY_KEY_LOOKUPS
 ROCKSDB_BLOCK_CACHE_ADD
 ROCKSDB_BLOCK_CACHE_DATA_HIT
 ROCKSDB_BLOCK_CACHE_DATA_MISS
@@ -1602,6 +1604,7 @@ ROCKSDB_MEMTABLE_TOTAL
 ROCKSDB_MEMTABLE_UNFLUSHED
 ROCKSDB_QUERIES_POINT
 ROCKSDB_QUERIES_RANGE
+ROCKSDB_COVERED_SECONDARY_KEY_LOOKUPS
 ROCKSDB_BLOCK_CACHE_ADD
 ROCKSDB_BLOCK_CACHE_DATA_HIT
 ROCKSDB_BLOCK_CACHE_DATA_MISS

mysql-test/suite/rocksdb/t/add_index_inplace.test

@@ -255,7 +255,7 @@ SET @prior_rocksdb_merge_buf_size = @@rocksdb_merge_buf_size;
 
 SET global rocksdb_strict_collation_check = off;
 SET session rocksdb_merge_combine_read_size = 566;
-SET session rocksdb_merge_buf_size = 336;
+SET session rocksdb_merge_buf_size = 340;
 
 show variables like '%rocksdb_bulk_load%';
 CREATE TABLE t1 (a VARCHAR(80)) ENGINE=RocksDB;

mysql-test/suite/rocksdb/t/covered_unpack_info_format.test

@@ -0,0 +1,79 @@
+# Disable until covered unpack_info format enabled in prod
+--source include/have_debug.inc
+--source include/have_nodebug.inc
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+#
+# Normal case
+#
+
+CREATE TABLE t1 (
+  id INT,
+  fake_id INT,
+  bigfield VARCHAR(4096),
+  PRIMARY KEY (id),
+  KEY bf (bigfield(32)),
+  KEY fid (fake_id, bigfield(32))
+) ENGINE=rocksdb;
+
+INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)),
+                      (8, 1008, REPEAT('b', 8)),
+                      (24, 1024, REPEAT('c', 24)),
+                      (31, 1031, REPEAT('d', 31)),
+                      (32, 1032, REPEAT('x', 32)),
+                      (33, 1033, REPEAT('y', 33)),
+                      (128, 1128, REPEAT('z', 128));
+
+SELECT * FROM t1;
+
+--let $prefix_index_check_title= Eligible for optimization, access via fake_id only
+--let $prefix_index_check_read_avoided_delta= 2
+--let $prefix_index_check_query= SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1031
+--source suite/rocksdb/include/prefix_index_only_query_check.inc
+
+--let $prefix_index_check_title= Not eligible for optimization, access via fake_id of big row.
+--let $prefix_index_check_read_avoided_delta= 0
+--let $prefix_index_check_query= SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1033
+--source suite/rocksdb/include/prefix_index_only_query_check.inc
+
+DROP TABLE t1;
+
+#
+# Create same table with older format to test compatibility
+#
+
+set session debug= '+d,MYROCKS_NO_COVERED_BITMAP_FORMAT';
+CREATE TABLE t1 (
+  id INT,
+  fake_id INT,
+  bigfield VARCHAR(4096),
+  PRIMARY KEY (id),
+  KEY bf (bigfield(32)),
+  KEY fid (fake_id, bigfield(32))
+) ENGINE=rocksdb;
+set session debug= '-d,MYROCKS_NO_COVERED_BITMAP_FORMAT';
+
+INSERT INTO t1 VALUES (1, 1001, REPEAT('a', 1)),
+                      (8, 1008, REPEAT('b', 8)),
+                      (24, 1024, REPEAT('c', 24)),
+                      (31, 1031, REPEAT('d', 31)),
+                      (32, 1032, REPEAT('x', 32)),
+                      (33, 1033, REPEAT('y', 33)),
+                      (128, 1128, REPEAT('z', 128));
+
+SELECT * FROM t1;
+
+--let $prefix_index_check_title= No longer eligible for optimization since no covered bitmap was stored.
+--let $prefix_index_check_read_avoided_delta= 0
+--let $prefix_index_check_query= SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1031
+--source suite/rocksdb/include/prefix_index_only_query_check.inc
+
+--let $prefix_index_check_title= Not eligible for optimization.
+--let $prefix_index_check_read_avoided_delta= 0
+--let $prefix_index_check_query= SELECT id, bigfield FROM t1 FORCE INDEX(fid) WHERE fake_id = 1033
+--source suite/rocksdb/include/prefix_index_only_query_check.inc
+
+DROP TABLE t1;