Prematerialize partial indexes during bulk loading

Upstream commit ID : fb-mysql-5.6.35/d054548154184f51ce297fc033e68b61aed71f3e
PS-7921 : Merge percona-202105

Summary: Instead of leaving an index empty after bulk loading a table, we can prematerialize groups that are over the materialization threshold.

Reviewed By: hermanlee

Differential Revision: D27344590

fbshipit-source-id: 3cd5f7dcc70

mysql-test/suite/rocksdb/r/add_index_inplace.result

@@ -295,12 +295,14 @@ Variable_name	Value
 rocksdb_bulk_load	ON
 rocksdb_bulk_load_allow_sk	OFF
 rocksdb_bulk_load_allow_unsorted	OFF
+rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
 show session variables like 'rocksdb_bulk_load%';
 Variable_name	Value
 rocksdb_bulk_load	ON
 rocksdb_bulk_load_allow_sk	OFF
 rocksdb_bulk_load_allow_unsorted	OFF
+rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
 CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB;
 INSERT INTO t1 VALUES (1,1);
@@ -345,6 +347,7 @@ Variable_name	Value
 rocksdb_bulk_load	OFF
 rocksdb_bulk_load_allow_sk	OFF
 rocksdb_bulk_load_allow_unsorted	OFF
+rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
 CREATE TABLE t1 (a VARCHAR(30)) ENGINE=RocksDB;
 INSERT INTO t1 (a) VALUES (REPEAT("a", 30));

mysql-test/suite/rocksdb/r/partial_index_bulk_load.result

@@ -0,0 +1,149 @@
+SET rocksdb_bulk_load_size=10;
+CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
+PRIMARY KEY (i, j),
+KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
+KEY ik2 (i, k) COMMENT 'cfname=cf'
+) ENGINE=ROCKSDB;
+Warnings:
+Warning	1831	Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+SET rocksdb_bulk_load_allow_sk=1;
+SET rocksdb_bulk_load=1;
+SET rocksdb_bulk_load=0;
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+ALTER TABLE t DROP INDEX ik1;
+ALTER TABLE t ADD INDEX ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5';
+Warnings:
+Warning	1831	Duplicate index 'ik1' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+DROP TABLE t;
+CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
+PRIMARY KEY (i, j),
+KEY ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5',
+KEY ik2 (i, k) COMMENT 'cfname=rev:cf'
+) ENGINE=ROCKSDB;
+Warnings:
+Warning	1831	Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+SET rocksdb_bulk_load_allow_sk=1;
+SET rocksdb_bulk_load=1;
+SET rocksdb_bulk_load=0;
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+ALTER TABLE t DROP INDEX ik1;
+ALTER TABLE t ADD INDEX ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5';
+Warnings:
+Warning	1831	Duplicate index 'ik1' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+DROP TABLE t;
+CREATE TABLE t (i int, j int, k int, l int,
+PRIMARY KEY (i, j),
+KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
+KEY ik2 (i, k) COMMENT 'cfname=cf'
+) ENGINE=ROCKSDB;
+Warnings:
+Warning	1831	Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+SET rocksdb_bulk_load_allow_sk=1;
+SET rocksdb_bulk_load=1;
+SET rocksdb_bulk_load=0;
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+ALTER TABLE t DROP INDEX ik1;
+ALTER TABLE t ADD INDEX ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5';
+Warnings:
+Warning	1831	Duplicate index 'ik1' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+DROP TABLE t;
+CREATE TABLE t (i int, j int, k int, l int,
+PRIMARY KEY (i, j),
+KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
+KEY ik2 (i, k) COMMENT 'cfname=cf'
+) ENGINE=ROCKSDB;
+Warnings:
+Warning	1831	Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+SET rocksdb_bulk_load_allow_sk=1;
+SET rocksdb_bulk_load=1;
+SET rocksdb_bulk_load=0;
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+ALTER TABLE t DROP INDEX ik1;
+ALTER TABLE t ADD INDEX ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5';
+Warnings:
+Warning	1831	Duplicate index 'ik1' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+include/assert.inc [Check that materialized groups are zero.]
+include/assert.inc [Check that materialized rows are zero.]
+include/assert.inc [Check that sorted groups are non-zero.]
+include/assert.inc [Check that sorted rows are non-zero.]
+DROP TABLE t1, t2;
+DROP TABLE t;

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -904,6 +904,7 @@ rocksdb_block_size_deviation	10
 rocksdb_bulk_load	OFF
 rocksdb_bulk_load_allow_sk	OFF
 rocksdb_bulk_load_allow_unsorted	OFF
+rocksdb_bulk_load_partial_index	ON
 rocksdb_bulk_load_size	1000
 rocksdb_bytes_per_sync	0
 rocksdb_cache_dump	ON

mysql-test/suite/rocksdb/t/partial_index_bulk_load.inc

@@ -0,0 +1,79 @@
+SET rocksdb_bulk_load_allow_sk=1;
+SET rocksdb_bulk_load=1;
+
+--disable_query_log
+let $i = 1;
+while ($i <= 10) {
+  let $j = 1;
+  while ($j <= $i) {
+    if ($varchar) {
+      eval INSERT INTO t VALUES (REPEAT("1", $i), REPEAT("2", $j), REPEAT("3", $j), "12345");
+    }
+    if (!$varchar) {
+      eval INSERT INTO t VALUES ($i, $j, 10 - $j, 12345);
+    }
+    inc $j;
+  }
+  inc $i;
+}
+--enable_query_log
+
+SET rocksdb_bulk_load=0;
+
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+
+--let $query1= SELECT i, j, k FROM t FORCE INDEX (ik1);
+--let $query2= SELECT i, j, k FROM t FORCE INDEX (ik2);
+--source include/diff_queries.inc
+
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+
+--let $assert_text = Check that materialized groups are zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_groups_materialized", diff, 1] = 0
+--source include/assert.inc
+--let $assert_text = Check that materialized rows are zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_rows_materialized", diff, 1] = 0
+--source include/assert.inc
+--let $assert_text = Check that sorted groups are non-zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_groups_sorted", diff, 1] > 0
+--source include/assert.inc
+--let $assert_text = Check that sorted rows are non-zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_rows_sorted", diff, 1] > 0
+--source include/assert.inc
+
+DROP TABLE t1, t2;
+
+--let $comment=`select index_comment from information_schema.statistics where table_schema = 'test' and table_name = 't' limit 1`
+ALTER TABLE t DROP INDEX ik1;
+--eval ALTER TABLE t ADD INDEX ik1 (i, k) COMMENT '$comment'
+
+CREATE TEMPORARY TABLE t1 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+
+--let $query1= SELECT i, j, k FROM t FORCE INDEX (ik1);
+--let $query2= SELECT i, j, k FROM t FORCE INDEX (ik2);
+--source include/diff_queries.inc
+
+CREATE TEMPORARY TABLE t2 AS
+SELECT * FROM performance_schema.global_status
+WHERE variable_name LIKE 'rocksdb_partial_index%';
+
+--let $assert_text = Check that materialized groups are zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_groups_materialized", diff, 1] = 0
+--source include/assert.inc
+--let $assert_text = Check that materialized rows are zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_rows_materialized", diff, 1] = 0
+--source include/assert.inc
+--let $assert_text = Check that sorted groups are non-zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_groups_sorted", diff, 1] > 0
+--source include/assert.inc
+--let $assert_text = Check that sorted rows are non-zero.
+--let $assert_cond = [SELECT t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name) WHERE variable_name = "rocksdb_partial_index_rows_sorted", diff, 1] > 0
+--source include/assert.inc
+
+DROP TABLE t1, t2;

mysql-test/suite/rocksdb/t/partial_index_bulk_load.test

@@ -0,0 +1,47 @@
+--source include/have_rocksdb.inc
+
+SET rocksdb_bulk_load_size=10;
+
+CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
+  PRIMARY KEY (i, j),
+  KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
+  KEY ik2 (i, k) COMMENT 'cfname=cf'
+) ENGINE=ROCKSDB;
+
+--let $varchar=1
+--source partial_index_bulk_load.inc
+
+DROP TABLE t;
+
+CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
+  PRIMARY KEY (i, j),
+  KEY ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5',
+  KEY ik2 (i, k) COMMENT 'cfname=rev:cf'
+) ENGINE=ROCKSDB;
+
+--let $varchar=1
+--source partial_index_bulk_load.inc
+
+DROP TABLE t;
+
+CREATE TABLE t (i int, j int, k int, l int,
+  PRIMARY KEY (i, j),
+  KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
+  KEY ik2 (i, k) COMMENT 'cfname=cf'
+) ENGINE=ROCKSDB;
+
+--let $varchar=0
+--source partial_index_bulk_load.inc
+
+DROP TABLE t;
+
+CREATE TABLE t (i int, j int, k int, l int,
+  PRIMARY KEY (i, j),
+  KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
+  KEY ik2 (i, k) COMMENT 'cfname=cf'
+) ENGINE=ROCKSDB;
+
+--let $varchar=0
+--source partial_index_bulk_load.inc
+
+DROP TABLE t;