Skip to content

Commit

Permalink
Support for fast secondary index creation for MyRocks
Browse files Browse the repository at this point in the history 
Summary:
This is part 2 of the changes required to support fast secondary index
creation. See Part 1 here (facebook@fbbd4aa).

Previously adding an index would require copying over the entire table and
recreating all the indexes. This diff implements the functionality required to
add an index "online", without having to rebuild the table.

Fixes facebook#182
Also part of facebook#47

Future improvements are planned in a separate diff, to support SSTFileWriter API.

update-submodule: rocksdb

Differential Revision: https://reviews.facebook.net/D62331

fbshipit-source-id: f777e7e
  • Loading branch information
@alxyang @inikep
alxyang authored and inikep committed Aug 12, 2020
1 parent d595ffc commit 1fc4982
Show file tree
Hide file tree
Showing 9 changed files with 1,202 additions and 170 deletions.
378 changes: 378 additions & 0 deletions mysql-test/suite/rocksdb/r/add_index_inplace.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,378 @@
drop table if exists t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
INSERT INTO t1 (a, b) VALUES (1, 5);
INSERT INTO t1 (a, b) VALUES (2, 6);
INSERT INTO t1 (a, b) VALUES (3, 7);
ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `ka` (`a`),
KEY `kab` (`a`,`b`),
KEY `kb` (`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5;
a b
2 6
3 7
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2;
a b
3 7
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
INSERT INTO t1 (a, b) VALUES (1, 5);
INSERT INTO t1 (a, b) VALUES (2, 6);
INSERT INTO t1 (a, b) VALUES (3, 7);
ALTER TABLE t1 ADD INDEX kb(b), DROP INDEX ka, ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `kab` (`a`,`b`),
KEY `kb` (`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5;
a b
2 6
3 7
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2;
a b
3 7
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
INSERT INTO t1 (a, b) VALUES (1, 5);
INSERT INTO t1 (a, b) VALUES (2, 6);
INSERT INTO t1 (a, b) VALUES (3, 7);
ALTER TABLE t1 DROP INDEX ka, DROP INDEX kab, ALGORITHM=INPLACE;
ALTER TABLE t1 ADD INDEX kb(b), ADD INDEX kab(a,b), ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `kb` (`b`),
KEY `kab` (`a`,`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5;
a b
2 6
3 7
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2;
a b
3 7
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
INSERT INTO t1 (a, b) VALUES (1, 5);
INSERT INTO t1 (a, b) VALUES (2, 6);
INSERT INTO t1 (a, b) VALUES (3, 7);
ALTER TABLE t1 ADD INDEX kb(b), DROP INDEX ka, ADD INDEX kba(b,a), DROP INDEX kab, ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `kb` (`b`),
KEY `kba` (`b`,`a`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5;
a b
2 6
3 7
SELECT * FROM t1 FORCE INDEX(kba) WHERE a > 2;
a b
3 7
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
ALTER TABLE t1 DROP INDEX ka, ADD INDEX ka(b), ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `kab` (`a`,`b`),
KEY `ka` (`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(ka) WHERE b > 5;
a b
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2;
a b
DROP TABLE t1;
CREATE TABLE t1 (pk CHAR(8) PRIMARY KEY, a VARCHAR(11), b INT UNSIGNED) ENGINE=rocksdb charset utf8 collate utf8_bin;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`pk` char(8) COLLATE utf8_bin NOT NULL,
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL,
`b` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`pk`)
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin
SHOW COLUMNS IN t1;
Field Type Null Key Default Extra
pk char(8) NO PRI NULL
a varchar(11) YES NULL
b int(10) unsigned YES NULL
INSERT INTO t1 VALUES ('aaa', '1111', 1);
INSERT INTO t1 VALUES ('bbb', '2222', 2);
INSERT INTO t1 VALUES ('ccc', '3333', 3);
ALTER TABLE t1 ADD INDEX kab(a,b), ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`pk` char(8) COLLATE utf8_bin NOT NULL,
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL,
`b` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`pk`),
KEY `kab` (`a`,`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > '2' AND b < 3;
pk a b
bbb 2222 2
DROP TABLE t1;
CREATE TABLE t1 (pk CHAR(8) PRIMARY KEY, a VARCHAR(11), b INT UNSIGNED) ENGINE=rocksdb charset utf8 collate utf8_bin;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`pk` char(8) COLLATE utf8_bin NOT NULL,
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL,
`b` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`pk`)
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin
SHOW COLUMNS IN t1;
Field Type Null Key Default Extra
pk char(8) NO PRI NULL
a varchar(11) YES NULL
b int(10) unsigned YES NULL
INSERT INTO t1 VALUES ('aaa', '1111', 1);
INSERT INTO t1 VALUES ('bbb', '2222', 2);
INSERT INTO t1 VALUES ('ccc', '3333', 3);
ALTER TABLE t1 ADD INDEX kab(a,b), ALGORITHM=INPLACE;
ALTER TABLE t1 ADD INDEX ka(a), DROP INDEX kab, ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`pk` char(8) COLLATE utf8_bin NOT NULL,
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL,
`b` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`pk`),
KEY `ka` (`a`)
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(ka) WHERE a > '2' AND b < 3;
pk a b
bbb 2222 2
DROP TABLE t1;
CREATE TABLE t1 (pk CHAR(8) PRIMARY KEY, a VARCHAR(11), b INT UNSIGNED) ENGINE=rocksdb charset utf8 collate utf8_bin;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`pk` char(8) COLLATE utf8_bin NOT NULL,
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL,
`b` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`pk`)
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin
SHOW COLUMNS IN t1;
Field Type Null Key Default Extra
pk char(8) NO PRI NULL
a varchar(11) YES NULL
b int(10) unsigned YES NULL
INSERT INTO t1 VALUES ('aaa', '1111', 1);
INSERT INTO t1 VALUES ('bbb', '2222', 2);
INSERT INTO t1 VALUES ('ccc', '3333', 3);
ALTER TABLE t1 ADD INDEX kab(a,b), ADD INDEX ka(a), ADD INDEX kb(b), ALGORITHM=INPLACE;
ALTER TABLE t1 DROP INDEX ka, DROP INDEX kb, ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`pk` char(8) COLLATE utf8_bin NOT NULL,
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL,
`b` int(10) unsigned DEFAULT NULL,
PRIMARY KEY (`pk`),
KEY `kab` (`a`,`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > '2' AND b < 3;
pk a b
bbb 2222 2
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
INSERT INTO t1 (a, b) VALUES (1, 5);
INSERT INTO t1 (a, b) VALUES (2, 6);
INSERT INTO t1 (a, b) VALUES (3, 7);
CREATE INDEX kb on t1 (b);
CREATE INDEX kba on t1 (b,a);
DROP INDEX ka on t1;
DROP INDEX kab on t1;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `kb` (`b`),
KEY `kba` (`b`,`a`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5;
a b
2 6
3 7
SELECT * FROM t1 FORCE INDEX(kba) WHERE a > 2;
a b
3 7
DROP TABLE t1;
CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4;
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
DROP INDEX kij ON t1;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`i` int(11) NOT NULL DEFAULT '0',
`j` int(11) DEFAULT NULL,
`k` int(11) DEFAULT NULL,
PRIMARY KEY (`i`),
KEY `j` (`j`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
/*!50100 PARTITION BY KEY (i)
PARTITIONS 4 */
SELECT * FROM t1 ORDER BY i LIMIT 10;
i j k
1 1 1
2 2 2
3 3 3
4 4 4
5 5 5
6 6 6
7 7 7
8 8 8
9 9 9
10 10 10
SELECT COUNT(*) FROM t1;
COUNT(*)
100
DROP TABLE t1;
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB;
INSERT INTO t1 (a, b) VALUES (1, 5);
INSERT INTO t1 (a, b) VALUES (2, 6);
INSERT INTO t1 (a, b) VALUES (3, 7);
# crash_during_online_index_creation
flush logs;
SET SESSION debug="+d,crash_during_online_index_creation";
ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE;
ERROR HY000: Lost connection to MySQL server during query
SET SESSION debug="-d,crash_during_online_index_creation";
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` int(11) DEFAULT NULL,
`b` int(11) DEFAULT NULL,
KEY `ka` (`a`),
KEY `kab` (`a`,`b`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
DROP TABLE t1;
CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4;
# crash_during_index_creation_partition
flush logs;
SET SESSION debug="+d,crash_during_index_creation_partition";
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
ERROR HY000: Lost connection to MySQL server during query
SET SESSION debug="-d,crash_during_index_creation_partition";
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`i` int(11) NOT NULL DEFAULT '0',
`j` int(11) DEFAULT NULL,
`k` int(11) DEFAULT NULL,
PRIMARY KEY (`i`),
KEY `j` (`j`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
/*!50100 PARTITION BY KEY (i)
PARTITIONS 4 */
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
SELECT * FROM t1 ORDER BY i LIMIT 10;
i j k
1 1 1
2 2 2
3 3 3
4 4 4
5 5 5
6 6 6
7 7 7
8 8 8
9 9 9
10 10 10
SELECT COUNT(*) FROM t1;
COUNT(*)
100
DROP TABLE t1;
CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4;
# crash_during_index_creation_partition
flush logs;
SET SESSION debug="+d,myrocks_simulate_index_create_rollback";
# expected assertion failure from sql layer here for alter rollback
call mtr.add_suppression("Assertion `0' failed.");
call mtr.add_suppression("Attempting backtrace. You can use the following information to find out");
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
ERROR HY000: Lost connection to MySQL server during query
SET SESSION debug="-d,myrocks_simulate_index_create_rollback";
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`i` int(11) NOT NULL DEFAULT '0',
`j` int(11) DEFAULT NULL,
`k` int(11) DEFAULT NULL,
PRIMARY KEY (`i`),
KEY `j` (`j`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
/*!50100 PARTITION BY KEY (i)
PARTITIONS 4 */
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`i` int(11) NOT NULL DEFAULT '0',
`j` int(11) DEFAULT NULL,
`k` int(11) DEFAULT NULL,
PRIMARY KEY (`i`),
KEY `j` (`j`),
KEY `kij` (`i`,`j`)
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
/*!50100 PARTITION BY KEY (i)
PARTITIONS 4 */
SELECT COUNT(*) FROM t1;
COUNT(*)
100
DROP TABLE t1;
CREATE TABLE t1 (a INT, b TEXT);
ALTER TABLE t1 ADD KEY kb(b(10));
ERROR HY000: Unsupported collation on string indexed column test.t1.b Use binary collation (binary, latin1_bin, utf8_bin).
ALTER TABLE t1 ADD PRIMARY KEY(a);
DROP TABLE t1;
2 changes: 1 addition & 1 deletion mysql-test/suite/rocksdb/r/information_schema.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK';
Variable_name Value
rocksdb_pause_background_work ON
DROP TABLE t3;
cf_id:0,index_id:271
cf_id:0,index_id:268
SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0;
SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK';
Variable_name Value
Expand Down
5 changes: 5 additions & 0 deletions mysql-test/suite/rocksdb/t/add_index_inplace.cnf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[mysql]
no-defaults

[mysqld.1]
rocksdb_strict_collation_check=1
Loading

0 comments on commit 1fc4982

Please sign in to comment.