forked from facebook/mysql-5.6
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support for fast secondary index creation for MyRocks
Summary: This is part 2 of the changes required to support fast secondary index creation. See Part 1 here (facebook@fbbd4aa). Previously adding an index would require copying over the entire table and recreating all the indexes. This diff implements the functionality required to add an index "online", without having to rebuild the table. Fixes facebook#182 Also part of facebook#47 Future improvements are planned in a separate diff, to support SSTFileWriter API. @update-submodule: rocksdb Test Plan: add_index_inplace test file which currently has basic tests to cover several corner cases + a crash recovery test using DBUG_SUICIDE Performance test results are included in the quip document here https://fb.quip.com/l3gDA9IjGUVw Also ran checksum tests to ensure that data matched between two instances, one which did inplace index creation and another that was initially created with the indexes Reviewers: yoshinorim, jkedgar, spetrunia, hermanlee4 Reviewed By: hermanlee4 Subscribers: sdong, webscalesql-eng Differential Revision: https://reviews.facebook.net/D60837 Differential Revision: https://reviews.facebook.net/D61659 Differential Revision: https://reviews.facebook.net/D62331
- Loading branch information
Showing
9 changed files
with
1,202 additions
and
170 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,378 @@ | ||
drop table if exists t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
INSERT INTO t1 (a, b) VALUES (1, 5); | ||
INSERT INTO t1 (a, b) VALUES (2, 6); | ||
INSERT INTO t1 (a, b) VALUES (3, 7); | ||
ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `ka` (`a`), | ||
KEY `kab` (`a`,`b`), | ||
KEY `kb` (`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5; | ||
a b | ||
2 6 | ||
3 7 | ||
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2; | ||
a b | ||
3 7 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
INSERT INTO t1 (a, b) VALUES (1, 5); | ||
INSERT INTO t1 (a, b) VALUES (2, 6); | ||
INSERT INTO t1 (a, b) VALUES (3, 7); | ||
ALTER TABLE t1 ADD INDEX kb(b), DROP INDEX ka, ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `kab` (`a`,`b`), | ||
KEY `kb` (`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5; | ||
a b | ||
2 6 | ||
3 7 | ||
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2; | ||
a b | ||
3 7 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
INSERT INTO t1 (a, b) VALUES (1, 5); | ||
INSERT INTO t1 (a, b) VALUES (2, 6); | ||
INSERT INTO t1 (a, b) VALUES (3, 7); | ||
ALTER TABLE t1 DROP INDEX ka, DROP INDEX kab, ALGORITHM=INPLACE; | ||
ALTER TABLE t1 ADD INDEX kb(b), ADD INDEX kab(a,b), ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `kb` (`b`), | ||
KEY `kab` (`a`,`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5; | ||
a b | ||
2 6 | ||
3 7 | ||
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2; | ||
a b | ||
3 7 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
INSERT INTO t1 (a, b) VALUES (1, 5); | ||
INSERT INTO t1 (a, b) VALUES (2, 6); | ||
INSERT INTO t1 (a, b) VALUES (3, 7); | ||
ALTER TABLE t1 ADD INDEX kb(b), DROP INDEX ka, ADD INDEX kba(b,a), DROP INDEX kab, ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `kb` (`b`), | ||
KEY `kba` (`b`,`a`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5; | ||
a b | ||
2 6 | ||
3 7 | ||
SELECT * FROM t1 FORCE INDEX(kba) WHERE a > 2; | ||
a b | ||
3 7 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
ALTER TABLE t1 DROP INDEX ka, ADD INDEX ka(b), ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `kab` (`a`,`b`), | ||
KEY `ka` (`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(ka) WHERE b > 5; | ||
a b | ||
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > 2; | ||
a b | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (pk CHAR(8) PRIMARY KEY, a VARCHAR(11), b INT UNSIGNED) ENGINE=rocksdb charset utf8 collate utf8_bin; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`pk` char(8) COLLATE utf8_bin NOT NULL, | ||
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL, | ||
`b` int(10) unsigned DEFAULT NULL, | ||
PRIMARY KEY (`pk`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin | ||
SHOW COLUMNS IN t1; | ||
Field Type Null Key Default Extra | ||
pk char(8) NO PRI NULL | ||
a varchar(11) YES NULL | ||
b int(10) unsigned YES NULL | ||
INSERT INTO t1 VALUES ('aaa', '1111', 1); | ||
INSERT INTO t1 VALUES ('bbb', '2222', 2); | ||
INSERT INTO t1 VALUES ('ccc', '3333', 3); | ||
ALTER TABLE t1 ADD INDEX kab(a,b), ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`pk` char(8) COLLATE utf8_bin NOT NULL, | ||
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL, | ||
`b` int(10) unsigned DEFAULT NULL, | ||
PRIMARY KEY (`pk`), | ||
KEY `kab` (`a`,`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > '2' AND b < 3; | ||
pk a b | ||
bbb 2222 2 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (pk CHAR(8) PRIMARY KEY, a VARCHAR(11), b INT UNSIGNED) ENGINE=rocksdb charset utf8 collate utf8_bin; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`pk` char(8) COLLATE utf8_bin NOT NULL, | ||
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL, | ||
`b` int(10) unsigned DEFAULT NULL, | ||
PRIMARY KEY (`pk`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin | ||
SHOW COLUMNS IN t1; | ||
Field Type Null Key Default Extra | ||
pk char(8) NO PRI NULL | ||
a varchar(11) YES NULL | ||
b int(10) unsigned YES NULL | ||
INSERT INTO t1 VALUES ('aaa', '1111', 1); | ||
INSERT INTO t1 VALUES ('bbb', '2222', 2); | ||
INSERT INTO t1 VALUES ('ccc', '3333', 3); | ||
ALTER TABLE t1 ADD INDEX kab(a,b), ALGORITHM=INPLACE; | ||
ALTER TABLE t1 ADD INDEX ka(a), DROP INDEX kab, ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`pk` char(8) COLLATE utf8_bin NOT NULL, | ||
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL, | ||
`b` int(10) unsigned DEFAULT NULL, | ||
PRIMARY KEY (`pk`), | ||
KEY `ka` (`a`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(ka) WHERE a > '2' AND b < 3; | ||
pk a b | ||
bbb 2222 2 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (pk CHAR(8) PRIMARY KEY, a VARCHAR(11), b INT UNSIGNED) ENGINE=rocksdb charset utf8 collate utf8_bin; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`pk` char(8) COLLATE utf8_bin NOT NULL, | ||
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL, | ||
`b` int(10) unsigned DEFAULT NULL, | ||
PRIMARY KEY (`pk`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin | ||
SHOW COLUMNS IN t1; | ||
Field Type Null Key Default Extra | ||
pk char(8) NO PRI NULL | ||
a varchar(11) YES NULL | ||
b int(10) unsigned YES NULL | ||
INSERT INTO t1 VALUES ('aaa', '1111', 1); | ||
INSERT INTO t1 VALUES ('bbb', '2222', 2); | ||
INSERT INTO t1 VALUES ('ccc', '3333', 3); | ||
ALTER TABLE t1 ADD INDEX kab(a,b), ADD INDEX ka(a), ADD INDEX kb(b), ALGORITHM=INPLACE; | ||
ALTER TABLE t1 DROP INDEX ka, DROP INDEX kb, ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`pk` char(8) COLLATE utf8_bin NOT NULL, | ||
`a` varchar(11) COLLATE utf8_bin DEFAULT NULL, | ||
`b` int(10) unsigned DEFAULT NULL, | ||
PRIMARY KEY (`pk`), | ||
KEY `kab` (`a`,`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kab) WHERE a > '2' AND b < 3; | ||
pk a b | ||
bbb 2222 2 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
INSERT INTO t1 (a, b) VALUES (1, 5); | ||
INSERT INTO t1 (a, b) VALUES (2, 6); | ||
INSERT INTO t1 (a, b) VALUES (3, 7); | ||
CREATE INDEX kb on t1 (b); | ||
CREATE INDEX kba on t1 (b,a); | ||
DROP INDEX ka on t1; | ||
DROP INDEX kab on t1; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `kb` (`b`), | ||
KEY `kba` (`b`,`a`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
SELECT * FROM t1 FORCE INDEX(kb) WHERE b > 5; | ||
a b | ||
2 6 | ||
3 7 | ||
SELECT * FROM t1 FORCE INDEX(kba) WHERE a > 2; | ||
a b | ||
3 7 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; | ||
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; | ||
DROP INDEX kij ON t1; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`i` int(11) NOT NULL DEFAULT '0', | ||
`j` int(11) DEFAULT NULL, | ||
`k` int(11) DEFAULT NULL, | ||
PRIMARY KEY (`i`), | ||
KEY `j` (`j`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
/*!50100 PARTITION BY KEY (i) | ||
PARTITIONS 4 */ | ||
SELECT * FROM t1 ORDER BY i LIMIT 10; | ||
i j k | ||
1 1 1 | ||
2 2 2 | ||
3 3 3 | ||
4 4 4 | ||
5 5 5 | ||
6 6 6 | ||
7 7 7 | ||
8 8 8 | ||
9 9 9 | ||
10 10 10 | ||
SELECT COUNT(*) FROM t1; | ||
COUNT(*) | ||
100 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; | ||
INSERT INTO t1 (a, b) VALUES (1, 5); | ||
INSERT INTO t1 (a, b) VALUES (2, 6); | ||
INSERT INTO t1 (a, b) VALUES (3, 7); | ||
# crash_during_online_index_creation | ||
flush logs; | ||
SET SESSION debug="+d,crash_during_online_index_creation"; | ||
ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; | ||
ERROR HY000: Lost connection to MySQL server during query | ||
SET SESSION debug="-d,crash_during_online_index_creation"; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`a` int(11) DEFAULT NULL, | ||
`b` int(11) DEFAULT NULL, | ||
KEY `ka` (`a`), | ||
KEY `kab` (`a`,`b`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
CHECK TABLE t1; | ||
Table Op Msg_type Msg_text | ||
test.t1 check status OK | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; | ||
# crash_during_index_creation_partition | ||
flush logs; | ||
SET SESSION debug="+d,crash_during_index_creation_partition"; | ||
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; | ||
ERROR HY000: Lost connection to MySQL server during query | ||
SET SESSION debug="-d,crash_during_index_creation_partition"; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`i` int(11) NOT NULL DEFAULT '0', | ||
`j` int(11) DEFAULT NULL, | ||
`k` int(11) DEFAULT NULL, | ||
PRIMARY KEY (`i`), | ||
KEY `j` (`j`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
/*!50100 PARTITION BY KEY (i) | ||
PARTITIONS 4 */ | ||
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; | ||
SELECT * FROM t1 ORDER BY i LIMIT 10; | ||
i j k | ||
1 1 1 | ||
2 2 2 | ||
3 3 3 | ||
4 4 4 | ||
5 5 5 | ||
6 6 6 | ||
7 7 7 | ||
8 8 8 | ||
9 9 9 | ||
10 10 10 | ||
SELECT COUNT(*) FROM t1; | ||
COUNT(*) | ||
100 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; | ||
# crash_during_index_creation_partition | ||
flush logs; | ||
SET SESSION debug="+d,myrocks_simulate_index_create_rollback"; | ||
# expected assertion failure from sql layer here for alter rollback | ||
call mtr.add_suppression("Assertion `0' failed."); | ||
call mtr.add_suppression("Attempting backtrace. You can use the following information to find out"); | ||
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; | ||
ERROR HY000: Lost connection to MySQL server during query | ||
SET SESSION debug="-d,myrocks_simulate_index_create_rollback"; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`i` int(11) NOT NULL DEFAULT '0', | ||
`j` int(11) DEFAULT NULL, | ||
`k` int(11) DEFAULT NULL, | ||
PRIMARY KEY (`i`), | ||
KEY `j` (`j`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
/*!50100 PARTITION BY KEY (i) | ||
PARTITIONS 4 */ | ||
ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; | ||
SHOW CREATE TABLE t1; | ||
Table Create Table | ||
t1 CREATE TABLE `t1` ( | ||
`i` int(11) NOT NULL DEFAULT '0', | ||
`j` int(11) DEFAULT NULL, | ||
`k` int(11) DEFAULT NULL, | ||
PRIMARY KEY (`i`), | ||
KEY `j` (`j`), | ||
KEY `kij` (`i`,`j`) | ||
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 | ||
/*!50100 PARTITION BY KEY (i) | ||
PARTITIONS 4 */ | ||
SELECT COUNT(*) FROM t1; | ||
COUNT(*) | ||
100 | ||
DROP TABLE t1; | ||
CREATE TABLE t1 (a INT, b TEXT); | ||
ALTER TABLE t1 ADD KEY kb(b(10)); | ||
ERROR HY000: Unsupported collation on string indexed column test.t1.b Use binary collation (binary, latin1_bin, utf8_bin). | ||
ALTER TABLE t1 ADD PRIMARY KEY(a); | ||
DROP TABLE t1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
[mysql] | ||
no-defaults | ||
|
||
[mysqld.1] | ||
rocksdb_strict_collation_check=1 |
Oops, something went wrong.