Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Embed RocksDB in MySQL repo
Summary: Use a git submodule to build RocksDB within MySQL.

@update-submodule: rocksdb

Test Plan:
Cleanly build MySQL with no environment variables or external linking.
Copy mysqld to testdb and verify it works and supports RocksDB storage engine.
Run mysqltest.sh.

Reviewers: steaphan, MarkCallaghan, yoshinorim

Reviewed By: yoshinorim

Subscribers: sdong, igor, jtolmer, hermanlee4, maykov

Differential Revision: https://reviews.facebook.net/D31059
Differential Revision: https://reviews.facebook.net/D49503
  • Loading branch information
Jonah Cohen authored and jtolmer committed Jan 5, 2016
1 parent d8483ee commit 7289b2d
Show file tree
Hide file tree
Showing 28 changed files with 291 additions and 28 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
@@ -0,0 +1,3 @@
[submodule "rocksdb"]
path = rocksdb
url = https://github.com/facebook/rocksdb.git
3 changes: 3 additions & 0 deletions CMakeLists.txt
Expand Up @@ -168,6 +168,7 @@ INCLUDE(character_sets)
INCLUDE(cpu_info)
INCLUDE(zlib)
INCLUDE(libevent)
INCLUDE(rocksdb)
INCLUDE(ssl)
INCLUDE(readline)
INCLUDE(mysql_version)
Expand Down Expand Up @@ -397,6 +398,8 @@ MYSQL_CHECK_SSL()
MYSQL_CHECK_READLINE()
# Add libevent
MYSQL_CHECK_LIBEVENT()
# Add rocksdb
MYSQL_CHECK_ROCKSDB()

#
# Setup maintainer mode options by the end. Platform checks are
Expand Down
96 changes: 96 additions & 0 deletions cmake/rocksdb.cmake
@@ -0,0 +1,96 @@
# Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

MACRO (MYSQL_USE_BUNDLED_ROCKSDB)
IF (NOT EXISTS "${CMAKE_SOURCE_DIR}/rocksdb/Makefile")
MESSAGE(SEND_ERROR "Missing Makefile in rocksdb directory. Try \"git submodule update\".")
ENDIF()
SET(ROCKSDB_LIBRARY ${CMAKE_SOURCE_DIR}/rocksdb)
SET(ROCKSDB_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/rocksdb/include)
SET(ROCKSDB_FOUND TRUE)
SET(WITH_ROCKSDB "bundled" CACHE STRING "Use bundled rocksdb")
GET_TARGET_PROPERTY(src rocksdb SOURCES)
FOREACH(file ${src})
SET(ROCKSDB_SOURCES ${ROCKSDB_SOURCES} ${CMAKE_SOURCE_DIR}/rocksdb/${file})
ENDFOREACH()
EXECUTE_PROCESS(
COMMAND env -u CXXFLAGS make release
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/rocksdb
RESULT_VARIABLE make_result
)
ENDMACRO()

# MYSQL_CHECK_ROCKSDB
#
# Provides the following configure options:
# WITH_ROCKSDB
# If this is set to "system", search for system rocksdb
# Otherwise, we use bundled rocksdb
# if system rocksdb is not found, use bundled copy
# ROCKSDB_LIBRARY, ROCKSDB_INCLUDE_DIR and ROCKSDB_SOURCES
# are set after this macro has run

MACRO (MYSQL_CHECK_ROCKSDB)

IF (NOT WITH_ROCKSDB)
SET(WITH_ROCKSDB "bundled" CACHE STRING "By default use bundled rocksdb on this platform")
ENDIF()

IF (WITH_ROCKSDB STREQUAL "bundled")
MYSQL_USE_BUNDLED_ROCKSDB()
ELSEIF (WITH_ROCKSDB STREQUAL "system" OR WITH_ROCKSDB STREQUAL "yes")
SET(ROCKSDB_FIND_QUIETLY TRUE)

IF (NOT ROCKSDB_INCLUDE_PATH)
set(ROCKSDB_INCLUDE_PATH /usr/local/include /opt/local/include)
ENDIF()

find_path(ROCKSDB_INCLUDE_DIR db.h PATHS ${ROCKSDB_INCLUDE_PATH})

IF (NOT ROCKSDB_INCLUDE_DIR)
MESSAGE(SEND_ERROR "Cannot find appropriate db.h in /usr/local/include or /opt/local/include. Use bundled rocksdb")
ENDIF()

IF (NOT ROCKSDB_LIB_PATHS)
set(ROCKSDB_LIB_PATHS /usr/local/lib /opt/local/lib)
ENDIF()

find_library(ROCKSDB_LIB rocksdb PATHS ${ROCKSDB_LIB_PATHS})

IF (NOT ROCKSDB_LIB)
MESSAGE(SEND_ERROR "Cannot find appropriate rocksdb lib in /usr/local/lib or /opt/local/lib. Use bundled rocksdb")
ENDIF()

IF (ROCKSDB_LIB AND ROCKSDB_INCLUDE_DIR)
set(ROCKSDB_FOUND TRUE)
set(ROCKSDB_LIBS ${ROCKSDB_LIB})
ELSE()
set(ROCKSDB_FOUND FALSE)
ENDIF()

IF(ROCKSDB_FOUND)
SET(ROCKSDB_SOURCES "")
SET(ROCKSDB_LIBRARY ${ROCKSDB_LIBS})
SET(ROCKSDB_INCLUDE_DIRS ${ROCKSDB_INCLUDE_DIR})
SET(ROCKSDB_DEFINES "-DHAVE_ROCKSDB")
ELSE()
IF(WITH_ROCKSDB STREQUAL "system")
MESSAGE(SEND_ERROR "Cannot find appropriate system libraries for rocksdb. Use bundled rocksdb")
ENDIF()
MYSQL_USE_BUNDLED_ROCKSDB()
ENDIF()

ENDIF()
ENDMACRO()
1 change: 1 addition & 0 deletions mysql-test/r/1st.result
Expand Up @@ -4,6 +4,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
show tables in mysql;
Tables_in_mysql
Expand Down
2 changes: 2 additions & 0 deletions mysql-test/r/drop.result
Expand Up @@ -51,6 +51,7 @@ mtr
mysql
mysqltest
performance_schema
rocksdb
test
flush tables with read lock;
drop database mysqltest;
Expand All @@ -63,6 +64,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
drop database mysqltest;
ERROR HY000: Can't drop database 'mysqltest'; database doesn't exist
Expand Down
4 changes: 4 additions & 0 deletions mysql-test/r/information_schema.result
Expand Up @@ -20,13 +20,15 @@ CATALOG_NAME SCHEMA_NAME DEFAULT_CHARACTER_SET_NAME DEFAULT_COLLATION_NAME SQL_P
def mtr latin1 latin1_swedish_ci NULL
def mysql latin1 latin1_swedish_ci NULL
def performance_schema utf8 utf8_general_ci NULL
def rocksdb latin1 latin1_swedish_ci NULL
def test latin1 latin1_swedish_ci NULL
select schema_name from information_schema.schemata;
schema_name
information_schema
mtr
mysql
performance_schema
rocksdb
test
show databases like 't%';
Database (t%)
Expand All @@ -37,6 +39,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
show databases where `database` = 't%';
Database
Expand Down Expand Up @@ -391,6 +394,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
explain select * from v0;
id select_type table type possible_keys key key_len ref rows Extra
Expand Down
2 changes: 2 additions & 0 deletions mysql-test/r/insert_select.result
Expand Up @@ -61,6 +61,8 @@ WHERE numeropost=9 ORDER BY numreponse ASC;
show variables like '%bulk%';
Variable_name Value
bulk_insert_buffer_size 8388608
rocksdb_bulk_load OFF
rocksdb_bulk_load_size 1000
INSERT INTO t1 (numeropost,icone,contenu,pseudo,date,signature,ip)
SELECT 1718,icone,contenu,pseudo,date,signature,ip FROM t2
WHERE numeropost=9 ORDER BY numreponse ASC;
Expand Down
1 change: 1 addition & 0 deletions mysql-test/r/lower_case_table_names_usage.result
Expand Up @@ -20,6 +20,7 @@ Bar
mtr
mysql
performance_schema
rocksdb
test
#
# Ensure this table is created with ascii_bin
Expand Down
2 changes: 2 additions & 0 deletions mysql-test/r/mysqlslap.result
Expand Up @@ -236,6 +236,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
# 'bug58090' database should be present.
SHOW DATABASES;
Expand All @@ -245,6 +246,7 @@ bug58090
mtr
mysql
performance_schema
rocksdb
test
DROP DATABASE bug58090;
#
Expand Down
1 change: 1 addition & 0 deletions mysql-test/r/ps_1general.result
Expand Up @@ -264,6 +264,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
prepare stmt4 from ' show tables from test like ''t2%'' ';
execute stmt4;
Expand Down
26 changes: 15 additions & 11 deletions mysql-test/r/rocksdb.result
Expand Up @@ -866,6 +866,10 @@ rocksdb_use_fsync OFF
rocksdb_wal_size_limit_mb 0
rocksdb_wal_ttl_seconds 0
rocksdb_whole_key_filtering ON
rocksdb_write_disable_wal OFF
rocksdb_write_ignore_missing_column_families OFF
rocksdb_write_sync off
rocksdb_write_timeout_hint_us 0
create table t47 (pk int primary key, col1 varchar(12)) engine=rocksdb;
insert into t47 values (1, 'row1');
insert into t47 values (2, 'row2');
Expand Down Expand Up @@ -1063,7 +1067,7 @@ DROP TABLE t1;
#
CREATE TABLE t1 (pk INT PRIMARY KEY, i INT NOT NULL, KEY(i)) ENGINE=RocksDB;
ALTER TABLE t1 DROP COLUMN `pk`;
ERROR HY000: Got error 193 'Table must have a PRIMARY KEY' from ROCKSDB
ERROR HY000: Got error 194 'Table must have a PRIMARY KEY' from ROCKSDB
DROP TABLE t1;
#
# MDEV-4324: RocksDB: Valgrind "Use of uninitialised value" warnings on inserting value into varchar field
Expand Down Expand Up @@ -1147,18 +1151,14 @@ SELECT * FROM t1 ORDER BY pk LIMIT 9;
pk
1
2
3
4
5
6
7
8
affected rows: 8
affected rows: 2
DELETE FROM t1 ORDER BY pk LIMIT 9;
affected rows: 8
affected rows: 2
SELECT * FROM t1 ORDER BY pk LIMIT 9;
pk
affected rows: 0
1
2
affected rows: 2
DROP TABLE t1,t2;
#
# MDEV-4374: RocksDB: Valgrind warnings 'Use of uninitialised value' on
Expand Down Expand Up @@ -1211,7 +1211,7 @@ insert into t1 select (@a:=@a+1), 1234 from information_schema.session_variables
set @tmp1= @@rocksdb_max_row_locks;
set rocksdb_max_row_locks= 20;
update t1 set a=a+10;
ERROR HY000: Got error 194 'Number of locks held reached @@rocksdb_max_row_locks' from ROCKSDB
ERROR HY000: Got error 195 'Number of locks held reached @@rocksdb_max_row_locks' from ROCKSDB
DROP TABLE t1;
#
# Test AUTO_INCREMENT behavior problem,
Expand Down Expand Up @@ -1281,6 +1281,10 @@ Collation latin1_swedish_ci
Checksum NULL
Create_options
Comment
Compression NULL
Compression_level 0
Key_block_size 0
Compact_metadata 0
drop table t1;
#
# Fix Issue #4: Crash when using pseudo-unique keys
Expand Down
16 changes: 8 additions & 8 deletions mysql-test/r/rocksdb_cf_options.result
Expand Up @@ -11,22 +11,22 @@ insert into t3 values (2);
Default options for all column families:

Options.db_write_buffer_size: 0
Options for column family "default":
Options for column family [default]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 12582912
Options.target_file_size_base: 1048576
Options.max_bytes_for_level_multiplier: 10
Options for column family "cf1":
Options for column family [cf1]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 12582912
Options.target_file_size_base: 1048576
Options.max_bytes_for_level_multiplier: 10
Options for column family "cf2":
Options for column family [cf2]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 12582912
Options.target_file_size_base: 1048576
Options.max_bytes_for_level_multiplier: 10
Options for column family "cf3":
Options for column family [cf3]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 12582912
Options.target_file_size_base: 1048576
Expand All @@ -35,22 +35,22 @@ Options.max_bytes_for_level_multiplier: 10
Individualized options for column families:

Options.db_write_buffer_size: 0
Options for column family "default":
Options for column family [default]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 12582912
Options.target_file_size_base: 1048576
Options.max_bytes_for_level_multiplier: 10
Options for column family "cf1":
Options for column family [cf1]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 8388608
Options.target_file_size_base: 2097152
Options.max_bytes_for_level_multiplier: 10
Options for column family "cf2":
Options for column family [cf2]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 16777216
Options.target_file_size_base: 1048576
Options.max_bytes_for_level_multiplier: 8
Options for column family "cf3":
Options for column family [cf3]:
Options.db_write_buffer_size: 0
Options.write_buffer_size: 12582912
Options.target_file_size_base: 1048576
Expand Down
1 change: 1 addition & 0 deletions mysql-test/r/schema.result
Expand Up @@ -10,6 +10,7 @@ foo
mtr
mysql
performance_schema
rocksdb
test
drop schema foo;
#
Expand Down
1 change: 1 addition & 0 deletions mysql-test/r/show_check.result
Expand Up @@ -144,6 +144,7 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
show databases like "test%";
Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr
Expand Down
1 change: 1 addition & 0 deletions mysql-test/suite/binlog/r/binlog_database.result
Expand Up @@ -257,4 +257,5 @@ information_schema
mtr
mysql
performance_schema
rocksdb
test
3 changes: 3 additions & 0 deletions mysql-test/suite/binlog/r/binlog_row_binlog.result
Expand Up @@ -791,6 +791,7 @@ c1
SHOW SESSION VARIABLES LIKE "%_checks";
Variable_name Value
foreign_key_checks ON
rocksdb_paranoid_checks ON
unique_checks ON

SET @@SESSION.foreign_key_checks= OFF;
Expand All @@ -809,6 +810,7 @@ c1
SHOW SESSION VARIABLES LIKE "%_checks";
Variable_name Value
foreign_key_checks OFF
rocksdb_paranoid_checks ON
unique_checks OFF
# INSERT INTO t1 VALUES(2)
# foreign_key_checks=1 and unique_checks=1
Expand All @@ -827,5 +829,6 @@ c1
SHOW SESSION VARIABLES LIKE "%_checks";
Variable_name Value
foreign_key_checks OFF
rocksdb_paranoid_checks ON
unique_checks OFF
DROP TABLE t1;

0 comments on commit 7289b2d

Please sign in to comment.