Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
MariaRocks port: fix rocksdb.collation, rocksdb.collation_exception
- port Regex_list_handler from facebook/mysql-5.6/sql/handler.cc
  put it into a separate file in storage/rocksdb directory

- Adjust the build process so that the main library is build with
  Regex_list_handler (which has dependencies on the server),
  while RocksDB tools are built without it.

- Un-comment @@rdb_collation_exceptions handling in ha_rocksdb.cc
- Also adjust rocksdb_set_collation_exception_list() to free the
  old variable value and alloc the new one.
  • Loading branch information
spetrunia committed Dec 4, 2016
1 parent 34b66fc commit 8e2cfde
Show file tree
Hide file tree
Showing 6 changed files with 193 additions and 21 deletions.
19 changes: 12 additions & 7 deletions storage/rocksdb/CMakeLists.txt
Expand Up @@ -43,9 +43,9 @@ set_source_files_properties(${ROCKSDB_LIB_SOURCES} PROPERTIES COMPILE_FLAGS -frt
set_source_files_properties(event_listener.cc PROPERTIES COMPILE_FLAGS -frtti)
set_source_files_properties(rdb_cf_options.cc PROPERTIES COMPILE_FLAGS -frtti)


SET(ROCKSDB_SOURCES
ha_rocksdb.cc ha_rocksdb.h ha_rocksdb_proto.h
ADD_CONVENIENCE_LIBRARY(
ROCKSDB_AUX_LIB
ha_rocksdb_proto.h
logger.h
rdb_comparator.h
rdb_datadic.cc rdb_datadic.h
Expand All @@ -64,6 +64,11 @@ SET(ROCKSDB_SOURCES
${ROCKSDB_LIB_SOURCES}
)

SET(ROCKSDB_SOURCES
rdb_mariadb_server_port.cc rdb_mariadb_server_port.h
ha_rocksdb.cc ha_rocksdb.h
)

IF(WITH_FB_TSAN)
SET(PIC_EXT "_pic")
ELSE()
Expand Down Expand Up @@ -102,7 +107,7 @@ IF (NOT "$ENV{WITH_ZSTD}" STREQUAL "")
ADD_DEFINITIONS(-DZSTD)
ENDIF()

SET(rocksdb_static_libs ${rocksdb_static_libs} ${ZLIB_LIBRARY} "-lrt")
SET(rocksdb_static_libs ROCKSDB_AUX_LIB ${rocksdb_static_libs} ${ZLIB_LIBRARY} "-lrt")

MYSQL_ADD_PLUGIN(rocksdb_se ${ROCKSDB_SOURCES} STORAGE_ENGINE DEFAULT STATIC_ONLY
LINK_LIBRARIES ${rocksdb_static_libs}
Expand All @@ -121,11 +126,11 @@ IF (WITH_ROCKSDB_SE_STORAGE_ENGINE)
)
set_source_files_properties(${ROCKSDB_TOOL_SOURCES} PROPERTIES COMPILE_FLAGS -frtti)
MYSQL_ADD_EXECUTABLE(sst_dump ${CMAKE_SOURCE_DIR}/storage/rocksdb/rocksdb/tools/sst_dump.cc ${ROCKSDB_TOOL_SOURCES})
TARGET_LINK_LIBRARIES(sst_dump rocksdb_se)
TARGET_LINK_LIBRARIES(sst_dump ${rocksdb_static_libs})

MYSQL_ADD_EXECUTABLE(ldb ${CMAKE_SOURCE_DIR}/storage/rocksdb/rocksdb/tools/ldb.cc ${ROCKSDB_TOOL_SOURCES})
TARGET_LINK_LIBRARIES(ldb rocksdb_se)
TARGET_LINK_LIBRARIES(ldb ${rocksdb_static_libs})

MYSQL_ADD_EXECUTABLE(mysql_ldb ${CMAKE_SOURCE_DIR}/storage/rocksdb/tools/mysql_ldb.cc ${ROCKSDB_TOOL_SOURCES})
TARGET_LINK_LIBRARIES(mysql_ldb rocksdb_se)
TARGET_LINK_LIBRARIES(mysql_ldb ${rocksdb_static_libs})
ENDIF()
19 changes: 6 additions & 13 deletions storage/rocksdb/ha_rocksdb.cc
Expand Up @@ -62,6 +62,7 @@
#include "./rdb_index_merge.h"
#include "./rdb_mutex_wrapper.h"
#include "./rdb_threads.h"
#include "./rdb_mariadb_server_port.h"

#ifdef TARGET_OS_LINUX
extern my_bool cachedev_enabled;
Expand Down Expand Up @@ -155,11 +156,9 @@ Rdb_binlog_manager binlog_manager;

static Rdb_background_thread rdb_bg_thread;

#ifdef MARIAROCKS_NOT_YET
// List of table names (using regex) that are exceptions to the strict
// collation check requirement.
Regex_list_handler *rdb_collation_exceptions;
#endif

static const char* const ERRSTR_ROLLBACK_ONLY
= "This transaction was rolled back and cannot be "
Expand Down Expand Up @@ -3345,13 +3344,11 @@ static int rocksdb_init_func(void *p)
mysql_mutex_init(rdb_mem_cmp_space_mutex_key, &rdb_mem_cmp_space_mutex,
MY_MUTEX_INIT_FAST);

#ifdef MARIAROCKS_NOT_YET
#if defined(HAVE_PSI_INTERFACE)
rdb_collation_exceptions = new Regex_list_handler(
key_rwlock_collation_exception_list);
#else
rdb_collation_exceptions = new Regex_list_handler();
#endif
#endif

mysql_mutex_init(rdb_sysvars_psi_mutex_key, &rdb_sysvars_mutex,
Expand Down Expand Up @@ -3731,9 +3728,9 @@ static int rocksdb_done_func(void *p)
mysql_mutex_destroy(&rdb_open_tables.m_mutex);
mysql_mutex_destroy(&rdb_sysvars_mutex);

#ifdef MARIAROCKS_NOT_YET

delete rdb_collation_exceptions;
#endif

mysql_mutex_destroy(&rdb_collation_data_mutex);
mysql_mutex_destroy(&rdb_mem_cmp_space_mutex);

Expand Down Expand Up @@ -5126,11 +5123,7 @@ int ha_rocksdb::create_cfs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg,
{
if (!rdb_is_index_collation_supported(
table_arg->key_info[i].key_part[part].field) &&
#ifdef MARIAROCKS_NOT_YET
!rdb_collation_exceptions->matches(tablename_sys))
#else
true)
#endif
{
std::string collation_err;
for (auto coll : RDB_INDEX_COLLATIONS)
Expand Down Expand Up @@ -10715,15 +10708,13 @@ rocksdb_set_rate_limiter_bytes_per_sec(

void rdb_set_collation_exception_list(const char *exception_list)
{
#ifdef MARIAROCKS_NOT_YET
DBUG_ASSERT(rdb_collation_exceptions != nullptr);

if (!rdb_collation_exceptions->set_patterns(exception_list))
{
my_core::warn_about_bad_patterns(rdb_collation_exceptions,
"strict_collation_exceptions");
}
#endif
}

void
Expand All @@ -10736,7 +10727,9 @@ rocksdb_set_collation_exception_list(THD* thd,

rdb_set_collation_exception_list(val);

*static_cast<const char**>(var_ptr) = val;
const char *val_copy= my_strdup(val, MYF(0));
my_free(*static_cast<char**>(var_ptr));
*static_cast<const char**>(var_ptr) = val_copy;
}

void
Expand Down
1 change: 1 addition & 0 deletions storage/rocksdb/mysql-test/rocksdb/r/collation.result
Expand Up @@ -116,6 +116,7 @@ CREATE TABLE b (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rock
CREATE TABLE c (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8;
ERROR HY000: Unsupported collation on string indexed column test.c.value Use binary collation (binary, latin1_bin, utf8_bin).
DROP TABLE a, b;
call mtr.add_suppression("Invalid pattern in strict_collation_exceptions:");
SET GLOBAL rocksdb_strict_collation_exceptions="abc\\";
Invalid pattern in strict_collation_exceptions: abc\
CREATE TABLE abc (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8;
Expand Down
5 changes: 4 additions & 1 deletion storage/rocksdb/mysql-test/rocksdb/t/collation.test
@@ -1,5 +1,7 @@
--source include/have_rocksdb.inc
--source include/have_fullregex.inc
# MariaDB doesn't have server variables to check for GCC version, so the
# following check is commented out:
# --source include/have_fullregex.inc

SET @start_global_value = @@global.ROCKSDB_STRICT_COLLATION_EXCEPTIONS;

Expand Down Expand Up @@ -165,6 +167,7 @@ CREATE TABLE b (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rock
CREATE TABLE c (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8;
DROP TABLE a, b;

call mtr.add_suppression("Invalid pattern in strict_collation_exceptions:");
# test invalid regex (trailing escape)
--exec echo "" >$MYSQLTEST_VARDIR/log/mysqld.1.err
SET GLOBAL rocksdb_strict_collation_exceptions="abc\\";
Expand Down
97 changes: 97 additions & 0 deletions storage/rocksdb/rdb_mariadb_server_port.cc
@@ -0,0 +1,97 @@
#include <my_config.h>


/* MySQL includes */
#include "./debug_sync.h"
#include "./my_bit.h"
#include "./my_stacktrace.h"
#include "./sql_table.h"
#include "./my_global.h"
#include "./log.h"
#include <mysys_err.h>
#include <mysql/psi/mysql_table.h>
#ifdef MARIAROCKS_NOT_YET
#include <mysql/thread_pool_priv.h>
#endif

#include <string>

/* MyRocks includes */
#include "./rdb_threads.h"

#include "rdb_mariadb_server_port.h"

void warn_about_bad_patterns(const Regex_list_handler* regex_list_handler,
const char *name)
{
// There was some invalid regular expression data in the patterns supplied

// NO_LINT_DEBUG
sql_print_warning("Invalid pattern in %s: %s", name,
regex_list_handler->bad_pattern().c_str());
}


/*
Set the patterns string. If there are invalid regex patterns they will
be stored in m_bad_patterns and the result will be false, otherwise the
result will be true.
*/
bool Regex_list_handler::set_patterns(const std::string& pattern_str)
{
bool pattern_valid= true;

// Create a normalized version of the pattern string with all delimiters
// replaced by the '|' character
std::string norm_pattern= pattern_str;
std::replace(norm_pattern.begin(), norm_pattern.end(), m_delimiter, '|');

// Make sure no one else is accessing the list while we are changing it.
mysql_rwlock_wrlock(&m_rwlock);

// Clear out any old error information
m_bad_pattern_str.clear();

try
{
// Replace all delimiters with the '|' operator and create the regex
// Note that this means the delimiter can not be part of a regular
// expression. This is currently not a problem as we are using the comma
// character as a delimiter and commas are not valid in table names.
const std::regex* pattern= new std::regex(norm_pattern);

// Free any existing regex information and setup the new one
delete m_pattern;
m_pattern= pattern;
}
catch (const std::regex_error& e)
{
// This pattern is invalid.
pattern_valid= false;

// Put the bad pattern into a member variable so it can be retrieved later.
m_bad_pattern_str= pattern_str;
}

// Release the lock
mysql_rwlock_unlock(&m_rwlock);

return pattern_valid;
}

bool Regex_list_handler::matches(const std::string& str) const
{
DBUG_ASSERT(m_pattern != nullptr);

// Make sure no one else changes the list while we are accessing it.
mysql_rwlock_rdlock(&m_rwlock);

// See if the table name matches the regex we have created
bool found= std::regex_match(str, *m_pattern);

// Release the lock
mysql_rwlock_unlock(&m_rwlock);

return found;
}

73 changes: 73 additions & 0 deletions storage/rocksdb/rdb_mariadb_server_port.h
@@ -0,0 +1,73 @@
/*
A temporary header to resolve WebScaleSQL vs MariaDB differences
when porting MyRocks to MariaDB.
*/
#ifndef RDB_MARIADB_SERVER_PORT_H
#define RDB_MARIADB_SERVER_PORT_H

#include "my_global.h" /* ulonglong */
#include "atomic_stat.h"
#include "my_pthread.h"
#include <mysql/psi/mysql_table.h>
#include <mysql/psi/mysql_thread.h>

/*
Code that is on SQL layer in facebook/mysql-5.6,
but is part of the storage engine in MariaRocks
*/
#include <regex>

class Regex_list_handler
{
private:
#if defined(HAVE_PSI_INTERFACE)
const PSI_rwlock_key& m_key;
#endif

char m_delimiter;
std::string m_bad_pattern_str;
const std::regex* m_pattern;

mutable mysql_rwlock_t m_rwlock;

Regex_list_handler(const Regex_list_handler& other)= delete;
Regex_list_handler& operator=(const Regex_list_handler& other)= delete;

public:
#if defined(HAVE_PSI_INTERFACE)
Regex_list_handler(const PSI_rwlock_key& key,
char delimiter= ',') :
m_key(key),
#else
Regex_list_handler(char delimiter= ',') :
#endif
m_delimiter(delimiter),
m_bad_pattern_str(""),
m_pattern(nullptr)
{
mysql_rwlock_init(key, &m_rwlock);
}

~Regex_list_handler()
{
mysql_rwlock_destroy(&m_rwlock);
delete m_pattern;
}

// Set the list of patterns
bool set_patterns(const std::string& patterns);

// See if a string matches at least one pattern
bool matches(const std::string& str) const;

// See the list of bad patterns
const std::string& bad_pattern() const
{
return m_bad_pattern_str;
}
};

void warn_about_bad_patterns(const Regex_list_handler* regex_list_handler,
const char *name);

#endif

0 comments on commit 8e2cfde

Please sign in to comment.