Merge MyRocks merge tree into bb-10.2-mariarocks, unfinished.

It compiles on Linux but fails a lot of tests still

sql/share/errmsg-utf8.txt

@@ -7455,3 +7455,70 @@ ER_SLAVE_SAME_ID
         eng "A slave with the same server_uuid/server_id as this slave has connected to the master"
 ER_FLASHBACK_NOT_SUPPORTED
         eng "Flashback does not support %s %s"
+
+# MARIAROCKS-TODO: Should we add RocksDB error messages here or use some other
+# solution?
+ER_KEYS_OUT_OF_ORDER
+  eng "Keys are out order during bulk load"
+
+ER_OVERLAPPING_KEYS
+  eng "Bulk load rows overlap existing rows"
+
+ER_REQUIRE_ROW_BINLOG_FORMAT
+  eng "Can't execute updates on master with binlog_format != ROW."
+
+ER_ISOLATION_MODE_NOT_SUPPORTED
+  eng "MyRocks supports only READ COMMITTED and REPEATABLE READ isolation levels. Please change from current isolation level %s"
+
+ER_ON_DUPLICATE_DISABLED
+  eng "When unique checking is disabled in MyRocks, INSERT,UPDATE,LOAD statements with clauses that update or replace the key (i.e. INSERT ON DUPLICATE KEY UPDATE, REPLACE) are not allowed. Query: %s"
+
+ER_UPDATES_WITH_CONSISTENT_SNAPSHOT
+  eng "Can't execute updates when you started a transaction with START TRANSACTION WITH CONSISTENT [ROCKSDB] SNAPSHOT."
+
+ER_ROLLBACK_ONLY
+  eng "This transaction was rolled back and cannot be committed. Only supported operation is to roll it back, so all pending changes will be discarded. Please restart another transaction."
+
+ER_ROLLBACK_TO_SAVEPOINT
+  eng "MyRocks currently does not support ROLLBACK TO SAVEPOINT if modifying rows."
+
+ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT
+  eng "Only REPEATABLE READ isolation level is supported for START TRANSACTION WITH CONSISTENT SNAPSHOT in RocksDB Storage Engine."
+
+ER_UNSUPPORTED_COLLATION
+  eng "Unsupported collation on string indexed column %s.%s Use binary collation (%s)."
+
+ER_METADATA_INCONSISTENCY
+  eng "Table '%s' does not exist, but metadata information exists inside MyRocks. This is a sign of data inconsistency. Please check if '%s.frm' exists, and try to restore it if it does not exist."
+
+ER_KEY_CREATE_DURING_ALTER
+  eng "MyRocks failed creating new key definitions during alter."
+
+ER_SK_POPULATE_DURING_ALTER
+  eng "MyRocks failed populating secondary key during alter."
+
+ER_CF_DIFFERENT
+  eng "Column family ('%s') flag (%d) is different from an existing flag (%d). Assign a new CF flag, or do not change existing CF flag."
+
+ER_RDB_STATUS_GENERAL
+  eng "Status error %d received from RocksDB: %s"
+
+ER_RDB_STATUS_MSG
+  eng "%s, Status error %d received from RocksDB: %s"
+
+ER_NET_OK_PACKET_TOO_LARGE
+  eng "OK packet too large"
+
+ER_RDB_TTL_UNSUPPORTED
+  eng "TTL support is currently disabled when table has secondary indexes or hidden PK."
+
+ER_RDB_TTL_COL_FORMAT
+  eng "TTL column (%s) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration."
+
+ER_RDB_TTL_DURATION_FORMAT
+  eng "TTL duration (%s) in MyRocks must be an unsigned non-null 64-bit integer."
+
+ER_PER_INDEX_CF_DEPRECATED
+  eng "The per-index column family option has been deprecated"
+
+

storage/rocksdb/CMakeLists.txt

@@ -16,6 +16,11 @@ IF (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/Makefile")
   SKIP_ROCKSDB_PLUGIN("Missing Makefile in rocksdb directory. Try \"git submodule update\".")
 ENDIF()
 
+CHECK_FUNCTION_EXISTS(sched_getcpu  HAVE_SCHED_GETCPU)
+IF(HAVE_SCHED_GETCPU)
+  ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1 -DROCKSDB_SCHED_GETCPU_PRESENT)
+ENDIF()
+
 # We've had our builders hang during the build process. This prevents MariaRocks
 # to be built on 32 bit intel OS kernels.
 IF(CMAKE_SYSTEM_PROCESSOR MATCHES "i[36]86")
@@ -106,6 +111,19 @@ IF(HAVE_FALLOCATE)
   ADD_DEFINITIONS(-DROCKSDB_FALLOCATE_PRESENT)
 ENDIF()
 
+
+CHECK_CXX_SOURCE_COMPILES("
+#if defined(_MSC_VER) && !defined(__thread)
+#define __thread __declspec(thread)
+#endif
+int main() {
+  static __thread int tls;
+}
+" HAVE_THREAD_LOCAL)
+if(HAVE_THREAD_LOCAL)
+  ADD_DEFINITIONS(-DROCKSDB_SUPPORT_THREAD_LOCAL)
+endif()
+
 INCLUDE(build_rocksdb.cmake)
 
 ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib
@@ -120,13 +138,16 @@ ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib
   rdb_perf_context.h
   rdb_sst_info.cc
   rdb_sst_info.h
+  rdb_io_watchdog.cc rdb_io_watchdog.h
   rdb_buff.h
   rdb_mariadb_port.h
 )
 
 ADD_DEPENDENCIES(rocksdb_aux_lib GenError)
 
-TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY})
+# MARIAROCKS-TODO: how to properly depend on -lrt ?
+TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY} -lrt)
+
 TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib)
 
 IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
@@ -161,6 +182,8 @@ ENDIF()
 #  ADD_SUBDIRECTORY(unittest)
 #ENDIF()
 
+SET(rocksdb_static_libs ${rocksdb_static_libs} "-lrt")
+
 ADD_LIBRARY(rocksdb_tools STATIC
    rocksdb/tools/ldb_tool.cc
    rocksdb/tools/ldb_cmd.cc

storage/rocksdb/README

@@ -1,12 +1,20 @@
 == Summary ==
-This directory contains RocksDB-based Storage Engine (RDBSE) for MySQL = "MyRocks". 
+This directory contains RocksDB-based Storage Engine (RDBSE) for MySQL,
+also known as "MyRocks".
 
 == Resources ==
-See https://github.com/facebook/mysql-5.6/wiki/Getting-Started-with-MyRocks
-Facebook group: https://www.facebook.com/groups/mysqlonrocksdb/
+https://github.com/facebook/mysql-5.6/wiki/Getting-Started-with-MyRocks
+https://www.facebook.com/groups/MyRocks/
 
 == Coding Conventions ==
-The baseline for MyRocks coding conventions is the MySQL set, available at
+The baseline for MyRocks coding conventions for the code in storage/rocksdb/
+is based on the default clang format with a few minor changes.  The file
+storage/rocksdb/.clang-format describes conventions and can be integrated
+with Vim or Emacs as described here:
+http://releases.llvm.org/3.6.0/tools/clang/docs/ClangFormat.html#vim-integration
+
+All code outside of storage/rocksdb/ should conform to the MySQL coding
+conventions:
 http://dev.mysql.com/doc/internals/en/coding-guidelines.html.
 
 Several refinements:
@@ -34,5 +42,9 @@ Several refinements:
      "_vect" for a std::vector etc.
 
 == Running Tests ==
-To run tests from rocksdb, rocksd_rpl or rocksdb_sys_vars packages, use the following parameters:
---mysqld=--default-storage-engine=rocksdb  --mysqld=--skip-innodb --mysqld=--default-tmp-storage-engine=MyISAM --mysqld=--rocksdb
+To run tests from rocksdb, rocksdb_rpl or other rocksdb_* packages, use the
+following parameters:
+  --default-storage-engine=rocksdb
+  --skip-innodb
+  --default-tmp-storage-engine=MyISAM
+  --rocksdb

storage/rocksdb/build_rocksdb.cmake

@@ -178,6 +178,7 @@ set(ROCKSDB_SOURCES
         db/internal_stats.cc
         db/log_reader.cc
         db/log_writer.cc
+        db/malloc_stats.cc
         db/managed_iterator.cc
         db/memtable.cc
         db/memtable_list.cc
@@ -200,13 +201,14 @@ set(ROCKSDB_SOURCES
         env/env.cc
         env/env_chroot.cc
         env/env_hdfs.cc
-        env/memenv.cc
+        env/mock_env.cc
+        memtable/alloc_tracker.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
-        memtable/memtable_allocator.cc
         memtable/skiplistrep.cc
         memtable/vectorrep.cc
+        memtable/write_buffer_manager.cc
         monitoring/histogram.cc
         monitoring/histogram_windowing.cc
         monitoring/instrumented_mutex.cc
@@ -216,6 +218,7 @@ set(ROCKSDB_SOURCES
         monitoring/statistics.cc
         monitoring/thread_status_impl.cc
         monitoring/thread_status_updater.cc
+        monitoring/thread_status_updater_debug.cc
         monitoring/thread_status_util.cc
         monitoring/thread_status_util_debug.cc
         options/cf_options.cc
@@ -245,6 +248,7 @@ set(ROCKSDB_SOURCES
         table/iterator.cc
         table/merging_iterator.cc
         table/meta_blocks.cc
+        table/mock_table.cc
         table/partitioned_filter_block.cc
         table/persistent_cache_helper.cc
         table/plain_table_builder.cc
@@ -293,12 +297,20 @@ set(ROCKSDB_SOURCES
         util/xxhash.cc
         utilities/backupable/backupable_db.cc
         utilities/blob_db/blob_db.cc
+        utilities/blob_db/blob_db_impl.cc
+        utilities/blob_db/blob_db_options_impl.cc
+        utilities/blob_db/blob_dump_tool.cc
+        utilities/blob_db/blob_file.cc
+        utilities/blob_db/blob_log_format.cc
+        utilities/blob_db/blob_log_reader.cc
+        utilities/blob_db/blob_log_writer.cc
         utilities/checkpoint/checkpoint_impl.cc
         utilities/col_buf_decoder.cc
         utilities/col_buf_encoder.cc
         utilities/column_aware_encoding_util.cc
         utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
         utilities/date_tiered/date_tiered_db_impl.cc
+        utilities/debug.cc
         utilities/document/document_db.cc
         utilities/document/json_document.cc
         utilities/document/json_document_builder.cc