MDEV-16523 update RocksDB to 6.29fb

Author: spetrunia

storage/rocksdb/build_rocksdb.cmake

@@ -176,35 +176,53 @@ set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 #  - *_test.cc
 #  - *_bench.cc
 set(ROCKSDB_SOURCES
+        cache/cache.cc
+        cache/cache_entry_roles.cc
+        cache/cache_key.cc
+        cache/cache_reservation_manager.cc
         cache/clock_cache.cc
         cache/lru_cache.cc
         cache/sharded_cache.cc
         db/arena_wrapped_db_iter.cc
+        db/blob/blob_fetcher.cc
+        db/blob/blob_file_addition.cc
+        db/blob/blob_file_builder.cc
+        db/blob/blob_file_builder.cc
+        db/blob/blob_file_cache.cc
+        db/blob/blob_file_garbage.cc
+        db/blob/blob_file_meta.cc
+        db/blob/blob_file_reader.cc
+        db/blob/blob_garbage_meter.cc
+        db/blob/blob_log_format.cc
+        db/blob/blob_log_sequential_reader.cc
+        db/blob/blob_log_writer.cc
+        db/blob/prefetch_buffer_collection.cc
         db/builder.cc
         db/c.cc
         db/column_family.cc
-        db/compacted_db_impl.cc
         db/compaction/compaction.cc
         db/compaction/compaction_iterator.cc
-        db/compaction/compaction_picker.cc
         db/compaction/compaction_job.cc
+        db/compaction/compaction_picker.cc
         db/compaction/compaction_picker_fifo.cc
         db/compaction/compaction_picker_level.cc
         db/compaction/compaction_picker_universal.cc
+        db/compaction/sst_partitioner.cc
         db/convenience.cc
         db/db_filesnapshot.cc
+        db/dbformat.cc
+        db/db_impl/compacted_db_impl.cc
         db/db_impl/db_impl.cc
-        db/db_impl/db_impl_write.cc
         db/db_impl/db_impl_compaction_flush.cc
-        db/db_impl/db_impl_files.cc
-        db/db_impl/db_impl_open.cc
         db/db_impl/db_impl_debug.cc
         db/db_impl/db_impl_experimental.cc
+        db/db_impl/db_impl_files.cc
+        db/db_impl/db_impl_open.cc
         db/db_impl/db_impl_readonly.cc
         db/db_impl/db_impl_secondary.cc
+        db/db_impl/db_impl_write.cc
         db/db_info_dumper.cc
         db/db_iter.cc
-        db/dbformat.cc
         db/error_handler.cc
         db/event_helpers.cc
         db/experimental.cc
@@ -215,14 +233,16 @@ set(ROCKSDB_SOURCES
         db/forward_iterator.cc
         db/import_column_family_job.cc
         db/internal_stats.cc
-        db/logs_with_prep_tracker.cc
         db/log_reader.cc
+        db/logs_with_prep_tracker.cc
         db/log_writer.cc
         db/malloc_stats.cc
         db/memtable.cc
         db/memtable_list.cc
         db/merge_helper.cc
         db/merge_operator.cc
+        db/output_validator.cc
+        db/periodic_work_scheduler.cc
         db/range_del_aggregator.cc
         db/range_tombstone_fragmenter.cc
         db/repair.cc
@@ -233,25 +253,32 @@ set(ROCKSDB_SOURCES
         db/trim_history_scheduler.cc
         db/version_builder.cc
         db/version_edit.cc
+        db/version_edit_handler.cc
         db/version_set.cc
+        db/wal_edit.cc
         db/wal_manager.cc
-        db/write_batch.cc
         db/write_batch_base.cc
+        db/write_batch.cc
         db/write_controller.cc
         db/write_thread.cc
+        env/composite_env.cc
         env/env.cc
         env/env_chroot.cc
         env/env_encryption.cc
         env/env_hdfs.cc
         env/file_system.cc
+        env/file_system_tracer.cc
+        env/fs_remap.cc
         env/mock_env.cc
+        env/unique_id_gen.cc
         file/delete_scheduler.cc
+        file/filename.cc
         file/file_prefetch_buffer.cc
         file/file_util.cc
-        file/filename.cc
+        file/line_file_reader.cc
         file/random_access_file_reader.cc
-        file/read_write_util.cc
         file/readahead_raf.cc
+        file/read_write_util.cc
         file/sequence_file_reader.cc
         file/sst_file_manager_impl.cc
         file/writable_file_writer.cc
@@ -281,29 +308,38 @@ set(ROCKSDB_SOURCES
         monitoring/thread_status_util.cc
         monitoring/thread_status_util_debug.cc
         options/cf_options.cc
+        options/configurable.cc
+        options/customizable.cc
         options/db_options.cc
         options/options.cc
         options/options_helper.cc
         options/options_parser.cc
-        options/options_sanity_check.cc
         port/stack_trace.cc
         table/adaptive/adaptive_table_factory.cc
-        table/block_based/block.cc
+        table/block_based/binary_search_index_reader.cc
         table/block_based/block_based_filter_block.cc
         table/block_based/block_based_table_builder.cc
         table/block_based/block_based_table_factory.cc
+        table/block_based/block_based_table_iterator.cc
         table/block_based/block_based_table_reader.cc
         table/block_based/block_builder.cc
+        table/block_based/block.cc
+        table/block_based/block_prefetcher.cc
         table/block_based/block_prefix_index.cc
-        table/block_based/data_block_hash_index.cc
         table/block_based/data_block_footer.cc
+        table/block_based/data_block_hash_index.cc
         table/block_based/filter_block_reader_common.cc
         table/block_based/filter_policy.cc
         table/block_based/flush_block_policy.cc
         table/block_based/full_filter_block.cc
+        table/block_based/hash_index_reader.cc
         table/block_based/index_builder.cc
+        table/block_based/index_reader_common.cc
         table/block_based/parsed_full_filter_block.cc
         table/block_based/partitioned_filter_block.cc
+        table/block_based/partitioned_index_iterator.cc
+        table/block_based/partitioned_index_reader.cc
+        table/block_based/reader_common.cc
         table/block_based/uncompression_dict_reader.cc
         table/block_fetcher.cc
         table/cuckoo/cuckoo_table_builder.cc
@@ -321,10 +357,13 @@ set(ROCKSDB_SOURCES
         table/plain/plain_table_index.cc
         table/plain/plain_table_key_coding.cc
         table/plain/plain_table_reader.cc
+        table/sst_file_dumper.cc
         table/sst_file_reader.cc
         table/sst_file_writer.cc
+        table/table_factory.cc
         table/table_properties.cc
         table/two_level_iterator.cc
+        table/unique_id.cc
         test_util/sync_point.cc
         test_util/sync_point_impl.cc
         test_util/testutil.cc
@@ -335,49 +374,43 @@ set(ROCKSDB_SOURCES
         tools/ldb_tool.cc
         tools/sst_dump_tool.cc
         tools/trace_analyzer_tool.cc
-        trace_replay/trace_replay.cc
         trace_replay/block_cache_tracer.cc
+        trace_replay/io_tracer.cc
+        trace_replay/trace_record.cc
+        trace_replay/trace_record_handler.cc
+        trace_replay/trace_record_result.cc
+        trace_replay/trace_replay.cc
         util/coding.cc
         util/compaction_job_stats_impl.cc
         util/comparator.cc
         util/compression_context_cache.cc
         util/concurrent_task_limiter_impl.cc
         util/crc32c.cc
         util/dynamic_bloom.cc
-        util/hash.cc
-        util/murmurhash.cc
-        util/random.cc
-        util/rate_limiter.cc
-        util/slice.cc
         util/file_checksum_helper.cc
-        util/status.cc
-        util/string_util.cc
-        util/thread_local.cc
-        util/threadpool_imp.cc
-        util/xxhash.cc
+        util/hash.cc
         utilities/backupable/backupable_db.cc
         utilities/blob_db/blob_compaction_filter.cc
         utilities/blob_db/blob_db.cc
         utilities/blob_db/blob_db_impl.cc
         utilities/blob_db/blob_db_impl_filesnapshot.cc
         utilities/blob_db/blob_dump_tool.cc
         utilities/blob_db/blob_file.cc
-        utilities/blob_db/blob_log_reader.cc
-        utilities/blob_db/blob_log_writer.cc
-        utilities/blob_db/blob_log_format.cc
         utilities/checkpoint/checkpoint_impl.cc
+        utilities/compaction_filters.cc
         utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
         utilities/debug.cc
         utilities/env_mirror.cc
         utilities/env_timed.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/memory/memory_util.cc
         utilities/merge_operators/bytesxor.cc
+        utilities/merge_operators.cc
         utilities/merge_operators/max.cc
         utilities/merge_operators/put.cc
         utilities/merge_operators/sortlist.cc
-        utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/string_append/stringappend2.cc
+        utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/uint64add.cc
         utilities/object_registry.cc
         utilities/option_change_migration/option_change_migration.cc
@@ -391,22 +424,37 @@ set(ROCKSDB_SOURCES
         utilities/simulator_cache/sim_cache.cc
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
         utilities/trace/file_trace_reader_writer.cc
-        utilities/transactions/optimistic_transaction_db_impl.cc
+        utilities/trace/replayer_impl.cc
+        utilities/transactions/lock/lock_manager.cc
+        utilities/transactions/lock/point/point_lock_manager.cc
+        utilities/transactions/lock/point/point_lock_tracker.cc
         utilities/transactions/optimistic_transaction.cc
+        utilities/transactions/optimistic_transaction_db_impl.cc
         utilities/transactions/pessimistic_transaction.cc
         utilities/transactions/pessimistic_transaction_db.cc
         utilities/transactions/snapshot_checker.cc
         utilities/transactions/transaction_base.cc
         utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/transaction_lock_mgr.cc
         utilities/transactions/transaction_util.cc
         utilities/transactions/write_prepared_txn.cc
         utilities/transactions/write_prepared_txn_db.cc
         utilities/transactions/write_unprepared_txn.cc
         utilities/transactions/write_unprepared_txn_db.cc
         utilities/ttl/db_ttl_impl.cc
+        utilities/wal_filter.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
         utilities/write_batch_with_index/write_batch_with_index_internal.cc
+        util/murmurhash.cc
+        util/random.cc
+        util/rate_limiter.cc
+        util/regex.cc
+        util/ribbon_config.cc
+        util/slice.cc
+        util/status.cc
+        util/string_util.cc
+        util/thread_local.cc
+        util/threadpool_imp.cc
+        util/xxhash.cc
 )
 
 
@@ -484,8 +532,10 @@ IF(CMAKE_VERSION VERSION_GREATER "2.8.10")
   STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S")
 ENDIF()
 
+# psergey-added:
+SET(GIT_MOD 0)
 CONFIGURE_FILE(${ROCKSDB_SOURCE_DIR}/util/build_version.cc.in build_version.cc @ONLY)
-INCLUDE_DIRECTORIES(${ROCKSDB_SOURCE_DIR}/util)
+
 list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/build_version.cc)
 
 ADD_CONVENIENCE_LIBRARY(rocksdblib ${SOURCES})

storage/rocksdb/ha_rocksdb.cc

@@ -1596,7 +1596,7 @@ static MYSQL_SYSVAR_BOOL(
     "BlockBasedTableOptions::no_block_cache for RocksDB", nullptr, nullptr,
     rocksdb_tbl_options->no_block_cache);
 
-static MYSQL_SYSVAR_SIZE_T(block_size, rocksdb_tbl_options->block_size,
+static MYSQL_SYSVAR_UINT64_T(block_size, rocksdb_tbl_options->block_size,
                           PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
                           "BlockBasedTableOptions::block_size for RocksDB",
                           nullptr, nullptr, rocksdb_tbl_options->block_size,
@@ -3992,7 +3992,7 @@ static int rocksdb_commit_by_xid(handlerton *const hton, XID *const xid) {
   DBUG_ASSERT(xid != nullptr);
   DBUG_ASSERT(commit_latency_stats != nullptr);
 
-  rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true);
+  rocksdb::StopWatchNano timer(rocksdb::SystemClock::Default().get(), true);
 
   const auto name = rdb_xid_to_string(*xid);
   DBUG_ASSERT(!name.empty());
@@ -4187,7 +4187,7 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx)
   DBUG_ASSERT(thd != nullptr);
   DBUG_ASSERT(commit_latency_stats != nullptr);
 
-  rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true);
+  rocksdb::StopWatchNano timer(rocksdb::SystemClock::Default().get(), true);
 
   /* note: h->external_lock(F_UNLCK) is called after this function is called) */
   Rdb_transaction *tx = get_tx_from_thd(thd);
@@ -4732,8 +4732,7 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd,
 
         if (tf_name.find("BlockBasedTable") != std::string::npos) {
           const rocksdb::BlockBasedTableOptions *const bbt_opt =
-              reinterpret_cast<rocksdb::BlockBasedTableOptions *>(
-                  table_factory->GetOptions());
+                  table_factory->GetOptions<rocksdb::BlockBasedTableOptions>();
 
           if (bbt_opt != nullptr) {
             if (bbt_opt->block_cache.get() != nullptr) {

storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result

@@ -20,19 +20,19 @@ set @tmp1=@@rocksdb_verify_row_debug_checksums;
 set rocksdb_verify_row_debug_checksums=1;
 set session debug_dbug= "+d,myrocks_simulate_bad_row_read1";
 select * from t1 where pk=1;
-ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
+ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
 set session debug_dbug= "-d,myrocks_simulate_bad_row_read1";
 set rocksdb_verify_row_debug_checksums=@tmp1;
 select * from t1 where pk=1;
 pk	col1
 1	1
 set session debug_dbug= "+d,myrocks_simulate_bad_row_read2";
 select * from t1 where pk=1;
-ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
+ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
 set session debug_dbug= "-d,myrocks_simulate_bad_row_read2";
 set session debug_dbug= "+d,myrocks_simulate_bad_row_read3";
 select * from t1 where pk=1;
-ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
+ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
 set session debug_dbug= "-d,myrocks_simulate_bad_row_read3";
 insert into t1 values(4,'0123456789');
 select * from t1;
@@ -56,7 +56,7 @@ pk	col1
 ABCD	1
 set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1";
 select * from t2;
-ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
+ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
 set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1";
 drop table t2;
 create table t2 (
@@ -69,6 +69,6 @@ pk	col1
 ABCD	1
 set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1";
 select * from t2;
-ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
+ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
 set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1";
 drop table t2;

storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result

@@ -1,12 +1,6 @@
-call mtr.add_suppression("Column family 'cf1' not found");
-call mtr.add_suppression("Column family 'rev:cf2' not found");
 DROP TABLE IF EXISTS t1;
 call mtr.add_suppression("Column family 'cf1' not found");
 call mtr.add_suppression("Column family 'rev:cf2' not found");
-set global rocksdb_compact_cf = 'cf1';
-set global rocksdb_compact_cf = 'rev:cf2';
-set global rocksdb_signal_drop_index_thread = 1;
-# restart
 CREATE TABLE t1 (
 a int not null,
 b int not null,
@@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1',
 key (b) comment 'rev:cf2'
 ) ENGINE=RocksDB;
 DELETE FROM t1;
+set global rocksdb_compact_cf = 'cf1';
+set global rocksdb_compact_cf = 'rev:cf2';
+set global rocksdb_signal_drop_index_thread = 1;
+# restart
 select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';
 drop table t1;
 select case when variable_value-@a < 500000 then 'true' else 'false' end  from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';

storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result

@@ -982,7 +982,7 @@ rocksdb_skip_fill_cache	OFF
 rocksdb_skip_unique_check_tables	.*
 rocksdb_sst_mgr_rate_bytes_per_sec	0
 rocksdb_stats_dump_period_sec	600
-rocksdb_stats_level	0
+rocksdb_stats_level	1
 rocksdb_stats_recalc_rate	0
 rocksdb_store_row_debug_checksums	OFF
 rocksdb_strict_collation_check	OFF

storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result

@@ -1,2 +1,2 @@
 Check for MANIFEST files
-MANIFEST-000006
+MANIFEST-000004