Implement RocksDB file-level checksums (#1280)

Summary:
Add a new boolean system variable rocksdb_file_checksums.

Pull Request resolved: #1280

Reviewed By: hermanlee

Differential Revision: D43964275

Pulled By: sunshine-Chun

fbshipit-source-id: fece9ae

mysql-test/suite/rocksdb/r/file_checksums.result

@@ -0,0 +1,15 @@
+CREATE TABLE t1 (
+a INT NOT NULL, b CHAR(128),
+PRIMARY KEY (a) COMMENT 'cfname=cf1') ENGINE=ROCKSDB;
+INSERT INTO t1 VALUES (1, "foo");
+INSERT INTO t1 VALUES (2, "bar");
+# restart
+include/assert.inc [RocksDB file checksums must be enabled]
+SELECT * FROM t1;
+a	b
+1	foo
+2	bar
+SET GLOBAL rocksdb_pause_background_work = 1;
+# Kill the server
+# restart
+DROP TABLE t1;

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -979,6 +979,7 @@ rocksdb_enable_write_thread_adaptive_yield	OFF
 rocksdb_error_if_exists	OFF
 rocksdb_error_on_suboptimal_collation	ON
 rocksdb_fault_injection_options	
+rocksdb_file_checksums	OFF
 rocksdb_flush_log_at_trx_commit	1
 rocksdb_force_compute_memtable_stats	ON
 rocksdb_force_compute_memtable_stats_cachetime	0

mysql-test/suite/rocksdb/r/rocksdb_checksums.result

@@ -6,6 +6,7 @@ drop table if exists t1,t2,t3;
 show variables like 'rocksdb_%checksum%';
 Variable_name	Value
 rocksdb_checksums_pct	100
+rocksdb_file_checksums	OFF
 rocksdb_store_row_debug_checksums	OFF
 rocksdb_verify_row_debug_checksums	OFF
 create table t1 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb;

mysql-test/suite/rocksdb/t/file_checksums-master.opt

@@ -0,0 +1 @@
+--rocksdb-file-checksums

mysql-test/suite/rocksdb/t/file_checksums.test

@@ -0,0 +1,53 @@
+--source include/have_rocksdb.inc
+
+--let $sst_dir=`SELECT CONCAT(@@datadir, @@rocksdb_datadir)`
+
+CREATE TABLE t1 (
+       a INT NOT NULL, b CHAR(128),
+       PRIMARY KEY (a) COMMENT 'cfname=cf1') ENGINE=ROCKSDB;
+INSERT INTO t1 VALUES (1, "foo");
+INSERT INTO t1 VALUES (2, "bar");
+
+# Restart to check the file checksums
+--source include/restart_mysqld.inc
+
+--let $assert_text = RocksDB file checksums must be enabled
+--let $assert_cond = @@rocksdb_file_checksums = 1
+--source include/assert.inc
+
+SELECT * FROM t1;
+
+SET GLOBAL rocksdb_pause_background_work = 1;
+
+let sst_file=`SELECT SST_NAME
+       FROM INFORMATION_SCHEMA.rocksdb_index_file_map AS file_map
+       INNER JOIN INFORMATION_SCHEMA.rocksdb_ddl AS ddl
+       ON file_map.COLUMN_FAMILY = ddl.COLUMN_FAMILY`;
+
+--let sst=$sst_dir/$sst_file
+--file_exists $sst
+
+--copy_file $sst $sst.bak
+
+--exec echo "corrupt me" | dd if=$sst of=$sst conv=notrunc seek=100 bs=1 count=10
+
+--let $fail_err_log=$MYSQLTEST_VARDIR/checksum-fail.err
+
+# Cannot shutdown with background threads stopped, thus kill it
+--source include/kill_mysqld.inc
+
+# Must fail to start. This used to grep $fail_err_log for the specific file
+# checksum corruption error, but intermittently it hit block checksum error on
+# DB open first.
+--error 1
+exec $MYSQLD --defaults-group-suffix=.1 --defaults-file=$MYSQLTEST_VARDIR/my.cnf
+     --rocksdb-file-checksums > $fail_err_log 2>&1;
+
+--remove_file $sst
+--move_file $sst.bak $sst
+
+--source include/start_mysqld.inc
+
+--remove_file $fail_err_log
+
+DROP TABLE t1;

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_file_checksums_basic.result

@@ -0,0 +1,7 @@
+SET @start_global_value = @@global.ROCKSDB_FILE_CHECKSUMS;
+SELECT @start_global_value;
+@start_global_value
+0
+"Trying to set variable @@global.ROCKSDB_FILE_CHECKSUMS to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_FILE_CHECKSUMS   = 444;
+ERROR HY000: Variable 'rocksdb_file_checksums' is a read only variable

mysql-test/suite/rocksdb_sys_vars/t/rocksdb_file_checksums_basic.test

@@ -0,0 +1,6 @@
+--source include/have_rocksdb.inc
+
+--let $sys_var=ROCKSDB_FILE_CHECKSUMS
+--let $read_only=1
+--let $session=0
+--source ../include/rocksdb_sys_var.inc

storage/rocksdb/ha_rocksdb.cc

@@ -905,6 +905,7 @@ bool rocksdb_enable_delete_range_for_drop_index = false;
 uint rocksdb_clone_checkpoint_max_age;
 uint rocksdb_clone_checkpoint_max_count;
 unsigned long long rocksdb_converter_record_cached_length = 0;
+static bool rocksdb_file_checksums = false;
 static std::time_t last_binlog_ttl_compaction_ts = std::time(nullptr);
 
 static std::atomic<uint64_t> rocksdb_row_lock_deadlocks(0);
@@ -2834,6 +2835,12 @@ static MYSQL_SYSVAR_ULONGLONG(
     nullptr, nullptr, /* default */ rocksdb_converter_record_cached_length,
     /* min */ 0, /* max */ UINT64_MAX, 0);
 
+static MYSQL_SYSVAR_BOOL(
+    file_checksums, rocksdb_file_checksums,
+    PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY,
+    "Whether to write and check RocksDB file-level checksums", nullptr, nullptr,
+    false);
+
 static const int ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE = 100;
 
 static struct SYS_VAR *rocksdb_system_variables[] = {
@@ -3055,6 +3062,7 @@ static struct SYS_VAR *rocksdb_system_variables[] = {
     MYSQL_SYSVAR(clone_checkpoint_max_age),
     MYSQL_SYSVAR(clone_checkpoint_max_count),
     MYSQL_SYSVAR(converter_record_cached_length),
+    MYSQL_SYSVAR(file_checksums),
     nullptr};
 
 static bool is_tmp_table(const std::string &tablename) {
@@ -7549,6 +7557,11 @@ static int rocksdb_init_internal(void *const p) {
       rocksdb_db_options->env, myrocks_logger, trash_dir,
       rocksdb_sst_mgr_rate_bytes_per_sec, true /* delete_existing_trash */));
 
+  if (rocksdb_file_checksums) {
+    rocksdb_db_options->file_checksum_gen_factory =
+        rocksdb::GetFileChecksumGenCrc32cFactory();
+  }
+
   std::vector<std::string> cf_names;
   rocksdb::Status status;
   status = rocksdb::DB::ListColumnFamilies(*rocksdb_db_options, rocksdb_datadir,
@@ -7794,6 +7807,20 @@ static int rocksdb_init_internal(void *const p) {
     DBUG_RETURN(HA_EXIT_FAILURE);
   }
 
+  if (rocksdb_file_checksums) {
+    LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG,
+                    "Verifying file checksums...");
+    rocksdb::ReadOptions checksum_read_options;
+    checksum_read_options.readahead_size = 2 * 1024 * 1024;
+    status = rdb->VerifyFileChecksums(checksum_read_options);
+    if (!status.ok()) {
+      rdb_log_status_error(status, "Instance failed checksum verification");
+      for (auto cfh_ptr : cf_handles) delete (cfh_ptr);
+      DBUG_RETURN(HA_EXIT_FAILURE);
+    }
+    LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, "...done");
+  }
+
   // NO_LINT_DEBUG
   LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG,
                   "RocksDB:Init column families...");