cherry-pick: roadmap and rocksdb 5.15 (#3701)

* Roadmap: improve and update (#3641)

Signed-off-by: Huachao Huang <huachao.huang@gmail.com>

* rocksdb: update to release-5.15 (#3661)

Signed-off-by: Huachao Huang <huachao.huang@gmail.com>

docs/ROADMAP.md

@@ -7,27 +7,38 @@ category: Roadmap
 
 This document defines the roadmap for TiKV development.
 
-- [ ] Raft
-    - [x] Region merge
-    - [ ] Local read thread
-    - [ ] Multi-thread raftstore
-    - [x] None voter
-    - [x] Pre-vote
-    - [ ] Multi-thread apply pool
-    - [ ] Split region in batch
-    - [ ] Raft Engine
-- [x] RocksDB 
-    - [x] DeleteRange
-    - [ ] BlobDB 
-- [x] Transaction
-    - [x] Optimize transaction conflicts
-    - [ ] Distributed GC
-- [x] Coprocessor
-    - [x] Streaming
-- [ ] Tool
-    - [x] Import distributed data
-    - [ ] Export distributed data
-    - [ ] Disaster Recovery
-- [ ] Flow control and degradation
-- [ ] Client
-    - [ ] TiKV client (Rust crate)
+## Raft
+- [x] Region Merge - Merge small Regions together to reduce overhead
+- [x] Local Read Thread - Process read requests in a local read thread
+- [x] Split Region in Batch - Speed up Region split for large Regions
+- [x] Raft Learner - Support Raft learner to smooth the configuration change process
+- [x] Raft Pre-vote - Support Raft pre-vote to avoid unnecessary leader election on network isolation
+- [ ] Joint Consensus - Change multi members safely.
+- [ ] Multi-thread Raftstore - Process Region Raft logic in multiple threads
+- [ ] Multi-thread apply pool - Apply Region Raft committed entries in multiple threads
+
+## Engine
+- [ ] Titan - Separate large key-values from LSM-Tree
+- [ ] Pluggable Engine Interface - Clean up the engine wrapper code and provide more extendibility
+
+## Storage
+- [ ] Flow Control - Do flow control in scheduler to avoid write stall in advance
+
+## Transaction
+- [x] Optimize transaction conflicts
+- [ ] Distributed GC - Distribute MVCC garbage collection control to TiKV
+
+## Coprocessor
+- [x] Streaming - Cut large data set into small chunks to optimize memory consumption
+- [ ] Chunk Execution - Process data in chunk to improve performance
+- [ ] Request Tracing - Provide per-request execution details
+
+## Tools
+- [x] TiKV Importer - Speed up data importing by SST file ingestion
+
+## Client
+- [ ] TiKV client (Rust crate)
+- [ ] Batch gRPC Message - Reduce message overhead
+
+## PD
+- [ ] Optimize Region metadata - Save Region metadata in detached storage engine

src/raftstore/store/snap.rs

@@ -1451,6 +1451,7 @@ impl SnapManagerBuilder {
 #[cfg(test)]
 mod test {
     use protobuf::Message;
+    use std::cmp;
     use std::fs::{self, File, OpenOptions};
     use std::io::{self, Read, Seek, SeekFrom, Write};
     use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
@@ -2339,8 +2340,9 @@ mod test {
         let kv = get_test_db_for_regions(&kv_path, &regions).unwrap();
 
         let snapfiles_path = TempDir::new("test-snapshot-max-total-size-snapshots").unwrap();
+        let max_total_size = 10240;
         let snap_mgr = SnapManagerBuilder::default()
-            .max_total_size(10240)
+            .max_total_size(max_total_size)
             .build(snapfiles_path.path().to_str().unwrap(), None);
         let snapshot = DbSnapshot::new(kv);
 
@@ -2388,13 +2390,15 @@ mod test {
                 Box::new(snap_mgr.clone()),
             ).unwrap();
 
-            if i < 5 {
-                // sizeof(snapshot) == 1918 bytes. The first 1918 is for region 100.
-                // That snapshot won't be deleted because it's not for generating.
-                assert_eq!(snap_mgr.get_total_snap_size() as usize, 1918 * (i + 2));
-            } else {
-                assert_eq!(snap_mgr.get_total_snap_size() as usize, 1918 * 6);
-            }
+            // TODO: this size may change in different RocksDB version.
+            let snap_size = 1342;
+            let max_snap_count = (max_total_size + snap_size - 1) / snap_size;
+            // The first snap_size is for region 100.
+            // That snapshot won't be deleted because it's not for generating.
+            assert_eq!(
+                snap_mgr.get_total_snap_size(),
+                snap_size * cmp::min(max_snap_count, (i + 2) as u64)
+            );
         }
     }
 }

src/storage/mod.rs

@@ -2126,12 +2126,11 @@ mod tests {
                 .wait(),
         );
 
-        // Delete range ["", ""), it means delete all
         storage
             .async_delete_range(
                 Context::new(),
                 Key::from_raw(b""),
-                Key::from_raw(b""),
+                Key::from_raw(&[255]),
                 expect_ok_callback(tx.clone(), 9),
             )
             .unwrap();

src/storage/mvcc/reader/backward_scanner.rs

@@ -1243,7 +1243,7 @@ mod tests {
             .unwrap();
         assert_eq!(scanner.read_next().unwrap(), None);
         let statistics = scanner.take_statistics();
-        assert_eq!(statistics.lock.prev, 256);
+        assert_eq!(statistics.lock.prev, 255);
         assert_eq!(statistics.write.prev, 1);
     }
 

src/util/rocksdb/engine_metrics.rs

@@ -50,8 +50,8 @@ pub const ENGINE_TICKER_TYPES: &[TickerType] = &[
     TickerType::BlockCacheDataHit,
     TickerType::BlockCacheDataAdd,
     TickerType::BlockCacheDataBytesInsert,
-    TickerType::BlockCacheByteRead,
-    TickerType::BlockCacheByteWrite,
+    TickerType::BlockCacheBytesRead,
+    TickerType::BlockCacheBytesWrite,
     TickerType::BloomFilterUseful,
     TickerType::MemtableHit,
     TickerType::MemtableMiss,
@@ -83,17 +83,17 @@ pub const ENGINE_TICKER_TYPES: &[TickerType] = &[
     TickerType::WalFileBytes,
     TickerType::WriteDoneBySelf,
     TickerType::WriteDoneByOther,
-    TickerType::WriteTimeout,
-    TickerType::WriteWithWAL,
+    TickerType::WriteTimedout,
+    TickerType::WriteWithWal,
     TickerType::CompactReadBytes,
     TickerType::CompactWriteBytes,
     TickerType::FlushWriteBytes,
     TickerType::ReadAmpEstimateUsefulBytes,
     TickerType::ReadAmpTotalReadBytes,
 ];
 pub const ENGINE_HIST_TYPES: &[HistType] = &[
-    HistType::GetMicros,
-    HistType::WriteMicros,
+    HistType::DbGet,
+    HistType::DbWrite,
     HistType::CompactionTime,
     HistType::TableSyncMicros,
     HistType::CompactionOutfileSyncMicros,
@@ -105,9 +105,9 @@ pub const ENGINE_HIST_TYPES: &[HistType] = &[
     HistType::HardRateLimitDelayCount,
     HistType::SoftRateLimitDelayCount,
     HistType::NumFilesInSingleCompaction,
-    HistType::SeekMicros,
+    HistType::DbSeek,
     HistType::WriteStall,
-    HistType::SSTReadMicros,
+    HistType::SstReadMicros,
     HistType::NumSubcompactionsScheduled,
     HistType::BytesPerRead,
     HistType::BytesPerWrite,
@@ -215,12 +215,12 @@ pub fn flush_engine_ticker_metrics(t: TickerType, value: u64, name: &str) {
                 .with_label_values(&[name, "block_cache_data_bytes_insert"])
                 .inc_by(v);
         }
-        TickerType::BlockCacheByteRead => {
+        TickerType::BlockCacheBytesRead => {
             STORE_ENGINE_FLOW_VEC
                 .with_label_values(&[name, "block_cache_byte_read"])
                 .inc_by(v);
         }
-        TickerType::BlockCacheByteWrite => {
+        TickerType::BlockCacheBytesWrite => {
             STORE_ENGINE_FLOW_VEC
                 .with_label_values(&[name, "block_cache_byte_write"])
                 .inc_by(v);
@@ -390,12 +390,12 @@ pub fn flush_engine_ticker_metrics(t: TickerType, value: u64, name: &str) {
                 .with_label_values(&[name, "write_done_by_other"])
                 .inc_by(v);
         }
-        TickerType::WriteTimeout => {
+        TickerType::WriteTimedout => {
             STORE_ENGINE_WRITE_SERVED_VEC
                 .with_label_values(&[name, "write_timeout"])
                 .inc_by(v);
         }
-        TickerType::WriteWithWAL => {
+        TickerType::WriteWithWal => {
             STORE_ENGINE_WRITE_SERVED_VEC
                 .with_label_values(&[name, "write_with_wal"])
                 .inc_by(v);
@@ -431,7 +431,7 @@ pub fn flush_engine_ticker_metrics(t: TickerType, value: u64, name: &str) {
 
 pub fn flush_engine_histogram_metrics(t: HistType, value: HistogramData, name: &str) {
     match t {
-        HistType::GetMicros => {
+        HistType::DbGet => {
             STORE_ENGINE_GET_MICROS_VEC
                 .with_label_values(&[name, "get_median"])
                 .set(value.median);
@@ -451,7 +451,7 @@ pub fn flush_engine_histogram_metrics(t: HistType, value: HistogramData, name: &
                 .with_label_values(&[name, "get_max"])
                 .set(value.max);
         }
-        HistType::WriteMicros => {
+        HistType::DbWrite => {
             STORE_ENGINE_WRITE_MICROS_VEC
                 .with_label_values(&[name, "write_median"])
                 .set(value.median);
@@ -691,7 +691,7 @@ pub fn flush_engine_histogram_metrics(t: HistType, value: HistogramData, name: &
                 .with_label_values(&[name, "num_files_in_single_compaction_max"])
                 .set(value.max);
         }
-        HistType::SeekMicros => {
+        HistType::DbSeek => {
             STORE_ENGINE_SEEK_MICROS_VEC
                 .with_label_values(&[name, "seek_median"])
                 .set(value.median);
@@ -731,7 +731,7 @@ pub fn flush_engine_histogram_metrics(t: HistType, value: HistogramData, name: &
                 .with_label_values(&[name, "write_stall_max"])
                 .set(value.max);
         }
-        HistType::SSTReadMicros => {
+        HistType::SstReadMicros => {
             STORE_ENGINE_SST_READ_MICROS_VEC
                 .with_label_values(&[name, "sst_read_micros_median"])
                 .set(value.median);