Move SST code to engine traits (tikv#5790)

* engine_*: outline SST traits Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_rocks: Use Rc inside RocksSstReader Signed-off-by: Brian Anderson <andersrb@gmail.com> * *: temporarily patch local rust-rocksdb Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_rocks: Implement SstReader Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_traits: Add SstExt to KvEngine Signed-off-by: Brian Anderson <andersrb@gmail.com> * sst_importer: replace rocks SstReader with abstract SstReader Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_*: Add SstWriter SstWriterBuilder Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_rocks: cleanup warnings Signed-off-by: Brian Anderson <andersrb@gmail.com> * sst_importer: Use generic SstWriter Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_rocks: add test_helpers mod and new_default_engine fn Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_rocks: port sst tests Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine: remove sst module Signed-off-by: Brian Anderson <andersrb@gmail.com> * raftstore: Use traits for SstWriter Signed-off-by: Brian Anderson <andersrb@gmail.com> * sst_service: Fix type annotations Signed-off-by: Brian Anderson <andersrb@gmail.com> * raftstore: Use SST traits Signed-off-by: Brian Anderson <andersrb@gmail.com> * *: temporarily use brson's rocksdb branch Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_traits: Rename CFOptions to ColumnFamilyOptions For consistency with existing types. Less confusing. Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_traits: Rename get_cf_handle to cf_handle For consistency with engine crate Signed-off-by: Brian Anderson <andersrb@gmail.com> wip * engine_rocks: Move sst test cases from engine to engine_rocks Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine_traits: Introduce TitanDBOptions and re-enable titan sst test Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine: cleanup unused Signed-off-by: Brian Anderson <andersrb@gmail.com> * backup: use engine traits for SST Signed-off-by: Brian Anderson <andersrb@gmail.com> * *: rustfmt Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine: Move more sst functions into engine_rocks Signed-off-by: Brian Anderson <andersrb@gmail.com> * engine: cleanup unused Signed-off-by: Brian Anderson <andersrb@gmail.com> * *: fix rocksdb patch Signed-off-by: Brian Anderson <andersrb@gmail.com> * Revert "*: fix rocksdb patch" This reverts commit afd744d. Signed-off-by: Brian Anderson <andersrb@gmail.com> * Revert "*: temporarily use brson's rocksdb branch" This reverts commit d86c374. Signed-off-by: Brian Anderson <andersrb@gmail.com> * Revert "*: temporarily patch local rust-rocksdb" This reverts commit be532b0. Signed-off-by: Brian Anderson <andersrb@gmail.com>
hawkingrei · Dec 1, 2019 · 563337c · 563337c
1 parent 21205e2
commit 563337c
Show file tree

Hide file tree

Showing 30 changed files with 843 additions and 518 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/components/backup/Cargo.toml b/components/backup/Cargo.toml
@@ -17,6 +17,8 @@ slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debu
 # better to not use slog-global, but pass in the logger
 slog-global = { version = "0.1", git = "https://github.com/breeswish/slog-global.git", rev = "0e23a5baff302a9d7bccd85f8f31e43339c2f2c1" }
 engine = { path = "../engine" }
+engine_traits = { path = "../engine_traits" }
+engine_rocks = { path = "../engine_rocks" }
 tikv_util = { path = "../tikv_util" }
 futures = "0.1"
 tempfile = "3.0"

diff --git a/components/backup/src/errors.rs b/components/backup/src/errors.rs
@@ -3,6 +3,7 @@
 use std::io::Error as IoError;
 use std::{error, result};
 
+use engine_traits::Error as EngineTraitError;
 use kvproto::backup::Error as ErrorPb;
 use kvproto::errorpb::{Error as RegionError, ServerIsBusy};
 use kvproto::kvrpcpb::KeyError;
@@ -97,6 +98,8 @@ pub enum Error {
     Io(IoError),
     #[fail(display = "Engine error {}", _0)]
     Engine(EngineError),
+    #[fail(display = "Engine error {}", _0)]
+    EngineTrait(EngineTraitError),
     #[fail(display = "Transaction error {}", _0)]
     Txn(TxnError),
     #[fail(display = "ClusterID error current {}, request {}", current, request)]
@@ -120,6 +123,7 @@ impl_from! {
     String => Rocks,
     IoError => Io,
     EngineError => Engine,
+    EngineTraitError => EngineTrait,
     TxnError => Txn,
 }
 

diff --git a/components/backup/src/writer.rs b/components/backup/src/writer.rs
@@ -4,8 +4,9 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use engine::rocks::util::io_limiter::{IOLimiter, LimitReader};
-use engine::rocks::{SstWriter, SstWriterBuilder};
 use engine::{CF_DEFAULT, CF_WRITE, DB};
+use engine_rocks::{RocksEngine, RocksSstWriter, RocksSstWriterBuilder};
+use engine_traits::{SstWriter, SstWriterBuilder};
 use external_storage::ExternalStorage;
 use kvproto::backup::File;
 use tikv::coprocessor::checksum_crc64_xor;
@@ -17,14 +18,14 @@ use crate::metrics::*;
 use crate::{Error, Result};
 
 struct Writer {
-    writer: SstWriter,
+    writer: RocksSstWriter,
     total_kvs: u64,
     total_bytes: u64,
     checksum: u64,
 }
 
 impl Writer {
-    fn new(writer: SstWriter) -> Self {
+    fn new(writer: RocksSstWriter) -> Self {
         Writer {
             writer,
             total_kvs: 0,
@@ -98,15 +99,15 @@ pub struct BackupWriter {
 impl BackupWriter {
     /// Create a new BackupWriter.
     pub fn new(db: Arc<DB>, name: &str, limiter: Option<Arc<IOLimiter>>) -> Result<BackupWriter> {
-        let default = SstWriterBuilder::new()
+        let default = RocksSstWriterBuilder::new()
             .set_in_memory(true)
             .set_cf(CF_DEFAULT)
-            .set_db(db.clone())
+            .set_db(RocksEngine::from_ref(&db))
             .build(name)?;
-        let write = SstWriterBuilder::new()
+        let write = RocksSstWriterBuilder::new()
             .set_in_memory(true)
             .set_cf(CF_WRITE)
-            .set_db(db.clone())
+            .set_db(RocksEngine::from_ref(&db))
             .build(name)?;
         let name = name.to_owned();
         Ok(BackupWriter {

diff --git a/components/engine/src/rocks/mod.rs b/components/engine/src/rocks/mod.rs
@@ -1,8 +1,6 @@
 // Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0.
 
 mod db;
-mod sst;
-pub use sst::{SstReader, SstWriter, SstWriterBuilder};
 
 pub mod util;
 

diff --git a/components/engine/src/rocks/sst.rs b/components/engine/src/rocks/sst.rs