From 4e787a9f79346fef44bb49290a0d6e2ff9127a39 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 18 Sep 2019 13:10:47 +0800 Subject: [PATCH] cmd, backup: enable backup feature (#5476) Signed-off-by: Neil Shen --- Cargo.lock | 1 + cmd/Cargo.toml | 1 + cmd/src/server.rs | 52 ++++++++++++++++++++++++++++--- cmd/src/setup.rs | 6 ++-- components/backup/src/endpoint.rs | 36 +++++++-------------- components/backup/src/lib.rs | 18 ----------- 6 files changed, 64 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f109acef2cd..c2f126249ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -338,6 +338,7 @@ dependencies = [ name = "cmd" version = "0.0.1" dependencies = [ + "backup", "chrono", "clap", "engine", diff --git a/cmd/Cargo.toml b/cmd/Cargo.toml index 3083910b7ff..6e957e46e15 100644 --- a/cmd/Cargo.toml +++ b/cmd/Cargo.toml @@ -52,6 +52,7 @@ kvproto = { git = "https://github.com/pingcap/kvproto.git" } tikv_util = { path = "../components/tikv_util" } engine = { path = "../components/engine" } pd_client = { path = "../components/pd_client" } +backup = { path = "../components/backup" } [target.'cfg(unix)'.dependencies] signal = "0.6" diff --git a/cmd/src/server.rs b/cmd/src/server.rs index 1cdfdc5c823..f796c460a32 100644 --- a/cmd/src/server.rs +++ b/cmd/src/server.rs @@ -7,6 +7,7 @@ use engine::rocks::util::metrics_flusher::{MetricsFlusher, DEFAULT_FLUSHER_INTER use engine::rocks::util::security::encrypted_env_from_cipher_file; use engine::Engines; use fs2::FileExt; +use kvproto::backup::create_backup; use kvproto::deadlock::create_deadlock; use kvproto::debugpb::create_debug; use kvproto::import_sstpb::create_import_sst; @@ -225,6 +226,11 @@ fn run_raft_server(pd_client: RpcClient, cfg: &TiKvConfig, security_mgr: Arc ({ + ($lvl:expr $(, $arg:expr)*) => ({ if $crate::setup::LOG_INITIALIZED.load(::std::sync::atomic::Ordering::SeqCst) { - crit!($lvl, $($arg)+); + crit!($lvl $(, $arg)*); } else { - eprintln!($lvl, $($arg)+); + eprintln!($lvl $(, $arg)*); } slog_global::clear_global(); ::std::process::exit(1) diff --git a/components/backup/src/endpoint.rs b/components/backup/src/endpoint.rs index f86ce996909..8d4e6800329 100644 --- a/components/backup/src/endpoint.rs +++ b/components/backup/src/endpoint.rs @@ -1,10 +1,5 @@ // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. -// TODO: remove it after all code been merged. -#![allow(unused_variables)] -#![allow(dead_code)] - -use std::cmp; use std::fmt; use std::sync::atomic::*; use std::sync::*; @@ -13,24 +8,17 @@ use std::time::*; use engine::rocks::util::io_limiter::IOLimiter; use engine::DB; use external_storage::*; +use futures::lazy; use futures::sync::mpsc::*; -use futures::{lazy, Future}; use kvproto::backup::*; use kvproto::kvrpcpb::{Context, IsolationLevel}; use kvproto::metapb::*; use raft::StateRole; -use tikv::raftstore::coprocessor::RegionInfoAccessor; use tikv::raftstore::store::util::find_peer; -use tikv::server::transport::ServerRaftStoreRouter; -use tikv::storage::kv::{ - Engine, Error as EngineError, RegionInfoProvider, ScanMode, StatisticsSummary, -}; -use tikv::storage::txn::{ - EntryBatch, Error as TxnError, Msg, Scanner, SnapshotStore, Store, TxnEntryScanner, - TxnEntryStore, -}; +use tikv::storage::kv::{Engine, RegionInfoProvider}; +use tikv::storage::txn::{EntryBatch, SnapshotStore, TxnEntryScanner, TxnEntryStore}; use tikv::storage::{Key, Statistics}; -use tikv_util::worker::{Runnable, RunnableWithTimer}; +use tikv_util::worker::Runnable; use tokio_threadpool::{Builder as ThreadPoolBuilder, ThreadPool}; use crate::metrics::*; @@ -141,12 +129,14 @@ pub struct Endpoint { impl Endpoint { pub fn new(store_id: u64, engine: E, region_info: R, db: Arc) -> Endpoint { - let workers = ThreadPoolBuilder::new().name_prefix("backworker").build(); + let workers = ThreadPoolBuilder::new() + .name_prefix("backworker") + .pool_size(8) // TODO: make it configure. + .build(); Endpoint { store_id, engine, region_info, - // TODO: support more config. workers, db, } @@ -326,7 +316,7 @@ impl Endpoint { response.set_end_key(end_key.clone()); match res { Ok((mut files, stat)) => { - info!("backup region finish"; + debug!("backup region finish"; "region" => ?brange.region, "start_key" => hex::encode_upper(&start_key), "end_key" => hex::encode_upper(&end_key), @@ -439,18 +429,14 @@ pub mod tests { use external_storage::LocalStorage; use futures::{Future, Stream}; use kvproto::metapb; - use std::collections::BTreeMap; - use std::sync::mpsc::{channel, Receiver, Sender}; use tempfile::TempDir; use tikv::raftstore::coprocessor::RegionCollector; - use tikv::raftstore::coprocessor::{RegionInfo, SeekRegionCallback}; + use tikv::raftstore::coprocessor::SeekRegionCallback; use tikv::raftstore::store::util::new_peer; use tikv::storage::kv::Result as EngineResult; use tikv::storage::mvcc::tests::*; use tikv::storage::SHORT_VALUE_MAX_LEN; - use tikv::storage::{ - Mutation, Options, RocksEngine, Storage, TestEngineBuilder, TestStorageBuilder, - }; + use tikv::storage::{RocksEngine, TestEngineBuilder}; #[derive(Clone)] pub struct MockRegionInfoProvider { diff --git a/components/backup/src/lib.rs b/components/backup/src/lib.rs index 8152bd89717..0b8f645a2e4 100644 --- a/components/backup/src/lib.rs +++ b/components/backup/src/lib.rs @@ -1,28 +1,10 @@ // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. #![recursion_limit = "200"] -// TODO: remove it after all code been merged. -#![allow(unused_imports)] -#[macro_use( - kv, - slog_kv, - slog_trace, - slog_debug, - slog_info, - slog_warn, - slog_error, - slog_record, - slog_b, - slog_log, - slog_record_static -)] -extern crate slog; #[macro_use] extern crate slog_global; #[macro_use] -extern crate serde_derive; -#[macro_use] extern crate failure; #[allow(unused_extern_crates)] extern crate tikv_alloc;