*: rearrange loggers (#3987)

Signed-off-by: Neil Shen <overvenus@gmail.com>

components/test_util/src/lib.rs

@@ -16,7 +16,6 @@
 extern crate test;
 
 extern crate rand;
-#[macro_use]
 extern crate slog;
 extern crate slog_scope;
 extern crate time;
@@ -37,7 +36,7 @@ pub use security::*;
 
 pub fn setup_for_ci() {
     let guard = if env::var("CI").is_ok() && env::var("LOG_FILE").is_ok() {
-        Some(logging::init_log())
+        Some(logging::init_log_for_test())
     } else {
         None
     };

components/test_util/src/logging.rs

@@ -67,22 +67,40 @@ impl Drop for CaseTraceLogger {
 }
 
 // A help function to initial logger.
-pub fn init_log() -> GlobalLoggerGuard {
+pub fn init_log_for_test() -> GlobalLoggerGuard {
     let output = env::var("LOG_FILE").ok();
     let level = tikv::util::logger::get_level_by_string(
         &env::var("LOG_LEVEL").unwrap_or_else(|_| "debug".to_owned()),
     ).unwrap();
     let writer = output.map(|f| Mutex::new(File::create(f).unwrap()));
     // we don't mind set it multiple times.
     let drain = CaseTraceLogger { f: writer };
+
+    // Collects following targes.
+    const ENABLED_TARGETS: &[&str] = &[
+        "tikv::",
+        "tests::",
+        "benches::",
+        "integrations::",
+        "failpoints::",
+        "raft::",
+        // Collects logs for test components.
+        "test_",
+    ];
+    let filtered = drain.filter(|record| {
+        ENABLED_TARGETS
+            .iter()
+            .any(|target| record.module().starts_with(target))
+    });
+
     // CaseTraceLogger relies on test's thread name, however slog_async has
     // its own thread, and the name is "".
     // TODO: Enable the slog_async when the [Custom test frameworks][1] is mature,
     //       and hook the slog_async logger to every test cases.
     //
     // [1]: https://github.com/rust-lang/rfcs/blob/master/text/2318-custom-test-frameworks.md
-    //
-    // let drain = slog_async::Async::new(drain).build().fuse();
-    let logger = slog::Logger::root_typed(drain, slog_o!());
-    tikv::util::logger::init_log_for_tikv_only(logger, level).unwrap()
+    tikv::util::logger::init_log(
+        filtered, level, false, // disable async drainer
+        true,  // init std log
+    ).unwrap()
 }

src/bin/tikv-ctl.rs

@@ -22,24 +22,22 @@ extern crate kvproto;
 extern crate libc;
 #[macro_use]
 extern crate log;
-extern crate protobuf;
-extern crate raft;
-extern crate rocksdb;
-#[macro_use]
-extern crate tikv;
-extern crate toml;
-#[macro_use(slog_o, slog_kv)]
-extern crate slog;
 extern crate hex;
 #[cfg(unix)]
 extern crate nix;
+extern crate protobuf;
+extern crate raft;
 extern crate rand;
+extern crate rocksdb;
 #[cfg(unix)]
 extern crate signal;
+extern crate slog;
 extern crate slog_async;
 extern crate slog_scope;
 extern crate slog_stdlog;
 extern crate slog_term;
+extern crate tikv;
+extern crate toml;
 
 mod util;
 

src/bin/tikv-importer.rs

@@ -22,20 +22,18 @@ extern crate jemallocator;
 extern crate libc;
 #[macro_use]
 extern crate log;
-#[macro_use(slog_o, slog_kv)]
-extern crate slog;
 #[cfg(unix)]
 extern crate nix;
 extern crate prometheus;
 extern crate rocksdb;
 extern crate serde_json;
 #[cfg(unix)]
 extern crate signal;
+extern crate slog;
 extern crate slog_async;
 extern crate slog_scope;
 extern crate slog_stdlog;
 extern crate slog_term;
-#[macro_use]
 extern crate tikv;
 extern crate toml;
 
@@ -100,7 +98,7 @@ fn main() {
         .get_matches();
 
     let config = setup_config(&matches);
-    let guard = init_log(&config);
+    let guard = initial_logger(&config);
     tikv_util::set_exit_hook(false, Some(guard), &config.storage.data_dir);
 
     initial_metric(&config.metric, None);

src/bin/tikv-server.rs

@@ -22,21 +22,19 @@ extern crate jemallocator;
 extern crate libc;
 #[macro_use]
 extern crate log;
-#[macro_use(slog_o, slog_kv)]
-extern crate slog;
+extern crate hyper;
 #[cfg(unix)]
 extern crate nix;
 extern crate rocksdb;
 extern crate serde_json;
 #[cfg(unix)]
 extern crate signal;
+extern crate slog;
 extern crate slog_async;
 extern crate slog_scope;
 extern crate slog_stdlog;
 extern crate slog_term;
-#[macro_use]
 extern crate tikv;
-extern crate hyper;
 extern crate toml;
 
 #[cfg(unix)]
@@ -409,8 +407,8 @@ fn main() {
 
     // Sets the global logger ASAP.
     // It is okay to use the config w/o `validata()`,
-    // because `init_log()` handles various conditions.
-    let guard = init_log(&config);
+    // because `initial_logger()` handles various conditions.
+    let guard = initial_logger(&config);
     tikv_util::set_exit_hook(false, Some(guard), &config.storage.data_dir);
 
     // Print version information.

src/bin/util/setup.rs

@@ -12,31 +12,19 @@
 // limitations under the License.
 
 use std::env;
-use std::io::BufWriter;
 use std::process;
 use std::sync::atomic::{AtomicBool, Ordering, ATOMIC_BOOL_INIT};
 
 use chrono;
 use clap::ArgMatches;
-use slog::{Drain, Logger};
-use slog_async::{Async, OverflowStrategy};
 use slog_scope::GlobalLoggerGuard;
-use slog_term::{PlainDecorator, TermDecorator};
 
 use tikv::config::{MetricConfig, TiKvConfig};
-use tikv::util;
 use tikv::util::collections::HashMap;
-use tikv::util::file_log::RotatingFileLogger;
-use tikv::util::logger;
+use tikv::util::{self, logger};
 
 // A workaround for checking if log is initialized.
 pub static LOG_INITIALIZED: AtomicBool = ATOMIC_BOOL_INIT;
-// Default is 128.
-// Extended since blocking is set, and we don't want to block very often.
-const SLOG_CHANNEL_SIZE: usize = 10240;
-// Default is DropAndReport.
-// It is not desirable to have dropped logs in our use case.
-const SLOG_CHANNEL_OVERFLOW_STRATEGY: OverflowStrategy = OverflowStrategy::Block;
 
 macro_rules! fatal {
     ($lvl:expr, $($arg:tt)+) => ({
@@ -50,43 +38,27 @@ macro_rules! fatal {
 }
 
 #[allow(dead_code)]
-pub fn init_log(config: &TiKvConfig) -> GlobalLoggerGuard {
+pub fn initial_logger(config: &TiKvConfig) -> GlobalLoggerGuard {
     let log_rotation_timespan = chrono::Duration::from_std(
         config.log_rotation_timespan.clone().into(),
     ).expect("config.log_rotation_timespan is an invalid duration.");
     let guard = if config.log_file.is_empty() {
-        let decorator = TermDecorator::new().build();
-        let drain = logger::TikvFormat::new(decorator).fuse();
-        let drain = Async::new(drain)
-            .chan_size(SLOG_CHANNEL_SIZE)
-            .overflow_strategy(SLOG_CHANNEL_OVERFLOW_STRATEGY)
-            .thread_name(thd_name!("term-slogger"))
-            .build()
-            .fuse();
-        let logger = Logger::root_typed(drain, slog_o!());
-        logger::init_log(logger, config.log_level).unwrap_or_else(|e| {
+        let drainer = logger::term_drainer();
+        // use async drainer and init std log.
+        logger::init_log(drainer, config.log_level, true, true).unwrap_or_else(|e| {
             fatal!("failed to initialize log: {:?}", e);
         })
     } else {
-        let logger = BufWriter::new(
-            RotatingFileLogger::new(&config.log_file, log_rotation_timespan).unwrap_or_else(|e| {
+        let drainer =
+            logger::file_drainer(&config.log_file, log_rotation_timespan).unwrap_or_else(|e| {
                 fatal!(
                     "failed to initialize log with file {:?}: {:?}",
                     config.log_file,
                     e
                 );
-            }),
-        );
-        let decorator = PlainDecorator::new(logger);
-        let drain = logger::TikvFormat::new(decorator).fuse();
-        let drain = Async::new(drain)
-            .chan_size(SLOG_CHANNEL_SIZE)
-            .overflow_strategy(SLOG_CHANNEL_OVERFLOW_STRATEGY)
-            .thread_name(thd_name!("file-slogger"))
-            .build()
-            .fuse();
-        let logger = Logger::root_typed(drain, slog_o!());
-        logger::init_log(logger, config.log_level).unwrap_or_else(|e| {
+            });
+        // use async drainer and init std log.
+        logger::init_log(drainer, config.log_level, true, true).unwrap_or_else(|e| {
             fatal!("failed to initialize log: {:?}", e);
         })
     };