From 48bfca551d1a7932ddc1095ae065cb0103bf1ab4 Mon Sep 17 00:00:00 2001 From: Nivras <12605142+Nivras@users.noreply.github.com> Date: Tue, 28 Dec 2021 15:28:55 +0800 Subject: [PATCH 1/3] add LogMonitor to check the log disk is full --- src/common/CMakeLists.txt | 1 + src/common/log/CMakeLists.txt | 8 ++++++ src/common/log/LogMonitor.cpp | 53 +++++++++++++++++++++++++++++++++++ src/common/log/LogMonitor.h | 43 ++++++++++++++++++++++++++++ src/daemons/CMakeLists.txt | 1 + src/storage/StorageServer.cpp | 3 ++ src/storage/StorageServer.h | 3 ++ 7 files changed, 112 insertions(+) create mode 100644 src/common/log/CMakeLists.txt create mode 100644 src/common/log/LogMonitor.cpp create mode 100644 src/common/log/LogMonitor.h diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index a3470a8baa0..b0abdc68f49 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -28,3 +28,4 @@ nebula_add_subdirectory(ssl) nebula_add_subdirectory(geo) nebula_add_subdirectory(memory) nebula_add_subdirectory(id) +nebula_add_subdirectory(log) diff --git a/src/common/log/CMakeLists.txt b/src/common/log/CMakeLists.txt new file mode 100644 index 00000000000..633a091d1eb --- /dev/null +++ b/src/common/log/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. + +nebula_add_library( + log_monitor_obj OBJECT + LogMonitor.cpp +) diff --git a/src/common/log/LogMonitor.cpp b/src/common/log/LogMonitor.cpp new file mode 100644 index 00000000000..da6f826463a --- /dev/null +++ b/src/common/log/LogMonitor.cpp @@ -0,0 +1,53 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include "common/log/LogMonitor.h" + +namespace nebula { + +DEFINE_uint64(log_min_reserved_bytes_to_warn, + 256 * (1UL << 20), + "if freebytes in logdir less than this, will change log level to WARN"); +DEFINE_uint64(log_min_reserved_bytes_to_error, + 64 * (1UL << 20), + "if freebytes in logdir less than this, will change log level to ERROR"); +DEFINE_uint64(log_min_reserved_bytes_to_fatal, + 4 * (1UL << 20), + "if freebytes in logdir less than this, will change log level to FATAL"); +DEFINE_int32(log_disk_check_interval_secs, 10, "interval to check free space of log path"); + +void LogMonitor::getLogDiskFreeByte() { + boost::system::error_code ec; + auto info = boost::filesystem::space(FLAGS_log_dir, ec); + if (!ec) { + freeByte_ = info.available; + } else { + LOG(WARNING) << "Get filesystem info of logdir: " << FLAGS_log_dir << " failed"; + } +} + +void LogMonitor::checkAndChangeLogLevel() { + getLogDiskFreeByte(); + + if (freeByte_ < FLAGS_log_min_reserved_bytes_to_warn) { + LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_warn + << ", change log level to WARNING"; + FLAGS_minloglevel = google::GLOG_WARNING; + } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_error) { + LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_error + << ", change log level to ERROR"; + FLAGS_minloglevel = google::GLOG_ERROR; + } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_fatal) { + LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_fatal + << ", change log level to FATAL"; + FLAGS_minloglevel = google::GLOG_FATAL; + } else { + // if freeByte_ is bigger than every min_log_flag, reset the FLAGS_minloglevel to old value + if (FLAGS_minloglevel != oldMinLogLevel_) { + FLAGS_minloglevel = oldMinLogLevel_; + } + } +} + +} // namespace nebula diff --git a/src/common/log/LogMonitor.h b/src/common/log/LogMonitor.h new file mode 100644 index 00000000000..86e206e1930 --- /dev/null +++ b/src/common/log/LogMonitor.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#pragma once + +#include +#include + +#include "common/thread/GenericWorker.h" + +namespace nebula { + +DECLARE_uint64(log_min_reserved_bytes_to_warn); +DECLARE_uint64(log_min_reserved_bytes_to_error); +DECLARE_uint64(log_min_reserved_bytes_to_fatal); +DECLARE_int32(log_disk_check_interval_secs); + +class LogMonitor { + public: + LogMonitor() : oldMinLogLevel_(FLAGS_minloglevel), freeByte_(1UL << 60) { + worker_ = std::make_shared(); + CHECK(worker_->start()); + worker_->addRepeatTask( + FLAGS_log_disk_check_interval_secs * 1000, &LogMonitor::checkAndChangeLogLevel, this); + } + + ~LogMonitor() { + worker_->stop(); + worker_->wait(); + } + + void getLogDiskFreeByte(); + + void checkAndChangeLogLevel(); + + private: + int32_t oldMinLogLevel_; + std::shared_ptr worker_; + std::atomic_uint64_t freeByte_; +}; + +} // namespace nebula diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 1d476baf543..104acf8de20 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -34,6 +34,7 @@ set(common_deps $ $ $ + $ ) set(storage_meta_deps diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp index 66b697031c3..60a06b14f26 100644 --- a/src/storage/StorageServer.cpp +++ b/src/storage/StorageServer.cpp @@ -198,6 +198,9 @@ bool StorageServer::start() { LOG(INFO) << "Init kvstore"; kvstore_ = getStoreInstance(); + LOG(INFO) << "Init LogMonitor"; + logMonitor_ = std::make_unique(); + if (nullptr == kvstore_) { LOG(ERROR) << "Init kvstore failed"; return false; diff --git a/src/storage/StorageServer.h b/src/storage/StorageServer.h index 2f907263ef9..3637f6f9091 100644 --- a/src/storage/StorageServer.h +++ b/src/storage/StorageServer.h @@ -11,6 +11,7 @@ #include "clients/meta/MetaClient.h" #include "common/base/Base.h" #include "common/hdfs/HdfsHelper.h" +#include "common/log/LogMonitor.h" #include "common/meta/IndexManager.h" #include "common/meta/SchemaManager.h" #include "kvstore/NebulaStore.h" @@ -97,6 +98,8 @@ class StorageServer final { // used for communicate between one storaged to another std::unique_ptr interClient_; + std::unique_ptr logMonitor_; + ServiceStatus serverStatus_{STATUS_UNINITIALIZED}; std::mutex muStop_; std::condition_variable cvStop_; From 492e883594f4bbf2ded174bf72b28346ff40c40e Mon Sep 17 00:00:00 2001 From: Nivras <12605142+Nivras@users.noreply.github.com> Date: Tue, 4 Jan 2022 10:55:13 +0800 Subject: [PATCH 2/3] fix comments --- src/common/log/LogMonitor.cpp | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/common/log/LogMonitor.cpp b/src/common/log/LogMonitor.cpp index da6f826463a..71869746420 100644 --- a/src/common/log/LogMonitor.cpp +++ b/src/common/log/LogMonitor.cpp @@ -30,18 +30,27 @@ void LogMonitor::getLogDiskFreeByte() { void LogMonitor::checkAndChangeLogLevel() { getLogDiskFreeByte(); - if (freeByte_ < FLAGS_log_min_reserved_bytes_to_warn) { - LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_warn - << ", change log level to WARNING"; - FLAGS_minloglevel = google::GLOG_WARNING; + if (FLAGS_log_min_reserved_bytes_to_fatal > FLAGS_log_min_reserved_bytes_to_error || + FLAGS_log_min_reserved_bytes_to_fatal > FLAGS_log_min_reserved_bytes_to_warn || + FLAGS_log_min_reserved_bytes_to_error > FLAGS_log_min_reserved_bytes_to_warn) { + LOG(ERROR) << "Get Invalid config in LogMonitor, the LogMonitor config should be " + << "FLAGS_log_min_reserved_bytes_to_warn >" + << "FLAGS_log_min_reserved_bytes_to_error > FLAGS_log_min_reserved_bytes_to_fatal;"; + return; + } + + if (freeByte_ < FLAGS_log_min_reserved_bytes_to_fatal) { + LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_fatal + << ", change log level to FATAL"; + FLAGS_minloglevel = google::GLOG_FATAL; } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_error) { LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_error << ", change log level to ERROR"; FLAGS_minloglevel = google::GLOG_ERROR; - } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_fatal) { - LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_fatal - << ", change log level to FATAL"; - FLAGS_minloglevel = google::GLOG_FATAL; + } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_warn) { + LOG(ERROR) << "log disk freebyte is less than " << FLAGS_log_min_reserved_bytes_to_warn + << ", change log level to WARNING"; + FLAGS_minloglevel = google::GLOG_WARNING; } else { // if freeByte_ is bigger than every min_log_flag, reset the FLAGS_minloglevel to old value if (FLAGS_minloglevel != oldMinLogLevel_) { From 089ad1d7ec7eef51b2c68dea5f9fc63ee9cd76a8 Mon Sep 17 00:00:00 2001 From: Nivras <12605142+Nivras@users.noreply.github.com> Date: Thu, 6 Jan 2022 17:23:38 +0800 Subject: [PATCH 3/3] add comments for default flags --- src/common/log/LogMonitor.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/common/log/LogMonitor.cpp b/src/common/log/LogMonitor.cpp index 71869746420..30f5f995af1 100644 --- a/src/common/log/LogMonitor.cpp +++ b/src/common/log/LogMonitor.cpp @@ -6,15 +6,22 @@ namespace nebula { +// default min_warn is 256M, disk freebytes < 256M will change LOG_LEVEL to WARNING DEFINE_uint64(log_min_reserved_bytes_to_warn, 256 * (1UL << 20), "if freebytes in logdir less than this, will change log level to WARN"); + +// default min_error is 64M, disk freebytes < 64M will change LOG_LEVEL to ERROR DEFINE_uint64(log_min_reserved_bytes_to_error, 64 * (1UL << 20), "if freebytes in logdir less than this, will change log level to ERROR"); + +// default min_fatal is 4M, disk freebytes < 4M will change LOG_LEVEL to FATAL DEFINE_uint64(log_min_reserved_bytes_to_fatal, 4 * (1UL << 20), "if freebytes in logdir less than this, will change log level to FATAL"); + +// default check log_disk interval is 10s DEFINE_int32(log_disk_check_interval_secs, 10, "interval to check free space of log path"); void LogMonitor::getLogDiskFreeByte() {