diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 5f7d027e739..cb893b32623 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -2,6 +2,7 @@ nebula_add_library( storage_common_obj OBJECT StorageFlags.cpp CommonUtils.cpp + LogMonitor.cpp ) nebula_add_library( diff --git a/src/storage/LogMonitor.cpp b/src/storage/LogMonitor.cpp new file mode 100644 index 00000000000..842b44d3ee8 --- /dev/null +++ b/src/storage/LogMonitor.cpp @@ -0,0 +1,49 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#include "storage/LogMonitor.h" + +namespace nebula { +namespace storage { + +DEFINE_uint64(log_min_reserved_bytes_to_warn, + 256 * (1UL << 20), + "if freebytes in logdir less than this, will change log level to WARN"); +DEFINE_uint64(log_min_reserved_bytes_to_error, + 64 * (1UL << 20), + "if freebytes in logdir less than this, will change log level to ERROR"); +DEFINE_uint64(log_min_reserved_bytes_to_fatal, + 4 * (1UL << 20), + "if freebytes in logdir less than this, will change log level to FATAL"); +DEFINE_int32(log_disk_check_interval_secs, 10, "interval to check free space of log path"); + +void LogMonitor::getLogDiskFreeByte() { + boost::system::error_code ec; + auto info = boost::filesystem::space(FLAGS_log_dir, ec); + if (!ec) { + freeByte_ = info.available; + } else { + LOG(WARNING) << "Get filesystem info of logdir: " << FLAGS_log_dir << " failed"; + } +} + +void LogMonitor::checkAndChangeLogLevel() { + getLogDiskFreeByte(); + + if (freeByte_ < FLAGS_log_min_reserved_bytes_to_warn) { + FLAGS_minloglevel = google::GLOG_WARNING; + } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_error) { + FLAGS_minloglevel = google::GLOG_ERROR; + } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_fatal) { + FLAGS_minloglevel = google::GLOG_FATAL; + } else { + // if freeByte_ is bigger than every min_log_flag, reset the FLAGS_minloglevel to old value + if (FLAGS_minloglevel != oldMinLogLevel_) { + FLAGS_minloglevel = oldMinLogLevel_; + } + } +} + +} // namespace storage +} // namespace nebula diff --git a/src/storage/LogMonitor.h b/src/storage/LogMonitor.h new file mode 100644 index 00000000000..1aa6cecdc2a --- /dev/null +++ b/src/storage/LogMonitor.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ +#pragma once + +#include +#include + +#include "common/thread/GenericWorker.h" + +namespace nebula { +namespace storage { + +DECLARE_uint64(log_min_reserved_bytes_to_warn); +DECLARE_uint64(log_min_reserved_bytes_to_error); +DECLARE_uint64(log_min_reserved_bytes_to_fatal); +DECLARE_int32(log_disk_check_interval_secs); + +class LogMonitor { + public: + LogMonitor() : oldMinLogLevel_(FLAGS_minloglevel), freeByte_(1UL << 60) { + worker_ = std::make_shared(); + CHECK(worker_->start()); + worker_->addRepeatTask( + FLAGS_log_disk_check_interval_secs * 1000, &LogMonitor::checkAndChangeLogLevel, this); + } + + ~LogMonitor() { + worker_->stop(); + worker_->wait(); + } + + void getLogDiskFreeByte(); + + void checkAndChangeLogLevel(); + + private: + int32_t oldMinLogLevel_; + std::shared_ptr worker_; + std::atomic_uint64_t freeByte_; +}; + +} // namespace storage + +} // namespace nebula diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp index 66b697031c3..cec1b59cbd0 100644 --- a/src/storage/StorageServer.cpp +++ b/src/storage/StorageServer.cpp @@ -198,6 +198,9 @@ bool StorageServer::start() { LOG(INFO) << "Init kvstore"; kvstore_ = getStoreInstance(); + LOG(INFO) << "Init LogMonitor"; + logMonitor_ = std::make_unique(); + if (nullptr == kvstore_) { LOG(ERROR) << "Init kvstore failed"; return false; diff --git a/src/storage/StorageServer.h b/src/storage/StorageServer.h index 2f907263ef9..b645a708d3d 100644 --- a/src/storage/StorageServer.h +++ b/src/storage/StorageServer.h @@ -16,6 +16,7 @@ #include "kvstore/NebulaStore.h" #include "storage/CommonUtils.h" #include "storage/GraphStorageLocalServer.h" +#include "storage/LogMonitor.h" #include "storage/admin/AdminTaskManager.h" #include "storage/transaction/TransactionManager.h" @@ -97,6 +98,8 @@ class StorageServer final { // used for communicate between one storaged to another std::unique_ptr interClient_; + std::unique_ptr logMonitor_; + ServiceStatus serverStatus_{STATUS_UNINITIALIZED}; std::mutex muStop_; std::condition_variable cvStop_;