From 596b4be8ebae8d4b8d0a14ac2c7396a437774cd7 Mon Sep 17 00:00:00 2001
From: Nivras <12605142+Nivras@users.noreply.github.com>
Date: Tue, 28 Dec 2021 15:28:55 +0800
Subject: [PATCH] add LogMonitor to check the log disk is full

---
 src/storage/CMakeLists.txt    |  1 +
 src/storage/LogMonitor.cpp    | 49 +++++++++++++++++++++++++++++++++++
 src/storage/LogMonitor.h      | 46 ++++++++++++++++++++++++++++++++
 src/storage/StorageServer.cpp |  3 +++
 src/storage/StorageServer.h   |  3 +++
 5 files changed, 102 insertions(+)
 create mode 100644 src/storage/LogMonitor.cpp
 create mode 100644 src/storage/LogMonitor.h

diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt
index b4b71d961a7..5e579c7d63a 100644
--- a/src/storage/CMakeLists.txt
+++ b/src/storage/CMakeLists.txt
@@ -2,6 +2,7 @@ nebula_add_library(
     storage_common_obj OBJECT
     StorageFlags.cpp
     CommonUtils.cpp
+    LogMonitor.cpp
 )
 
 nebula_add_library(
diff --git a/src/storage/LogMonitor.cpp b/src/storage/LogMonitor.cpp
new file mode 100644
index 00000000000..842b44d3ee8
--- /dev/null
+++ b/src/storage/LogMonitor.cpp
@@ -0,0 +1,49 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+#include "storage/LogMonitor.h"
+
+namespace nebula {
+namespace storage {
+
+DEFINE_uint64(log_min_reserved_bytes_to_warn,
+              256 * (1UL << 20),
+              "if freebytes in logdir less than this, will change log level to WARN");
+DEFINE_uint64(log_min_reserved_bytes_to_error,
+              64 * (1UL << 20),
+              "if freebytes in logdir less than this, will change log level to ERROR");
+DEFINE_uint64(log_min_reserved_bytes_to_fatal,
+              4 * (1UL << 20),
+              "if freebytes in logdir less than this, will change log level to FATAL");
+DEFINE_int32(log_disk_check_interval_secs, 10, "interval to check free space of log path");
+
+void LogMonitor::getLogDiskFreeByte() {
+  boost::system::error_code ec;
+  auto info = boost::filesystem::space(FLAGS_log_dir, ec);
+  if (!ec) {
+    freeByte_ = info.available;
+  } else {
+    LOG(WARNING) << "Get filesystem info of logdir: " << FLAGS_log_dir << " failed";
+  }
+}
+
+void LogMonitor::checkAndChangeLogLevel() {
+  getLogDiskFreeByte();
+
+  if (freeByte_ < FLAGS_log_min_reserved_bytes_to_warn) {
+    FLAGS_minloglevel = google::GLOG_WARNING;
+  } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_error) {
+    FLAGS_minloglevel = google::GLOG_ERROR;
+  } else if (freeByte_ < FLAGS_log_min_reserved_bytes_to_fatal) {
+    FLAGS_minloglevel = google::GLOG_FATAL;
+  } else {
+    // if freeByte_ is bigger than every min_log_flag, reset the FLAGS_minloglevel to old value
+    if (FLAGS_minloglevel != oldMinLogLevel_) {
+      FLAGS_minloglevel = oldMinLogLevel_;
+    }
+  }
+}
+
+}  // namespace storage
+}  // namespace nebula
diff --git a/src/storage/LogMonitor.h b/src/storage/LogMonitor.h
new file mode 100644
index 00000000000..1aa6cecdc2a
--- /dev/null
+++ b/src/storage/LogMonitor.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+#pragma once
+
+#include <boost/filesystem.hpp>
+#include <boost/system/error_code.hpp>
+
+#include "common/thread/GenericWorker.h"
+
+namespace nebula {
+namespace storage {
+
+DECLARE_uint64(log_min_reserved_bytes_to_warn);
+DECLARE_uint64(log_min_reserved_bytes_to_error);
+DECLARE_uint64(log_min_reserved_bytes_to_fatal);
+DECLARE_int32(log_disk_check_interval_secs);
+
+class LogMonitor {
+ public:
+  LogMonitor() : oldMinLogLevel_(FLAGS_minloglevel), freeByte_(1UL << 60) {
+    worker_ = std::make_shared<thread::GenericWorker>();
+    CHECK(worker_->start());
+    worker_->addRepeatTask(
+        FLAGS_log_disk_check_interval_secs * 1000, &LogMonitor::checkAndChangeLogLevel, this);
+  }
+
+  ~LogMonitor() {
+    worker_->stop();
+    worker_->wait();
+  }
+
+  void getLogDiskFreeByte();
+
+  void checkAndChangeLogLevel();
+
+ private:
+  int32_t oldMinLogLevel_;
+  std::shared_ptr<thread::GenericWorker> worker_;
+  std::atomic_uint64_t freeByte_;
+};
+
+}  // namespace storage
+
+}  // namespace nebula
diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp
index f29fdb490a1..3ba03a4fbac 100644
--- a/src/storage/StorageServer.cpp
+++ b/src/storage/StorageServer.cpp
@@ -182,6 +182,9 @@ bool StorageServer::start() {
   LOG(INFO) << "Init kvstore";
   kvstore_ = getStoreInstance();
 
+  LOG(INFO) << "Init LogMonitor";
+  logMonitor_ = std::make_unique<storage::LogMonitor>();
+
   if (nullptr == kvstore_) {
     LOG(ERROR) << "Init kvstore failed";
     return false;
diff --git a/src/storage/StorageServer.h b/src/storage/StorageServer.h
index 27795c5eaf5..a31b17041ef 100644
--- a/src/storage/StorageServer.h
+++ b/src/storage/StorageServer.h
@@ -15,6 +15,7 @@
 #include "common/meta/SchemaManager.h"
 #include "kvstore/NebulaStore.h"
 #include "storage/CommonUtils.h"
+#include "storage/LogMonitor.h"
 #include "storage/admin/AdminTaskManager.h"
 #include "storage/transaction/TransactionManager.h"
 
@@ -92,6 +93,8 @@ class StorageServer final {
   // used for communicate between one storaged to another
   std::unique_ptr<InternalStorageClient> interClient_;
 
+  std::unique_ptr<storage::LogMonitor> logMonitor_;
+
   ServiceStatus serverStatus_{STATUS_UNINITIALIZED};
   std::mutex muStop_;
   std::condition_variable cvStop_;