From 4ede36eb26ea11cbf2ec19edfdc006fc2919459d Mon Sep 17 00:00:00 2001 From: CameronNg Date: Mon, 9 Sep 2024 14:38:09 +0700 Subject: [PATCH 1/7] feat-logger-update --- engine/CMakeLists.txt | 2 + engine/main.cc | 26 ++++- engine/utils/config_yaml_utils.h | 4 +- engine/utils/cortex_utils.h | 4 +- engine/utils/file_logger.cc | 175 ++++++++++++++++++++++++++++++ engine/utils/file_logger.h | 68 ++++++++++++ engine/utils/file_manager_utils.h | 22 ++++ engine/utils/logging_utils.h | 1 - 8 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 engine/utils/file_logger.cc create mode 100644 engine/utils/file_logger.h diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index c90be9f48..48f5a5ead 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -115,6 +115,7 @@ if(DEFINED CMAKE_JS_INC) add_library(${PROJECT_NAME} SHARED addon.cc ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc + ${CMAKE_CURRENT_SOURCE_DIR}/utils/file_logger.cc ${CMAKE_JS_SRC} ) @@ -131,6 +132,7 @@ if(DEFINED CMAKE_JS_INC) else() # Official build add_executable(${PROJECT_NAME} main.cc ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc + ${CMAKE_CURRENT_SOURCE_DIR}/utils/file_logger.cc ) endif() diff --git a/engine/main.cc b/engine/main.cc index 5c83d1103..70e5eec08 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -6,6 +6,7 @@ #include "utils/archive_utils.h" #include "utils/cortex_utils.h" #include "utils/dylib.h" +#include "utils/file_logger.h" #include "utils/file_manager_utils.h" #include "utils/logging_utils.h" @@ -29,16 +30,19 @@ void RunServer() { LOG_INFO << "Host: " << config.host << " Port: " << config.port << "\n"; // Create logs/ folder and setup log to file - std::filesystem::create_directory(cortex_utils::logs_folder); - trantor::AsyncFileLogger asyncFileLogger; - asyncFileLogger.setFileName(cortex_utils::logs_base_name); + std::filesystem::create_directory(config.logFolderPath + "/" + + cortex_utils::logs_folder); + trantor::FileLogger asyncFileLogger; + asyncFileLogger.setFileName(config.logFolderPath + "/" + + cortex_utils::logs_base_name); + asyncFileLogger.setMaxLines( + cortex_utils::log_file_max_lines); // Keep last 100000 lines asyncFileLogger.startLogging(); trantor::Logger::setOutputFunction( [&](const char* msg, const uint64_t len) { - asyncFileLogger.output(msg, len); + asyncFileLogger.output_(msg, len); }, [&]() { asyncFileLogger.flush(); }); - asyncFileLogger.setFileSizeLimit(cortex_utils::log_file_size_limit); // Number of cortex.cpp threads // if (argc > 1) { // thread_num = std::atoi(argv[1]); @@ -154,6 +158,18 @@ int main(int argc, char* argv[]) { RunServer(); return 0; } else { + auto config = file_manager_utils::GetCortexConfig(); + trantor::FileLogger asyncFileLogger; + asyncFileLogger.setFileName(config.logFolderPath + "/" + + cortex_utils::logs_cli_base_name); + asyncFileLogger.setMaxLines( + cortex_utils::log_file_max_lines); // Keep last 1 million lines + asyncFileLogger.startLogging(); + trantor::Logger::setOutputFunction( + [&](const char* msg, const uint64_t len) { + asyncFileLogger.output_(msg, len); + }, + [&]() { asyncFileLogger.flush(); }); CommandLineParser clp; clp.SetupCommand(argc, argv); return 0; diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 4330f3527..04c435171 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -8,6 +8,7 @@ namespace config_yaml_utils { struct CortexConfig { + std::string logFolderPath; std::string dataFolderPath; std::string host; std::string port; @@ -27,6 +28,7 @@ inline void DumpYamlConfig(const CortexConfig& config, throw std::runtime_error("Failed to open output file."); } YAML::Node node; + node["logFolderPath"] = config.logFolderPath; node["dataFolderPath"] = config.dataFolderPath; node["host"] = config.host; node["port"] = config.port; @@ -49,6 +51,7 @@ inline CortexConfig FromYaml(const std::string& path, try { auto node = YAML::LoadFile(config_file_path.string()); CortexConfig config = { + .logFolderPath = node["logFolderPath"].as(), .dataFolderPath = node["dataFolderPath"].as(), .host = node["host"].as(), .port = node["port"].as(), @@ -60,5 +63,4 @@ inline CortexConfig FromYaml(const std::string& path, } } - } // namespace config_yaml_utils diff --git a/engine/utils/cortex_utils.h b/engine/utils/cortex_utils.h index 32dea9321..1772a58a9 100644 --- a/engine/utils/cortex_utils.h +++ b/engine/utils/cortex_utils.h @@ -34,8 +34,10 @@ constexpr static auto kTensorrtLlmPath = "/engines/cortex.tensorrt-llm"; inline std::string models_folder = "./models"; inline std::string logs_folder = "./logs"; -inline std::string logs_base_name = "./logs/cortex"; +inline std::string logs_base_name = "./logs/cortex.log"; +inline std::string logs_cli_base_name = "./logs/cortex-cli.log"; inline size_t log_file_size_limit = 20000000; // ~20 mb +inline size_t log_file_max_lines = 100000; // 100,000 lines inline std::string extractBase64(const std::string& input) { std::regex pattern("base64,(.*)"); diff --git a/engine/utils/file_logger.cc b/engine/utils/file_logger.cc new file mode 100644 index 000000000..edd20c94f --- /dev/null +++ b/engine/utils/file_logger.cc @@ -0,0 +1,175 @@ +#include "file_logger.h" +#include +#include +#include + +#ifdef _WIN32 +#include +#define ftruncate _chsize +#else +#include +#endif +#include + +using namespace trantor; + +FileLogger::FileLogger() : AsyncFileLogger() {} + +FileLogger::~FileLogger() = default; + +void FileLogger::output_(const char* msg, const uint64_t len) { + if (!circular_log_file_ptr_) { + circular_log_file_ptr_ = + std::make_unique(fileBaseName_, max_lines_); + } + circular_log_file_ptr_->writeLog(msg, len); +} + +FileLogger::CircularLogFile::CircularLogFile(const std::string& fileName, + uint64_t maxLines) + : max_lines_(maxLines), file_name_(fileName) { + OpenFile(); + LoadExistingLines(); + TruncateFileIfNeeded(); +} + +FileLogger::CircularLogFile::~CircularLogFile() { + CloseFile(); +} +void FileLogger::CircularLogFile::writeLog(const char* logLine, + const uint64_t len) { + if (!fp_) + return; + + std::string logString(logLine, len); + std::istringstream iss(logString); + std::string line; + while (std::getline(iss, line)) { + if (totalLines_ >= max_lines_) { + lineBuffer_.pop_front(); + --totalLines_; + } + lineBuffer_.push_back(line); + ++totalLines_; + AppendToFile(line + "\n"); + ++linesWrittenSinceLastTruncate_; + if (linesWrittenSinceLastTruncate_ >= TRUNCATE_CHECK_INTERVAL) { + + TruncateFileIfNeeded(); + } + } +} +void FileLogger::CircularLogFile::flush() { + if (fp_) { + fflush(fp_); + } +} + +void FileLogger::CircularLogFile::TruncateFileIfNeeded() { + // std::cout<<"Truncating file "<< totalLines_ < max_lines_ ? lineBuffer_.size() - max_lines_ : 0; + + for (size_t i = startIndex; i < lineBuffer_.size(); ++i) { + fprintf(tempFile, "%s\n", lineBuffer_[i].c_str()); + } + + fclose(tempFile); + + // Replace the original file with the temporary file + if (std::rename(tempFileName.c_str(), file_name_.c_str()) != 0) { + std::cout << "Error replacing original file with truncated file: " + << strerror(errno) << std::endl; + std::remove(tempFileName.c_str()); // Clean up the temporary file + } else { + // std::cout<<"Truncating file" < max_lines_ ? max_lines_ : lineBuffer_.size(); + } + + // Reopen the file + OpenFile(); + // LoadExistingLines(); + linesWrittenSinceLastTruncate_ = 0; +} + +void FileLogger::CircularLogFile::OpenFile() { +#ifdef _WIN32 + auto wFileName = utils::toNativePath(file_name_); + fp_ = _wfopen(wFileName.c_str(), L"r+"); +#else + fp_ = fopen(file_name_.c_str(), "r+"); +#endif + + if (!fp_) { +// If file doesn't exist, create it +#ifdef _WIN32 + fp_ = _wfopen(wFileName.c_str(), L"w+"); +#else + fp_ = fopen(file_name_.c_str(), "w+"); +#endif + + if (!fp_) { + std::cerr << "Error opening file: " << strerror(errno) << std::endl; + } + } +} +void FileLogger::CircularLogFile::LoadExistingLines() { + if (!fp_) + return; + + // Move to the beginning of the file + fseek(fp_, 0, SEEK_SET); + + lineBuffer_.clear(); + totalLines_ = 0; + + std::string line; + char buffer[4096]; + while (fgets(buffer, sizeof(buffer), fp_) != nullptr) { + line = buffer; + if (!line.empty() && line.back() == '\n') { + line.pop_back(); // Remove trailing newline + } + if (totalLines_ >= max_lines_) { + lineBuffer_.pop_front(); + } + lineBuffer_.push_back(line); + ++totalLines_; + } + + // Move back to the end of the file for appending + fseek(fp_, 0, SEEK_END); +} +void FileLogger::CircularLogFile::AppendToFile(const std::string& line) { + if (fp_) { + fwrite(line.c_str(), 1, line.length(), fp_); + fflush(fp_); + } +} + +void FileLogger::CircularLogFile::CloseFile() { + if (fp_) { + fclose(fp_); + fp_ = nullptr; + } +} \ No newline at end of file diff --git a/engine/utils/file_logger.h b/engine/utils/file_logger.h new file mode 100644 index 000000000..941899f21 --- /dev/null +++ b/engine/utils/file_logger.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace trantor { + +class TRANTOR_EXPORT FileLogger : public AsyncFileLogger { + public: + FileLogger(); + ~FileLogger(); + + /** + * @brief Set the maximum number of lines to keep in the log file. + * + * @param maxLines + */ + void setMaxLines(uint64_t maxLines) { max_lines_ = maxLines; } + + /** + * @brief Set the log file name. + * + * @param fileName The full name of the log file. + */ + void setFileName(const std::string& fileName) { + filePath_ = "./"; + fileBaseName_ = fileName; + fileExtName_ = ""; + } + void output_(const char* msg, const uint64_t len); + + protected: + class CircularLogFile { + public: + CircularLogFile(const std::string& fileName, uint64_t maxLines); + ~CircularLogFile(); + + void writeLog(const char* logLine, const uint64_t len); + void flush(); + uint64_t getLength() const { return totalLines_; } + + private: + FILE* fp_{nullptr}; + uint64_t max_lines_; + uint64_t totalLines_{0}; + std::string file_name_; + std::deque lineBuffer_; + uint64_t linesWrittenSinceLastTruncate_{0}; + static const uint64_t TRUNCATE_CHECK_INTERVAL = 1000; + + void LoadExistingLines(); + void TruncateFileIfNeeded(); + void AppendToFile(const std::string& line); + void OpenFile(); + void CloseFile(); + }; + std::unique_ptr circular_log_file_ptr_; + uint64_t max_lines_{100000}; // Default to 100000 lines +}; + +} // namespace trantor \ No newline at end of file diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index d20a39f13..b281a3b7a 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -135,6 +135,7 @@ inline void CreateConfigFileIfNotExist() { CTL_INF("Default data folder path: " + defaultDataFolderPath.string()); auto config = config_yaml_utils::CortexConfig{ + .logFolderPath = defaultDataFolderPath.string(), .dataFolderPath = defaultDataFolderPath.string(), .host = config_yaml_utils::kDefaultHost, .port = config_yaml_utils::kDefaultPort, @@ -169,6 +170,27 @@ inline std::filesystem::path GetCortexDataPath() { return data_folder_path; } +inline std::filesystem::path GetCortexLogPath() { + // TODO: We will need to support user to move the data folder to other place. + // TODO: get the variant of cortex. As discussed, we will have: prod, beta, nightly + // currently we will store cortex data at ~/cortexcpp + auto config = GetCortexConfig(); + std::filesystem::path log_folder_path; + if (!config.logFolderPath.empty()) { + log_folder_path = std::filesystem::path(config.logFolderPath); + } else { + auto home_path = GetHomeDirectoryPath(); + log_folder_path = home_path / config_yaml_utils::kCortexFolderName; + } + + if (!std::filesystem::exists(log_folder_path)) { + CTL_INF("Cortex log folder not found. Create one: " + + log_folder_path.string()); + std::filesystem::create_directory(log_folder_path); + } + return log_folder_path; +} + inline std::filesystem::path GetModelsContainerPath() { auto cortex_path = GetCortexDataPath(); auto models_container_path = cortex_path / "models"; diff --git a/engine/utils/logging_utils.h b/engine/utils/logging_utils.h index 77311c13e..fcaa3f4bb 100644 --- a/engine/utils/logging_utils.h +++ b/engine/utils/logging_utils.h @@ -30,4 +30,3 @@ inline bool log_verbose = false; } else { \ std::cout << msg << std::endl; \ } - From b6dd78f811cfd51172198ae8abcfb702dbfc6c2c Mon Sep 17 00:00:00 2001 From: CameronNg Date: Mon, 9 Sep 2024 15:34:01 +0700 Subject: [PATCH 2/7] feat-update-logger move max line to .cortexrc --- engine/main.cc | 5 ++--- engine/utils/config_yaml_utils.h | 10 ++++++++++ engine/utils/cortex_utils.h | 2 -- engine/utils/file_manager_utils.h | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/engine/main.cc b/engine/main.cc index 70e5eec08..df6b0a064 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -35,8 +35,7 @@ void RunServer() { trantor::FileLogger asyncFileLogger; asyncFileLogger.setFileName(config.logFolderPath + "/" + cortex_utils::logs_base_name); - asyncFileLogger.setMaxLines( - cortex_utils::log_file_max_lines); // Keep last 100000 lines + asyncFileLogger.setMaxLines(config.maxLogLines); // Keep last 100000 lines asyncFileLogger.startLogging(); trantor::Logger::setOutputFunction( [&](const char* msg, const uint64_t len) { @@ -163,7 +162,7 @@ int main(int argc, char* argv[]) { asyncFileLogger.setFileName(config.logFolderPath + "/" + cortex_utils::logs_cli_base_name); asyncFileLogger.setMaxLines( - cortex_utils::log_file_max_lines); // Keep last 1 million lines + config.maxLogLines); // Keep last 100000 lines asyncFileLogger.startLogging(); trantor::Logger::setOutputFunction( [&](const char* msg, const uint64_t len) { diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 04c435171..88fb7171f 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -12,11 +12,13 @@ struct CortexConfig { std::string dataFolderPath; std::string host; std::string port; + int maxLogLines; }; const std::string kCortexFolderName = "cortexcpp"; const std::string kDefaultHost{"127.0.0.1"}; const std::string kDefaultPort{"3928"}; +const int kDefaultMaxLines{100000}; inline void DumpYamlConfig(const CortexConfig& config, const std::string& path) { @@ -32,6 +34,7 @@ inline void DumpYamlConfig(const CortexConfig& config, node["dataFolderPath"] = config.dataFolderPath; node["host"] = config.host; node["port"] = config.port; + node["maxLogLines"] = config.maxLogLines; out_file << node; out_file.close(); @@ -50,11 +53,18 @@ inline CortexConfig FromYaml(const std::string& path, try { auto node = YAML::LoadFile(config_file_path.string()); + int max_lines_; + if (!node["maxLogLines"]) { + max_lines_ = kDefaultMaxLines; + } else { + max_lines_ = node["maxLogLines"].as(); + } CortexConfig config = { .logFolderPath = node["logFolderPath"].as(), .dataFolderPath = node["dataFolderPath"].as(), .host = node["host"].as(), .port = node["port"].as(), + .maxLogLines = max_lines_, }; return config; } catch (const YAML::BadFile& e) { diff --git a/engine/utils/cortex_utils.h b/engine/utils/cortex_utils.h index 1772a58a9..9673f0c1a 100644 --- a/engine/utils/cortex_utils.h +++ b/engine/utils/cortex_utils.h @@ -36,8 +36,6 @@ inline std::string models_folder = "./models"; inline std::string logs_folder = "./logs"; inline std::string logs_base_name = "./logs/cortex.log"; inline std::string logs_cli_base_name = "./logs/cortex-cli.log"; -inline size_t log_file_size_limit = 20000000; // ~20 mb -inline size_t log_file_max_lines = 100000; // 100,000 lines inline std::string extractBase64(const std::string& input) { std::regex pattern("base64,(.*)"); diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index b281a3b7a..1ee87b3dd 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -139,6 +139,7 @@ inline void CreateConfigFileIfNotExist() { .dataFolderPath = defaultDataFolderPath.string(), .host = config_yaml_utils::kDefaultHost, .port = config_yaml_utils::kDefaultPort, + .maxLogLines = config_yaml_utils::kDefaultMaxLines, }; DumpYamlConfig(config, config_path.string()); } From 8408543797c30992f9df595cf8a4d235a865e1d9 Mon Sep 17 00:00:00 2001 From: CameronNg Date: Tue, 10 Sep 2024 08:33:05 +0700 Subject: [PATCH 3/7] feat-update-logger add mutex for thread safe --- engine/utils/file_logger.cc | 4 ++++ engine/utils/file_logger.h | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/engine/utils/file_logger.cc b/engine/utils/file_logger.cc index edd20c94f..136a7b788 100644 --- a/engine/utils/file_logger.cc +++ b/engine/utils/file_logger.cc @@ -28,16 +28,19 @@ void FileLogger::output_(const char* msg, const uint64_t len) { FileLogger::CircularLogFile::CircularLogFile(const std::string& fileName, uint64_t maxLines) : max_lines_(maxLines), file_name_(fileName) { + std::lock_guard lock(mutex_); OpenFile(); LoadExistingLines(); TruncateFileIfNeeded(); } FileLogger::CircularLogFile::~CircularLogFile() { + std::lock_guard lock(mutex_); CloseFile(); } void FileLogger::CircularLogFile::writeLog(const char* logLine, const uint64_t len) { + std::lock_guard lock(mutex_); if (!fp_) return; @@ -60,6 +63,7 @@ void FileLogger::CircularLogFile::writeLog(const char* logLine, } } void FileLogger::CircularLogFile::flush() { + std::lock_guard lock(mutex_); if (fp_) { fflush(fp_); } diff --git a/engine/utils/file_logger.h b/engine/utils/file_logger.h index 941899f21..7754da64b 100644 --- a/engine/utils/file_logger.h +++ b/engine/utils/file_logger.h @@ -44,16 +44,17 @@ class TRANTOR_EXPORT FileLogger : public AsyncFileLogger { void writeLog(const char* logLine, const uint64_t len); void flush(); - uint64_t getLength() const { return totalLines_; } + uint64_t getLength() const { return totalLines_.load(); } private: FILE* fp_{nullptr}; uint64_t max_lines_; - uint64_t totalLines_{0}; + std::atomic totalLines_{0}; std::string file_name_; std::deque lineBuffer_; uint64_t linesWrittenSinceLastTruncate_{0}; static const uint64_t TRUNCATE_CHECK_INTERVAL = 1000; + mutable std::mutex mutex_; void LoadExistingLines(); void TruncateFileIfNeeded(); From 4f3b0b038d1c18bb64f86f320b2eb9614f047563 Mon Sep 17 00:00:00 2001 From: CameronNg Date: Tue, 10 Sep 2024 09:10:45 +0700 Subject: [PATCH 4/7] Fix comment --- engine/utils/config_yaml_utils.h | 8 ++++---- engine/utils/file_logger.cc | 16 ++++++++-------- engine/utils/file_logger.h | 6 ++++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 127617345..8e3668292 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -53,16 +53,16 @@ inline CortexConfig FromYaml(const std::string& path, try { auto node = YAML::LoadFile(config_file_path.string()); - int max_lines_; + int max_lines; if (!node["maxLogLines"]) { - max_lines_ = kDefaultMaxLines; + max_lines = kDefaultMaxLines; } else { - max_lines_ = node["maxLogLines"].as(); + max_lines = node["maxLogLines"].as(); } CortexConfig config = { .logFolderPath = node["logFolderPath"].as(), .dataFolderPath = node["dataFolderPath"].as(), - .maxLogLines = max_lines_, + .maxLogLines = max_lines, .apiServerHost = node["apiServerHost"].as(), .apiServerPort = node["apiServerPort"].as(), }; diff --git a/engine/utils/file_logger.cc b/engine/utils/file_logger.cc index 136a7b788..c20bbb7ef 100644 --- a/engine/utils/file_logger.cc +++ b/engine/utils/file_logger.cc @@ -48,7 +48,7 @@ void FileLogger::CircularLogFile::writeLog(const char* logLine, std::istringstream iss(logString); std::string line; while (std::getline(iss, line)) { - if (totalLines_ >= max_lines_) { + if (totalLines_.load() >= max_lines_) { lineBuffer_.pop_front(); --totalLines_; } @@ -56,7 +56,7 @@ void FileLogger::CircularLogFile::writeLog(const char* logLine, ++totalLines_; AppendToFile(line + "\n"); ++linesWrittenSinceLastTruncate_; - if (linesWrittenSinceLastTruncate_ >= TRUNCATE_CHECK_INTERVAL) { + if (linesWrittenSinceLastTruncate_.load() >= TRUNCATE_CHECK_INTERVAL) { TruncateFileIfNeeded(); } @@ -71,7 +71,7 @@ void FileLogger::CircularLogFile::flush() { void FileLogger::CircularLogFile::TruncateFileIfNeeded() { // std::cout<<"Truncating file "<< totalLines_ < max_lines_ ? max_lines_ : lineBuffer_.size(); + totalLines_.store(lineBuffer_.size() > max_lines_ ? max_lines_ + : lineBuffer_.size()); } // Reopen the file OpenFile(); // LoadExistingLines(); - linesWrittenSinceLastTruncate_ = 0; + linesWrittenSinceLastTruncate_.store(0); } void FileLogger::CircularLogFile::OpenFile() { @@ -145,7 +145,7 @@ void FileLogger::CircularLogFile::LoadExistingLines() { fseek(fp_, 0, SEEK_SET); lineBuffer_.clear(); - totalLines_ = 0; + totalLines_.store(0); std::string line; char buffer[4096]; @@ -154,7 +154,7 @@ void FileLogger::CircularLogFile::LoadExistingLines() { if (!line.empty() && line.back() == '\n') { line.pop_back(); // Remove trailing newline } - if (totalLines_ >= max_lines_) { + if (totalLines_.load() >= max_lines_) { lineBuffer_.pop_front(); } lineBuffer_.push_back(line); diff --git a/engine/utils/file_logger.h b/engine/utils/file_logger.h index 7754da64b..ba8bb0515 100644 --- a/engine/utils/file_logger.h +++ b/engine/utils/file_logger.h @@ -2,6 +2,8 @@ #include #include +#include +#include #include #ifdef _WIN32 @@ -49,10 +51,10 @@ class TRANTOR_EXPORT FileLogger : public AsyncFileLogger { private: FILE* fp_{nullptr}; uint64_t max_lines_; - std::atomic totalLines_{0}; + std::atomic totalLines_{0}; std::string file_name_; std::deque lineBuffer_; - uint64_t linesWrittenSinceLastTruncate_{0}; + std::atomic linesWrittenSinceLastTruncate_{0}; static const uint64_t TRUNCATE_CHECK_INTERVAL = 1000; mutable std::mutex mutex_; From 53ff23785ba3a8abdb7c50170988dba30eeb710c Mon Sep 17 00:00:00 2001 From: CameronNg Date: Tue, 10 Sep 2024 09:29:19 +0700 Subject: [PATCH 5/7] Fix comment --- engine/utils/file_logger.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/utils/file_logger.h b/engine/utils/file_logger.h index ba8bb0515..23dcf8e2c 100644 --- a/engine/utils/file_logger.h +++ b/engine/utils/file_logger.h @@ -51,10 +51,10 @@ class TRANTOR_EXPORT FileLogger : public AsyncFileLogger { private: FILE* fp_{nullptr}; uint64_t max_lines_; - std::atomic totalLines_{0}; + std::atomic totalLines_{0}; std::string file_name_; std::deque lineBuffer_; - std::atomic linesWrittenSinceLastTruncate_{0}; + std::atomic linesWrittenSinceLastTruncate_{0}; static const uint64_t TRUNCATE_CHECK_INTERVAL = 1000; mutable std::mutex mutex_; From 3ab763c8e16aab7c2a2fd8ff65bbeac91eb593df Mon Sep 17 00:00:00 2001 From: CameronNg Date: Tue, 10 Sep 2024 09:42:20 +0700 Subject: [PATCH 6/7] fix-comment --- engine/utils/file_logger.cc | 18 +++++++----------- engine/utils/file_logger.h | 5 ++--- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/engine/utils/file_logger.cc b/engine/utils/file_logger.cc index c20bbb7ef..f684d457f 100644 --- a/engine/utils/file_logger.cc +++ b/engine/utils/file_logger.cc @@ -48,12 +48,10 @@ void FileLogger::CircularLogFile::writeLog(const char* logLine, std::istringstream iss(logString); std::string line; while (std::getline(iss, line)) { - if (totalLines_.load() >= max_lines_) { + if (lineBuffer_.size() >= max_lines_) { lineBuffer_.pop_front(); - --totalLines_; } lineBuffer_.push_back(line); - ++totalLines_; AppendToFile(line + "\n"); ++linesWrittenSinceLastTruncate_; if (linesWrittenSinceLastTruncate_.load() >= TRUNCATE_CHECK_INTERVAL) { @@ -71,7 +69,7 @@ void FileLogger::CircularLogFile::flush() { void FileLogger::CircularLogFile::TruncateFileIfNeeded() { // std::cout<<"Truncating file "<< totalLines_ < max_lines_ ? max_lines_ - : lineBuffer_.size()); } + // else { + // totalLines_.store(lineBuffer_.size() > max_lines_ ? max_lines_ + // : lineBuffer_.size()); + // } // Reopen the file OpenFile(); @@ -145,7 +143,6 @@ void FileLogger::CircularLogFile::LoadExistingLines() { fseek(fp_, 0, SEEK_SET); lineBuffer_.clear(); - totalLines_.store(0); std::string line; char buffer[4096]; @@ -154,11 +151,10 @@ void FileLogger::CircularLogFile::LoadExistingLines() { if (!line.empty() && line.back() == '\n') { line.pop_back(); // Remove trailing newline } - if (totalLines_.load() >= max_lines_) { + if (lineBuffer_.size() >= max_lines_) { lineBuffer_.pop_front(); } lineBuffer_.push_back(line); - ++totalLines_; } // Move back to the end of the file for appending diff --git a/engine/utils/file_logger.h b/engine/utils/file_logger.h index 23dcf8e2c..f33cf4d96 100644 --- a/engine/utils/file_logger.h +++ b/engine/utils/file_logger.h @@ -46,15 +46,14 @@ class TRANTOR_EXPORT FileLogger : public AsyncFileLogger { void writeLog(const char* logLine, const uint64_t len); void flush(); - uint64_t getLength() const { return totalLines_.load(); } + uint64_t getLength() const { return lineBuffer_.size(); } private: FILE* fp_{nullptr}; uint64_t max_lines_; - std::atomic totalLines_{0}; std::string file_name_; std::deque lineBuffer_; - std::atomic linesWrittenSinceLastTruncate_{0}; + std::atomic linesWrittenSinceLastTruncate_{0}; static const uint64_t TRUNCATE_CHECK_INTERVAL = 1000; mutable std::mutex mutex_; From 82db9042072b8f7f216729cfa32df0bc77ee086b Mon Sep 17 00:00:00 2001 From: CameronNg Date: Tue, 10 Sep 2024 09:45:43 +0700 Subject: [PATCH 7/7] fix CI build --- engine/utils/file_logger.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/utils/file_logger.h b/engine/utils/file_logger.h index f33cf4d96..58b719019 100644 --- a/engine/utils/file_logger.h +++ b/engine/utils/file_logger.h @@ -2,9 +2,11 @@ #include #include +#include #include #include #include +#include #ifdef _WIN32 #include