From 91464ac19de755ff639a5d0014c484801542bdc6 Mon Sep 17 00:00:00 2001 From: CameronNg Date: Wed, 11 Sep 2024 20:57:22 +0700 Subject: [PATCH 1/4] feat: redirect log --- CMakeLists.txt | 32 ++++--- base/cortex-common/enginei.h | 3 +- src/file_logger.cc | 175 +++++++++++++++++++++++++++++++++++ src/file_logger.h | 72 ++++++++++++++ src/llama_engine.cc | 41 ++++---- src/llama_engine.h | 11 ++- 6 files changed, 297 insertions(+), 37 deletions(-) create mode 100644 src/file_logger.cc create mode 100644 src/file_logger.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f564309..79e7eed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,14 +3,15 @@ project(cortex.llamacpp) SET(TARGET engine) if(UNIX AND NOT APPLE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread") add_compile_options(-fPIC -pthread) - find_package( Threads ) + find_package(Threads) endif() set(THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/build_deps/_install) include(CheckIncludeFileCXX) + # CPP version check_include_file_cxx(any HAS_ANY) check_include_file_cxx(string_view HAS_STRING_VIEW) @@ -28,27 +29,28 @@ endif() add_subdirectory(llama.cpp/examples/llava) add_subdirectory(llama.cpp) -add_library(${TARGET} SHARED - src/llama_engine.cc - src/llama_server_context.cc - src/llama_client_slot.cc +add_library(${TARGET} SHARED + src/llama_engine.cc + src/llama_server_context.cc + src/llama_client_slot.cc + src/file_logger.cc ) find_library(JSONCPP - NAMES jsoncpp - HINTS "${THIRD_PARTY_PATH}/lib" + NAMES jsoncpp + HINTS "${THIRD_PARTY_PATH}/lib" ) find_library(TRANTOR - NAMES trantor - HINTS "${THIRD_PARTY_PATH}/lib" + NAMES trantor + HINTS "${THIRD_PARTY_PATH}/lib" ) target_link_libraries(${TARGET} PRIVATE common llama llava ${JSONCPP} ${TRANTOR} - ${CMAKE_THREAD_LIBS_INIT}) -target_include_directories(${TARGET} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/base - ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp - ${THIRD_PARTY_PATH}/include) + ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(${TARGET} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/base + ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp + ${THIRD_PARTY_PATH}/include) target_compile_features(${TARGET} PUBLIC cxx_std_17) \ No newline at end of file diff --git a/base/cortex-common/enginei.h b/base/cortex-common/enginei.h index 41005f1..763288a 100644 --- a/base/cortex-common/enginei.h +++ b/base/cortex-common/enginei.h @@ -36,7 +36,8 @@ class EngineI { } return false; } - virtual void SetFileLogger() = 0; + virtual void SetFileLogger(int max_log_lines, + const std::string& log_path) = 0; // API to get running models. virtual void GetModels( diff --git a/src/file_logger.cc b/src/file_logger.cc new file mode 100644 index 0000000..f684d45 --- /dev/null +++ b/src/file_logger.cc @@ -0,0 +1,175 @@ +#include "file_logger.h" +#include +#include +#include + +#ifdef _WIN32 +#include +#define ftruncate _chsize +#else +#include +#endif +#include + +using namespace trantor; + +FileLogger::FileLogger() : AsyncFileLogger() {} + +FileLogger::~FileLogger() = default; + +void FileLogger::output_(const char* msg, const uint64_t len) { + if (!circular_log_file_ptr_) { + circular_log_file_ptr_ = + std::make_unique(fileBaseName_, max_lines_); + } + circular_log_file_ptr_->writeLog(msg, len); +} + +FileLogger::CircularLogFile::CircularLogFile(const std::string& fileName, + uint64_t maxLines) + : max_lines_(maxLines), file_name_(fileName) { + std::lock_guard lock(mutex_); + OpenFile(); + LoadExistingLines(); + TruncateFileIfNeeded(); +} + +FileLogger::CircularLogFile::~CircularLogFile() { + std::lock_guard lock(mutex_); + CloseFile(); +} +void FileLogger::CircularLogFile::writeLog(const char* logLine, + const uint64_t len) { + std::lock_guard lock(mutex_); + if (!fp_) + return; + + std::string logString(logLine, len); + std::istringstream iss(logString); + std::string line; + while (std::getline(iss, line)) { + if (lineBuffer_.size() >= max_lines_) { + lineBuffer_.pop_front(); + } + lineBuffer_.push_back(line); + AppendToFile(line + "\n"); + ++linesWrittenSinceLastTruncate_; + if (linesWrittenSinceLastTruncate_.load() >= TRUNCATE_CHECK_INTERVAL) { + + TruncateFileIfNeeded(); + } + } +} +void FileLogger::CircularLogFile::flush() { + std::lock_guard lock(mutex_); + if (fp_) { + fflush(fp_); + } +} + +void FileLogger::CircularLogFile::TruncateFileIfNeeded() { + // std::cout<<"Truncating file "<< totalLines_ < max_lines_ ? lineBuffer_.size() - max_lines_ : 0; + + for (size_t i = startIndex; i < lineBuffer_.size(); ++i) { + fprintf(tempFile, "%s\n", lineBuffer_[i].c_str()); + } + + fclose(tempFile); + + // Replace the original file with the temporary file + if (std::rename(tempFileName.c_str(), file_name_.c_str()) != 0) { + std::cout << "Error replacing original file with truncated file: " + << strerror(errno) << std::endl; + std::remove(tempFileName.c_str()); // Clean up the temporary file + } + // else { + // totalLines_.store(lineBuffer_.size() > max_lines_ ? max_lines_ + // : lineBuffer_.size()); + // } + + // Reopen the file + OpenFile(); + // LoadExistingLines(); + linesWrittenSinceLastTruncate_.store(0); +} + +void FileLogger::CircularLogFile::OpenFile() { +#ifdef _WIN32 + auto wFileName = utils::toNativePath(file_name_); + fp_ = _wfopen(wFileName.c_str(), L"r+"); +#else + fp_ = fopen(file_name_.c_str(), "r+"); +#endif + + if (!fp_) { +// If file doesn't exist, create it +#ifdef _WIN32 + fp_ = _wfopen(wFileName.c_str(), L"w+"); +#else + fp_ = fopen(file_name_.c_str(), "w+"); +#endif + + if (!fp_) { + std::cerr << "Error opening file: " << strerror(errno) << std::endl; + } + } +} +void FileLogger::CircularLogFile::LoadExistingLines() { + if (!fp_) + return; + + // Move to the beginning of the file + fseek(fp_, 0, SEEK_SET); + + lineBuffer_.clear(); + + std::string line; + char buffer[4096]; + while (fgets(buffer, sizeof(buffer), fp_) != nullptr) { + line = buffer; + if (!line.empty() && line.back() == '\n') { + line.pop_back(); // Remove trailing newline + } + if (lineBuffer_.size() >= max_lines_) { + lineBuffer_.pop_front(); + } + lineBuffer_.push_back(line); + } + + // Move back to the end of the file for appending + fseek(fp_, 0, SEEK_END); +} +void FileLogger::CircularLogFile::AppendToFile(const std::string& line) { + if (fp_) { + fwrite(line.c_str(), 1, line.length(), fp_); + fflush(fp_); + } +} + +void FileLogger::CircularLogFile::CloseFile() { + if (fp_) { + fclose(fp_); + fp_ = nullptr; + } +} \ No newline at end of file diff --git a/src/file_logger.h b/src/file_logger.h new file mode 100644 index 0000000..58b7190 --- /dev/null +++ b/src/file_logger.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace trantor { + +class TRANTOR_EXPORT FileLogger : public AsyncFileLogger { + public: + FileLogger(); + ~FileLogger(); + + /** + * @brief Set the maximum number of lines to keep in the log file. + * + * @param maxLines + */ + void setMaxLines(uint64_t maxLines) { max_lines_ = maxLines; } + + /** + * @brief Set the log file name. + * + * @param fileName The full name of the log file. + */ + void setFileName(const std::string& fileName) { + filePath_ = "./"; + fileBaseName_ = fileName; + fileExtName_ = ""; + } + void output_(const char* msg, const uint64_t len); + + protected: + class CircularLogFile { + public: + CircularLogFile(const std::string& fileName, uint64_t maxLines); + ~CircularLogFile(); + + void writeLog(const char* logLine, const uint64_t len); + void flush(); + uint64_t getLength() const { return lineBuffer_.size(); } + + private: + FILE* fp_{nullptr}; + uint64_t max_lines_; + std::string file_name_; + std::deque lineBuffer_; + std::atomic linesWrittenSinceLastTruncate_{0}; + static const uint64_t TRUNCATE_CHECK_INTERVAL = 1000; + mutable std::mutex mutex_; + + void LoadExistingLines(); + void TruncateFileIfNeeded(); + void AppendToFile(const std::string& line); + void OpenFile(); + void CloseFile(); + }; + std::unique_ptr circular_log_file_ptr_; + uint64_t max_lines_{100000}; // Default to 100000 lines +}; + +} // namespace trantor \ No newline at end of file diff --git a/src/llama_engine.cc b/src/llama_engine.cc index b21d842..70a8082 100644 --- a/src/llama_engine.cc +++ b/src/llama_engine.cc @@ -121,7 +121,7 @@ std::string CreateReturnJson(const std::string& id, const std::string& model, return Json::writeString(writer, root); } } // namespace - +// derepted this function because we no longer support change log when load model void LlamaEngine::SetLoggerOption(const Json::Value& json_body) { if (!json_body["log_option"].isNull()) { int log_option = json_body["log_option"].asInt(); @@ -133,12 +133,18 @@ void LlamaEngine::SetLoggerOption(const Json::Value& json_body) { }, []() { fflush(stdout); }); } else { + std::string log_path = json_body.get("log_path", log_path_).asString(); + int max_log_lines = + json_body.get("max_log_lines", max_log_lines_).asInt(); + trantor::FileLogger asyncFileLogger; + asyncFileLogger.setFileName(log_path); + asyncFileLogger.setMaxLines(max_log_lines); // Keep last 100000 lines + // asyncFileLogger.startLogging(); trantor::Logger::setOutputFunction( [&](const char* msg, const uint64_t len) { - asynce_file_logger_->output(msg, len); + asynce_file_logger_->output_(msg, len); }, [&]() { asynce_file_logger_->flush(); }); - asynce_file_logger_->setFileSizeLimit(max_log_file_size); } } else { // For backward compatible @@ -170,18 +176,9 @@ void LlamaEngine::SetLoggerOption(const Json::Value& json_body) { } LlamaEngine::LlamaEngine(int log_option) { - trantor::Logger::setLogLevel(trantor::Logger::kError); + trantor::Logger::setLogLevel(trantor::Logger::kInfo); if (log_option == kFileLoggerOption) { - std::filesystem::create_directories(log_folder); - asynce_file_logger_ = std::make_unique(); - asynce_file_logger_->setFileName(log_base_name); - asynce_file_logger_->startLogging(); - trantor::Logger::setOutputFunction( - [&](const char* msg, const uint64_t len) { - asynce_file_logger_->output(msg, len); - }, - [&]() { asynce_file_logger_->flush(); }); - asynce_file_logger_->setFileSizeLimit(max_log_file_size); + asynce_file_logger_ = std::make_unique(); } log_disable(); @@ -237,7 +234,7 @@ void LlamaEngine::HandleEmbedding( void LlamaEngine::LoadModel( std::shared_ptr json_body, std::function&& callback) { - SetLoggerOption(*json_body); + // SetLoggerOption(*json_body); // dont update log option when load model if (std::exchange(print_version_, false)) { #if defined(CORTEXLLAMA_VERSION) LOG_INFO << "cortex.llamacpp version: " << CORTEXLLAMA_VERSION; @@ -370,7 +367,19 @@ void LlamaEngine::GetModels( LOG_INFO << "Running models responded"; } // should decrepted this function because it no longer used in cortex cpp -void LlamaEngine::SetFileLogger() { +void LlamaEngine::SetFileLogger(int max_log_lines, + const std::string& log_path) { + if (!asynce_file_logger_) { + asynce_file_logger_ = std::make_unique(); + } + asynce_file_logger_->setFileName(log_path); + asynce_file_logger_->setMaxLines(max_log_lines); // Keep last 100000 lines + asynce_file_logger_->startLogging(); + trantor::Logger::setOutputFunction( + [&](const char* msg, const uint64_t len) { + asynce_file_logger_->output_(msg, len); + }, + [&]() { asynce_file_logger_->flush(); }); llama_log_set( [](ggml_log_level level, const char* text, void* user_data) { (void)level; diff --git a/src/llama_engine.h b/src/llama_engine.h index 92244a8..97e8037 100644 --- a/src/llama_engine.h +++ b/src/llama_engine.h @@ -1,14 +1,15 @@ #pragma once +#include #include "chat_completion_request.h" #include "cortex-common/enginei.h" +#include "file_logger.h" +#include "llama.h" #include "llama_server_context.h" #include "trantor/utils/ConcurrentTaskQueue.h" -#include "llama.h" -#include constexpr char log_base_name[] = "logs/cortex"; constexpr char log_folder[] = "logs"; -constexpr size_t max_log_file_size = 20000000; // ~20mb +constexpr size_t max_log_file_size = 20000000; // ~20mb class LlamaEngine : public EngineI { public: @@ -33,7 +34,7 @@ class LlamaEngine : public EngineI { void GetModels( std::shared_ptr jsonBody, std::function&& callback) final; - void SetFileLogger(); + void SetFileLogger(int max_log_lines, const std::string& log_path); void SetLoggerOption(const Json::Value& json_body); private: @@ -74,5 +75,5 @@ class LlamaEngine : public EngineI { std::atomic no_of_chats_ = 0; bool print_version_ = true; - std::unique_ptr asynce_file_logger_; + std::unique_ptr asynce_file_logger_; }; \ No newline at end of file From 02609469f52de49ad873f8d49378bc42e8cf58b3 Mon Sep 17 00:00:00 2001 From: CameronNg Date: Wed, 11 Sep 2024 21:07:34 +0700 Subject: [PATCH 2/4] feat: redirect log --- src/llama_engine.cc | 101 ++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/src/llama_engine.cc b/src/llama_engine.cc index 70a8082..5b48f02 100644 --- a/src/llama_engine.cc +++ b/src/llama_engine.cc @@ -121,58 +121,59 @@ std::string CreateReturnJson(const std::string& id, const std::string& model, return Json::writeString(writer, root); } } // namespace + // derepted this function because we no longer support change log when load model void LlamaEngine::SetLoggerOption(const Json::Value& json_body) { - if (!json_body["log_option"].isNull()) { - int log_option = json_body["log_option"].asInt(); - if (log_option != kFileLoggerOption) { - // Revert to default trantor logger output function - trantor::Logger::setOutputFunction( - [](const char* msg, const uint64_t len) { - fwrite(msg, 1, static_cast(len), stdout); - }, - []() { fflush(stdout); }); - } else { - std::string log_path = json_body.get("log_path", log_path_).asString(); - int max_log_lines = - json_body.get("max_log_lines", max_log_lines_).asInt(); - trantor::FileLogger asyncFileLogger; - asyncFileLogger.setFileName(log_path); - asyncFileLogger.setMaxLines(max_log_lines); // Keep last 100000 lines - // asyncFileLogger.startLogging(); - trantor::Logger::setOutputFunction( - [&](const char* msg, const uint64_t len) { - asynce_file_logger_->output_(msg, len); - }, - [&]() { asynce_file_logger_->flush(); }); - } - } else { - // For backward compatible - trantor::Logger::setOutputFunction( - [](const char* msg, const uint64_t len) { - fwrite(msg, 1, static_cast(len), stdout); - }, - []() { fflush(stdout); }); - } - - if (!json_body["log_level"].isNull()) { - std::string log_level = json_body["log_level"].asString(); - if (log_level == "trace") { - trantor::Logger::setLogLevel(trantor::Logger::kTrace); - } else if (log_level == "debug") { - trantor::Logger::setLogLevel(trantor::Logger::kDebug); - } else if (log_level == "info") { - trantor::Logger::setLogLevel(trantor::Logger::kInfo); - } else if (log_level == "warn") { - trantor::Logger::setLogLevel(trantor::Logger::kWarn); - } else if (log_level == "fatal") { - trantor::Logger::setLogLevel(trantor::Logger::kFatal); - } else { - trantor::Logger::setLogLevel(trantor::Logger::kError); - } - } else { - trantor::Logger::setLogLevel(trantor::Logger::kDebug); - } + // if (!json_body["log_option"].isNull()) { + // int log_option = json_body["log_option"].asInt(); + // if (log_option != kFileLoggerOption) { + // // Revert to default trantor logger output function + // trantor::Logger::setOutputFunction( + // [](const char* msg, const uint64_t len) { + // fwrite(msg, 1, static_cast(len), stdout); + // }, + // []() { fflush(stdout); }); + // } else { + // std::string log_path = + // json_body.get("log_path", "./logs/cortex.log").asString(); + // int max_log_lines = json_body.get("max_log_lines", 100000).asInt(); + // trantor::FileLogger asyncFileLogger; + // asyncFileLogger.setFileName(log_path); + // asyncFileLogger.setMaxLines(max_log_lines); // Keep last 100000 lines + // // asyncFileLogger.startLogging(); + // trantor::Logger::setOutputFunction( + // [&](const char* msg, const uint64_t len) { + // asynce_file_logger_->output_(msg, len); + // }, + // [&]() { asynce_file_logger_->flush(); }); + // } + // } else { + // // For backward compatible + // trantor::Logger::setOutputFunction( + // [](const char* msg, const uint64_t len) { + // fwrite(msg, 1, static_cast(len), stdout); + // }, + // []() { fflush(stdout); }); + // } + + // if (!json_body["log_level"].isNull()) { + // std::string log_level = json_body["log_level"].asString(); + // if (log_level == "trace") { + // trantor::Logger::setLogLevel(trantor::Logger::kTrace); + // } else if (log_level == "debug") { + // trantor::Logger::setLogLevel(trantor::Logger::kDebug); + // } else if (log_level == "info") { + // trantor::Logger::setLogLevel(trantor::Logger::kInfo); + // } else if (log_level == "warn") { + // trantor::Logger::setLogLevel(trantor::Logger::kWarn); + // } else if (log_level == "fatal") { + // trantor::Logger::setLogLevel(trantor::Logger::kFatal); + // } else { + // trantor::Logger::setLogLevel(trantor::Logger::kError); + // } + // } else { + // trantor::Logger::setLogLevel(trantor::Logger::kDebug); + // } } LlamaEngine::LlamaEngine(int log_option) { From e17df18bdf2c2fe2b70c87a3e76270dd35f87be6 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Wed, 11 Sep 2024 21:15:55 +0700 Subject: [PATCH 3/4] feat: redirect log --- examples/server/server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cc b/examples/server/server.cc index d62384f..159d4f1 100644 --- a/examples/server/server.cc +++ b/examples/server/server.cc @@ -102,7 +102,7 @@ int main(int argc, char** argv) { Server server; //set logger here - server.engine_->SetFileLogger(); + // server.engine_->SetFileLogger(); SyncJsonReader r; auto svr = std::make_unique(); From 2fd04dd8acafb7ae49b189b14ccb9b282eb20a1b Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Thu, 12 Sep 2024 10:44:30 +0700 Subject: [PATCH 4/4] Fix comment --- base/cortex-common/enginei.h | 5 +++-- src/llama_engine.h | 4 ---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/base/cortex-common/enginei.h b/base/cortex-common/enginei.h index 763288a..b352952 100644 --- a/base/cortex-common/enginei.h +++ b/base/cortex-common/enginei.h @@ -36,11 +36,12 @@ class EngineI { } return false; } - virtual void SetFileLogger(int max_log_lines, - const std::string& log_path) = 0; // API to get running models. virtual void GetModels( std::shared_ptr json_body, std::function&& callback) = 0; + // API for set file logger + virtual void SetFileLogger(int max_log_lines, + const std::string& log_path) = 0; }; \ No newline at end of file diff --git a/src/llama_engine.h b/src/llama_engine.h index 97e8037..fa9c01e 100644 --- a/src/llama_engine.h +++ b/src/llama_engine.h @@ -7,10 +7,6 @@ #include "llama_server_context.h" #include "trantor/utils/ConcurrentTaskQueue.h" -constexpr char log_base_name[] = "logs/cortex"; -constexpr char log_folder[] = "logs"; -constexpr size_t max_log_file_size = 20000000; // ~20mb - class LlamaEngine : public EngineI { public: LlamaEngine(int log_option = 0);