From 1d6f419419ab6e9410760c1782ed5f476cb6cb1a Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 11:36:27 +0700 Subject: [PATCH 01/10] feat: cortex start --- engine/commands/server_start_cmd.cc | 79 +++++++++++++++++++++++ engine/commands/server_start_cmd.h | 15 +++++ engine/controllers/command_line_parser.cc | 12 ++++ 3 files changed, 106 insertions(+) create mode 100644 engine/commands/server_start_cmd.cc create mode 100644 engine/commands/server_start_cmd.h diff --git a/engine/commands/server_start_cmd.cc b/engine/commands/server_start_cmd.cc new file mode 100644 index 000000000..7b7a9f380 --- /dev/null +++ b/engine/commands/server_start_cmd.cc @@ -0,0 +1,79 @@ +#include "server_start_cmd.h" +#include "commands/cortex_upd_cmd.h" +#include "httplib.h" +#include "trantor/utils/Logger.h" +#include "utils/cortex_utils.h" +#include "utils/file_manager_utils.h" +#include "utils/logging_utils.h" + +namespace commands { +ServerStartCmd::ServerStartCmd(std::string host, int port) + : host_(std::move(host)), port_(port) {} + +void ServerStartCmd::Exec() { +#if defined(_WIN32) || defined(_WIN64) + // Windows-specific code to create a new process + STARTUPINFO si; + PROCESS_INFORMATION pi; + + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + ZeroMemory(&pi, sizeof(pi)); + auto exe = commands::GetCortexBinary(); + std::string cmds = + cortex_utils::GetCurrentPath() + "/" + exe + " --start-server"; + // Create child process + if (!CreateProcess( + NULL, // No module name (use command line) + const_cast( + cmds.c_str()), // Command line (replace with your actual executable) + NULL, // Process handle not inheritable + NULL, // Thread handle not inheritable + FALSE, // Set handle inheritance to FALSE + 0, // No creation flags + NULL, // Use parent's environment block + NULL, // Use parent's starting directory + &si, // Pointer to STARTUPINFO structure + &pi)) // Pointer to PROCESS_INFORMATION structure + { + std::cout << "Could not start server: " << GetLastError() << std::endl; + } else { + std::cout << "Server started" << std::endl; + } + +#else + // Unix-like system-specific code to fork a child process + pid_t pid = fork(); + + if (pid < 0) { + // Fork failed + std::cerr << "Could not start server: " << std::endl; + return; + } else if (pid == 0) { + // No need to configure LD_LIBRARY_PATH for macOS +#if !defined(__APPLE__) || !defined(__MACH__) + const char* name = "LD_LIBRARY_PATH"; + auto data = getenv(name); + std::string v; + if (auto g = getenv(name); g) { + v += g; + } + CTL_INF("LD_LIBRARY_PATH: " << v); + auto data_path = file_manager_utils::GetCortexDataPath(); + auto llamacpp_path = data_path / "engines" / "cortex.llamacpp/"; + auto trt_path = data_path / "engines" / "cortex.tensorrt-llm/"; + auto new_v = trt_path.string() + ":" + llamacpp_path.string() + ":" + v; + setenv(name, new_v.c_str(), true); + CTL_INF("LD_LIBRARY_PATH: " << getenv(name)); +#endif + auto exe = commands::GetCortexBinary(); + std::string p = cortex_utils::GetCurrentPath() + "/" + exe; + execl(p.c_str(), exe.c_str(), "--start-server", (char*)0); + } else { + // Parent process + std::cout << "Server started" << std::endl; + } +#endif +} + +}; // namespace commands \ No newline at end of file diff --git a/engine/commands/server_start_cmd.h b/engine/commands/server_start_cmd.h new file mode 100644 index 000000000..6f121d6bb --- /dev/null +++ b/engine/commands/server_start_cmd.h @@ -0,0 +1,15 @@ +#pragma once +#include + +namespace commands { + +class ServerStartCmd{ + public: + ServerStartCmd(std::string host, int port); + void Exec(); + + private: + std::string host_; + int port_; +}; +} // namespace commands \ No newline at end of file diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index b8baf3466..78f1ace54 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -14,6 +14,7 @@ #include "commands/run_cmd.h" #include "commands/server_stop_cmd.h" #include "commands/model_del_cmd.h" +#include "commands/server_start_cmd.h" #include "config/yaml_config.h" #include "services/engine_service.h" #include "utils/file_manager_utils.h" @@ -174,6 +175,14 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { }); } + auto start_cmd = app_.add_subcommand("start", "Start the API server"); + + start_cmd->callback([&config] { + commands::ServerStartCmd ssc(config.apiServerHost, + std::stoi(config.apiServerPort)); + ssc.Exec(); + }); + auto stop_cmd = app_.add_subcommand("stop", "Stop the API server"); stop_cmd->callback([&config] { @@ -208,6 +217,9 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { } CLI11_PARSE(app_, argc, argv); + if(argc == 1) { + std::cout << app_.help() << std::endl; + } // Check new update, only check for stable release for now #ifdef CORTEX_CPP_VERSION From cad529c4d93b95c81f731bab4d4dd5355f4b19ea Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 11:44:35 +0700 Subject: [PATCH 02/10] fix: print app.help --- engine/controllers/command_line_parser.cc | 3 +- engine/main.cc | 126 +++++----------------- 2 files changed, 29 insertions(+), 100 deletions(-) diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 78f1ace54..388a64ea1 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -218,7 +218,8 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { CLI11_PARSE(app_, argc, argv); if(argc == 1) { - std::cout << app_.help() << std::endl; + CLI_LOG(app_.help()); + return true; } // Check new update, only check for stable release for now diff --git a/engine/main.cc b/engine/main.cc index 79f1aaa98..bd3f97744 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -85,72 +85,6 @@ void RunServer() { // return 0; } -void ForkProcess() { -#if defined(_WIN32) || defined(_WIN64) - // Windows-specific code to create a new process - STARTUPINFO si; - PROCESS_INFORMATION pi; - - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - ZeroMemory(&pi, sizeof(pi)); - auto exe = commands::GetCortexBinary(); - std::string cmds = - cortex_utils::GetCurrentPath() + "/" + exe + " --start-server"; - // Create child process - if (!CreateProcess( - NULL, // No module name (use command line) - const_cast( - cmds.c_str()), // Command line (replace with your actual executable) - NULL, // Process handle not inheritable - NULL, // Thread handle not inheritable - FALSE, // Set handle inheritance to FALSE - 0, // No creation flags - NULL, // Use parent's environment block - NULL, // Use parent's starting directory - &si, // Pointer to STARTUPINFO structure - &pi)) // Pointer to PROCESS_INFORMATION structure - { - std::cout << "Could not start server: " << GetLastError() << std::endl; - } else { - std::cout << "Server started" << std::endl; - } - -#else - // Unix-like system-specific code to fork a child process - pid_t pid = fork(); - - if (pid < 0) { - // Fork failed - std::cerr << "Could not start server: " << std::endl; - return; - } else if (pid == 0) { - // No need to configure LD_LIBRARY_PATH for macOS -#if !defined(__APPLE__) || !defined(__MACH__) - const char* name = "LD_LIBRARY_PATH"; - auto data = getenv(name); - std::string v; - if (auto g = getenv(name); g) { - v += g; - } - CTL_INF("LD_LIBRARY_PATH: " << v); - auto data_path = file_manager_utils::GetCortexDataPath(); - auto llamacpp_path = data_path / "engines" / "cortex.llamacpp/"; - auto trt_path = data_path / "engines" / "cortex.tensorrt-llm/"; - auto new_v = trt_path.string() + ":" + llamacpp_path.string() + ":" + v; - setenv(name, new_v.c_str(), true); - CTL_INF("LD_LIBRARY_PATH: " << getenv(name)); -#endif - auto exe = commands::GetCortexBinary(); - std::string p = cortex_utils::GetCurrentPath() + "/" + exe; - execl(p.c_str(), exe.c_str(), "--start-server", (char*)0); - } else { - // Parent process - std::cout << "Server started" << std::endl; - } -#endif -} - int main(int argc, char* argv[]) { // Stop the program if the system is not supported auto system_info = system_info_utils::GetSystemInfo(); @@ -195,40 +129,34 @@ int main(int argc, char* argv[]) { } } - if (argc > 1) { - if (strcmp(argv[1], "--start-server") == 0) { - RunServer(); - return 0; - } else { - bool verbose = false; - for (int i = 0; i < argc; i++) { - if (strcmp(argv[i], "--verbose") == 0) { - verbose = true; - } - } - if (!verbose) { - auto config = file_manager_utils::GetCortexConfig(); - std::filesystem::create_directories( - std::filesystem::path(config.logFolderPath) / - std::filesystem::path(cortex_utils::logs_folder)); - trantor::FileLogger asyncFileLogger; - asyncFileLogger.setFileName(config.logFolderPath + "/" + - cortex_utils::logs_cli_base_name); - asyncFileLogger.setMaxLines( - config.maxLogLines); // Keep last 100000 lines - asyncFileLogger.startLogging(); - trantor::Logger::setOutputFunction( - [&](const char* msg, const uint64_t len) { - asyncFileLogger.output_(msg, len); - }, - [&]() { asyncFileLogger.flush(); }); - } - CommandLineParser clp; - clp.SetupCommand(argc, argv); - return 0; - } + if (argc > 1 && strcmp(argv[1], "--start-server") == 0) { + RunServer(); + return 0; } - ForkProcess(); + bool verbose = false; + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--verbose") == 0) { + verbose = true; + } + } + if (!verbose) { + auto config = file_manager_utils::GetCortexConfig(); + std::filesystem::create_directories( + std::filesystem::path(config.logFolderPath) / + std::filesystem::path(cortex_utils::logs_folder)); + trantor::FileLogger asyncFileLogger; + asyncFileLogger.setFileName(config.logFolderPath + "/" + + cortex_utils::logs_cli_base_name); + asyncFileLogger.setMaxLines(config.maxLogLines); // Keep last 100000 lines + asyncFileLogger.startLogging(); + trantor::Logger::setOutputFunction( + [&](const char* msg, const uint64_t len) { + asyncFileLogger.output_(msg, len); + }, + [&]() { asyncFileLogger.flush(); }); + } + CommandLineParser clp; + clp.SetupCommand(argc, argv); return 0; } From 43938b8749839ebc7b576b0bb57a781c1576114b Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 12:27:36 +0700 Subject: [PATCH 03/10] feat: add -p, --port for cortex start --- engine/controllers/command_line_parser.cc | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 388a64ea1..73f0d43e5 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -6,15 +6,15 @@ #include "commands/engine_init_cmd.h" #include "commands/engine_list_cmd.h" #include "commands/engine_uninstall_cmd.h" +#include "commands/model_del_cmd.h" #include "commands/model_get_cmd.h" #include "commands/model_list_cmd.h" #include "commands/model_pull_cmd.h" #include "commands/model_start_cmd.h" #include "commands/model_stop_cmd.h" #include "commands/run_cmd.h" -#include "commands/server_stop_cmd.h" -#include "commands/model_del_cmd.h" #include "commands/server_start_cmd.h" +#include "commands/server_stop_cmd.h" #include "config/yaml_config.h" #include "services/engine_service.h" #include "utils/file_manager_utils.h" @@ -176,10 +176,17 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { } auto start_cmd = app_.add_subcommand("start", "Start the API server"); - - start_cmd->callback([&config] { - commands::ServerStartCmd ssc(config.apiServerHost, - std::stoi(config.apiServerPort)); + int port = std::stoi(config.apiServerPort); + start_cmd->add_option("-p, --port", port, "Server port to listen"); + start_cmd->callback([&config, &port] { + if (port != stoi(config.apiServerPort)) { + CTL_INF("apiServerPort changed from " << config.apiServerPort << " to " + << port); + auto config_path = file_manager_utils::GetConfigurationPath(); + config.apiServerPort = std::to_string(port); + config_yaml_utils::DumpYamlConfig(config, config_path.string()); + } + commands::ServerStartCmd ssc(config.apiServerHost, port); ssc.Exec(); }); @@ -217,7 +224,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { } CLI11_PARSE(app_, argc, argv); - if(argc == 1) { + if (argc == 1) { CLI_LOG(app_.help()); return true; } From 4eb8181cd3368434fd981942d08f851d3852a243 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 12:51:24 +0700 Subject: [PATCH 04/10] fix: check if model is started before start model --- engine/commands/model_start_cmd.cc | 15 +++++++++++++++ engine/commands/run_cmd.cc | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/engine/commands/model_start_cmd.cc b/engine/commands/model_start_cmd.cc index 83d051891..b608a2b3d 100644 --- a/engine/commands/model_start_cmd.cc +++ b/engine/commands/model_start_cmd.cc @@ -2,6 +2,7 @@ #include "httplib.h" #include "nlohmann/json.hpp" #include "trantor/utils/Logger.h" +#include "utils/file_manager_utils.h" #include "utils/logging_utils.h" namespace commands { @@ -10,7 +11,21 @@ ModelStartCmd::ModelStartCmd(std::string host, int port, : host_(std::move(host)), port_(port), mc_(mc) {} bool ModelStartCmd::Exec() { + + // Check if server is started + { + httplib::Client cli(host_ + ":" + std::to_string(port_)); + auto res = cli.Get("/health/healthz"); + if (!res || res->status == httplib::StatusCode::OK_200) { + CLI_LOG( + "Server is not started yet, please run `cortex start` to start " + "server!"); + return false; + } + } + httplib::Client cli(host_ + ":" + std::to_string(port_)); + nlohmann::json json_data; if (mc_.files.size() > 0) { // TODO(sang) support multiple files diff --git a/engine/commands/run_cmd.cc b/engine/commands/run_cmd.cc index c8bde1b14..4a2a7f5ca 100644 --- a/engine/commands/run_cmd.cc +++ b/engine/commands/run_cmd.cc @@ -3,8 +3,10 @@ #include "cmd_info.h" #include "config/yaml_config.h" #include "engine_init_cmd.h" +#include "httplib.h" #include "model_pull_cmd.h" #include "model_start_cmd.h" +#include "server_start_cmd.h" #include "trantor/utils/Logger.h" #include "utils/cortex_utils.h" #include "utils/file_manager_utils.h" @@ -41,6 +43,17 @@ void RunCmd::Exec() { } } + // Start server if it is not running + { + httplib::Client cli(host_ + ":" + std::to_string(port_)); + auto res = cli.Get("/health/healthz"); + if (!res || res->status == httplib::StatusCode::OK_200) { + CLI_LOG("Starting server ..."); + commands::ServerStartCmd ssc(host_, port_); + ssc.Exec(); + } + } + // Start model config::YamlHandler yaml_handler; yaml_handler.ModelConfigFromFile( From 3fc09916161b1411186097045a217b4377fb1128 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 13:00:51 +0700 Subject: [PATCH 05/10] fix: correct binary name --- engine/commands/chat_cmd.cc | 12 ++++++++++++ engine/commands/model_start_cmd.cc | 7 +++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/engine/commands/chat_cmd.cc b/engine/commands/chat_cmd.cc index e535fa704..d842b1a9b 100644 --- a/engine/commands/chat_cmd.cc +++ b/engine/commands/chat_cmd.cc @@ -1,6 +1,7 @@ #include "chat_cmd.h" #include "httplib.h" +#include "cortex_upd_cmd.h" #include "trantor/utils/Logger.h" #include "utils/logging_utils.h" @@ -33,6 +34,17 @@ ChatCmd::ChatCmd(std::string host, int port, const config::ModelConfig& mc) : host_(std::move(host)), port_(port), mc_(mc) {} void ChatCmd::Exec(std::string msg) { + // Check if server is started + { + httplib::Client cli(host_ + ":" + std::to_string(port_)); + auto res = cli.Get("/health/healthz"); + if (!res || res->status == httplib::StatusCode::OK_200) { + CLI_LOG("Server is not started yet, please run `" + << commands::GetCortexBinary() << " start` to start server!"); + return; + } + } + auto address = host_ + ":" + std::to_string(port_); // Check if model is loaded // TODO(sang) only llamacpp support modelstatus for now diff --git a/engine/commands/model_start_cmd.cc b/engine/commands/model_start_cmd.cc index b608a2b3d..0352cb8d0 100644 --- a/engine/commands/model_start_cmd.cc +++ b/engine/commands/model_start_cmd.cc @@ -1,4 +1,5 @@ #include "model_start_cmd.h" +#include "cortex_upd_cmd.h" #include "httplib.h" #include "nlohmann/json.hpp" #include "trantor/utils/Logger.h" @@ -11,15 +12,13 @@ ModelStartCmd::ModelStartCmd(std::string host, int port, : host_(std::move(host)), port_(port), mc_(mc) {} bool ModelStartCmd::Exec() { - // Check if server is started { httplib::Client cli(host_ + ":" + std::to_string(port_)); auto res = cli.Get("/health/healthz"); if (!res || res->status == httplib::StatusCode::OK_200) { - CLI_LOG( - "Server is not started yet, please run `cortex start` to start " - "server!"); + CLI_LOG("Server is not started yet, please run `" + << commands::GetCortexBinary() << " start` to start server!"); return false; } } From 7d1141a614273e8d73212c2b20c3b07005a3079d Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 13:16:02 +0700 Subject: [PATCH 06/10] fix: e2e test --- engine/e2e-test/test_runner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/engine/e2e-test/test_runner.py b/engine/e2e-test/test_runner.py index bc89fb836..d9c0a58b3 100644 --- a/engine/e2e-test/test_runner.py +++ b/engine/e2e-test/test_runner.py @@ -7,7 +7,7 @@ from typing import List # You might want to change the path of the executable based on your build directory -executable_windows_path = "build\\Debug\\cortex.exe" +executable_windows_path = "build\\Release\\cortex-nightly.exe" executable_unix_path = "build/cortex" # Timeout @@ -47,7 +47,10 @@ def start_server() -> bool: def start_server_nix() -> bool: executable = getExecutablePath() process = subprocess.Popen( - executable, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + [executable] + ['start', '-p', '3928'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True ) start_time = time.time() @@ -74,7 +77,7 @@ def start_server_nix() -> bool: def start_server_windows() -> bool: executable = getExecutablePath() process = subprocess.Popen( - executable, + [executable] + ['start', '-p', '3928'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, From 9daa572ea08b9819ed424630791e2a6103ee0720 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 13:43:17 +0700 Subject: [PATCH 07/10] fix: host, port from config --- engine/commands/run_cmd.cc | 2 +- engine/commands/server_start_cmd.cc | 3 +-- engine/commands/server_start_cmd.h | 6 +----- engine/controllers/command_line_parser.cc | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/engine/commands/run_cmd.cc b/engine/commands/run_cmd.cc index 3ec08c3cf..ca1695324 100644 --- a/engine/commands/run_cmd.cc +++ b/engine/commands/run_cmd.cc @@ -47,7 +47,7 @@ void RunCmd::Exec() { auto res = cli.Get("/health/healthz"); if (!res || res->status == httplib::StatusCode::OK_200) { CLI_LOG("Starting server ..."); - commands::ServerStartCmd ssc(host_, port_); + commands::ServerStartCmd ssc; ssc.Exec(); } } diff --git a/engine/commands/server_start_cmd.cc b/engine/commands/server_start_cmd.cc index 7b7a9f380..a3d7fce71 100644 --- a/engine/commands/server_start_cmd.cc +++ b/engine/commands/server_start_cmd.cc @@ -7,8 +7,7 @@ #include "utils/logging_utils.h" namespace commands { -ServerStartCmd::ServerStartCmd(std::string host, int port) - : host_(std::move(host)), port_(port) {} +ServerStartCmd::ServerStartCmd() {} void ServerStartCmd::Exec() { #if defined(_WIN32) || defined(_WIN64) diff --git a/engine/commands/server_start_cmd.h b/engine/commands/server_start_cmd.h index 6f121d6bb..9eeb28b50 100644 --- a/engine/commands/server_start_cmd.h +++ b/engine/commands/server_start_cmd.h @@ -5,11 +5,7 @@ namespace commands { class ServerStartCmd{ public: - ServerStartCmd(std::string host, int port); + ServerStartCmd(); void Exec(); - - private: - std::string host_; - int port_; }; } // namespace commands \ No newline at end of file diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 97aa69508..e05e825b0 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -186,7 +186,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { config.apiServerPort = std::to_string(port); config_yaml_utils::DumpYamlConfig(config, config_path.string()); } - commands::ServerStartCmd ssc(config.apiServerHost, port); + commands::ServerStartCmd ssc; ssc.Exec(); }); From 1affcf89819952a87360b48aabdf6e21f16982e8 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 13:55:18 +0700 Subject: [PATCH 08/10] fix: correct condition --- engine/commands/chat_cmd.cc | 2 +- engine/commands/model_start_cmd.cc | 2 +- engine/commands/run_cmd.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/commands/chat_cmd.cc b/engine/commands/chat_cmd.cc index d842b1a9b..18e8ba507 100644 --- a/engine/commands/chat_cmd.cc +++ b/engine/commands/chat_cmd.cc @@ -38,7 +38,7 @@ void ChatCmd::Exec(std::string msg) { { httplib::Client cli(host_ + ":" + std::to_string(port_)); auto res = cli.Get("/health/healthz"); - if (!res || res->status == httplib::StatusCode::OK_200) { + if (!res || res->status != httplib::StatusCode::OK_200) { CLI_LOG("Server is not started yet, please run `" << commands::GetCortexBinary() << " start` to start server!"); return; diff --git a/engine/commands/model_start_cmd.cc b/engine/commands/model_start_cmd.cc index 0352cb8d0..051db800b 100644 --- a/engine/commands/model_start_cmd.cc +++ b/engine/commands/model_start_cmd.cc @@ -16,7 +16,7 @@ bool ModelStartCmd::Exec() { { httplib::Client cli(host_ + ":" + std::to_string(port_)); auto res = cli.Get("/health/healthz"); - if (!res || res->status == httplib::StatusCode::OK_200) { + if (!res || res->status != httplib::StatusCode::OK_200) { CLI_LOG("Server is not started yet, please run `" << commands::GetCortexBinary() << " start` to start server!"); return false; diff --git a/engine/commands/run_cmd.cc b/engine/commands/run_cmd.cc index ca1695324..55d19086b 100644 --- a/engine/commands/run_cmd.cc +++ b/engine/commands/run_cmd.cc @@ -45,7 +45,7 @@ void RunCmd::Exec() { { httplib::Client cli(host_ + ":" + std::to_string(port_)); auto res = cli.Get("/health/healthz"); - if (!res || res->status == httplib::StatusCode::OK_200) { + if (!res || res->status != httplib::StatusCode::OK_200) { CLI_LOG("Starting server ..."); commands::ServerStartCmd ssc; ssc.Exec(); From c4685541913845aca762a76c2a62db615f964377 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 14:23:41 +0700 Subject: [PATCH 09/10] fix: correct url --- engine/commands/chat_cmd.cc | 2 +- engine/commands/cortex_upd_cmd.cc | 2 +- engine/commands/model_start_cmd.cc | 2 +- engine/commands/run_cmd.cc | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/commands/chat_cmd.cc b/engine/commands/chat_cmd.cc index 18e8ba507..f79b064ec 100644 --- a/engine/commands/chat_cmd.cc +++ b/engine/commands/chat_cmd.cc @@ -37,7 +37,7 @@ void ChatCmd::Exec(std::string msg) { // Check if server is started { httplib::Client cli(host_ + ":" + std::to_string(port_)); - auto res = cli.Get("/health/healthz"); + auto res = cli.Get("/healthz"); if (!res || res->status != httplib::StatusCode::OK_200) { CLI_LOG("Server is not started yet, please run `" << commands::GetCortexBinary() << " start` to start server!"); diff --git a/engine/commands/cortex_upd_cmd.cc b/engine/commands/cortex_upd_cmd.cc index 7c2a1f423..3c892f6fc 100644 --- a/engine/commands/cortex_upd_cmd.cc +++ b/engine/commands/cortex_upd_cmd.cc @@ -15,7 +15,7 @@ void CortexUpdCmd::Exec(std::string v) { { auto config = file_manager_utils::GetCortexConfig(); httplib::Client cli(config.apiServerHost + ":" + config.apiServerPort); - auto res = cli.Get("/health/healthz"); + auto res = cli.Get("/healthz"); if (res) { CLI_LOG("Server is running. Stopping server before updating!"); commands::ServerStopCmd ssc(config.apiServerHost, diff --git a/engine/commands/model_start_cmd.cc b/engine/commands/model_start_cmd.cc index 051db800b..031324648 100644 --- a/engine/commands/model_start_cmd.cc +++ b/engine/commands/model_start_cmd.cc @@ -15,7 +15,7 @@ bool ModelStartCmd::Exec() { // Check if server is started { httplib::Client cli(host_ + ":" + std::to_string(port_)); - auto res = cli.Get("/health/healthz"); + auto res = cli.Get("/healthz"); if (!res || res->status != httplib::StatusCode::OK_200) { CLI_LOG("Server is not started yet, please run `" << commands::GetCortexBinary() << " start` to start server!"); diff --git a/engine/commands/run_cmd.cc b/engine/commands/run_cmd.cc index 55d19086b..6ef5fae88 100644 --- a/engine/commands/run_cmd.cc +++ b/engine/commands/run_cmd.cc @@ -44,7 +44,7 @@ void RunCmd::Exec() { // Start server if it is not running { httplib::Client cli(host_ + ":" + std::to_string(port_)); - auto res = cli.Get("/health/healthz"); + auto res = cli.Get("/healthz"); if (!res || res->status != httplib::StatusCode::OK_200) { CLI_LOG("Starting server ..."); commands::ServerStartCmd ssc; From 1589b90d48cd77837dbf9755e46346cbe2343e6c Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 17 Sep 2024 14:52:01 +0700 Subject: [PATCH 10/10] fix: add retry check health status --- engine/commands/chat_cmd.cc | 5 ++-- engine/commands/model_start_cmd.cc | 13 ++++----- engine/commands/run_cmd.cc | 8 +++--- engine/commands/server_start_cmd.cc | 32 +++++++++++++++++++++-- engine/commands/server_start_cmd.h | 14 ++++++++-- engine/controllers/command_line_parser.cc | 2 +- 6 files changed, 54 insertions(+), 20 deletions(-) diff --git a/engine/commands/chat_cmd.cc b/engine/commands/chat_cmd.cc index f79b064ec..67ed651a4 100644 --- a/engine/commands/chat_cmd.cc +++ b/engine/commands/chat_cmd.cc @@ -4,6 +4,7 @@ #include "cortex_upd_cmd.h" #include "trantor/utils/Logger.h" #include "utils/logging_utils.h" +#include "server_start_cmd.h" namespace commands { namespace { @@ -36,9 +37,7 @@ ChatCmd::ChatCmd(std::string host, int port, const config::ModelConfig& mc) void ChatCmd::Exec(std::string msg) { // Check if server is started { - httplib::Client cli(host_ + ":" + std::to_string(port_)); - auto res = cli.Get("/healthz"); - if (!res || res->status != httplib::StatusCode::OK_200) { + if (!commands::IsServerAlive(host_, port_)) { CLI_LOG("Server is not started yet, please run `" << commands::GetCortexBinary() << " start` to start server!"); return; diff --git a/engine/commands/model_start_cmd.cc b/engine/commands/model_start_cmd.cc index 031324648..2eb137dac 100644 --- a/engine/commands/model_start_cmd.cc +++ b/engine/commands/model_start_cmd.cc @@ -2,6 +2,7 @@ #include "cortex_upd_cmd.h" #include "httplib.h" #include "nlohmann/json.hpp" +#include "server_start_cmd.h" #include "trantor/utils/Logger.h" #include "utils/file_manager_utils.h" #include "utils/logging_utils.h" @@ -13,14 +14,10 @@ ModelStartCmd::ModelStartCmd(std::string host, int port, bool ModelStartCmd::Exec() { // Check if server is started - { - httplib::Client cli(host_ + ":" + std::to_string(port_)); - auto res = cli.Get("/healthz"); - if (!res || res->status != httplib::StatusCode::OK_200) { - CLI_LOG("Server is not started yet, please run `" - << commands::GetCortexBinary() << " start` to start server!"); - return false; - } + if (!commands::IsServerAlive(host_, port_)) { + CLI_LOG("Server is not started yet, please run `" + << commands::GetCortexBinary() << " start` to start server!"); + return false; } httplib::Client cli(host_ + ":" + std::to_string(port_)); diff --git a/engine/commands/run_cmd.cc b/engine/commands/run_cmd.cc index 6ef5fae88..1fb3706d7 100644 --- a/engine/commands/run_cmd.cc +++ b/engine/commands/run_cmd.cc @@ -43,12 +43,12 @@ void RunCmd::Exec() { // Start server if it is not running { - httplib::Client cli(host_ + ":" + std::to_string(port_)); - auto res = cli.Get("/healthz"); - if (!res || res->status != httplib::StatusCode::OK_200) { + if (!commands::IsServerAlive(host_, port_)) { CLI_LOG("Starting server ..."); commands::ServerStartCmd ssc; - ssc.Exec(); + if(!ssc.Exec(host_, port_)) { + return; + } } } diff --git a/engine/commands/server_start_cmd.cc b/engine/commands/server_start_cmd.cc index a3d7fce71..613554c83 100644 --- a/engine/commands/server_start_cmd.cc +++ b/engine/commands/server_start_cmd.cc @@ -7,9 +7,29 @@ #include "utils/logging_utils.h" namespace commands { + +namespace { +bool TryConnectToServer(const std::string& host, int port) { + constexpr const auto kMaxRetry = 3u; + auto count = 0u; + // Check if server is started + while (true) { + if (IsServerAlive(host, port)) + break; + // Wait for server up + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (count++ == kMaxRetry) { + std::cerr << "Could not start server" << std::endl; + return false; + } + } + return true; +} +} // namespace + ServerStartCmd::ServerStartCmd() {} -void ServerStartCmd::Exec() { +bool ServerStartCmd::Exec(const std::string& host, int port) { #if defined(_WIN32) || defined(_WIN64) // Windows-specific code to create a new process STARTUPINFO si; @@ -36,7 +56,11 @@ void ServerStartCmd::Exec() { &pi)) // Pointer to PROCESS_INFORMATION structure { std::cout << "Could not start server: " << GetLastError() << std::endl; + return false; } else { + if(!TryConnectToServer(host, port)) { + return false; + } std::cout << "Server started" << std::endl; } @@ -47,7 +71,7 @@ void ServerStartCmd::Exec() { if (pid < 0) { // Fork failed std::cerr << "Could not start server: " << std::endl; - return; + return false; } else if (pid == 0) { // No need to configure LD_LIBRARY_PATH for macOS #if !defined(__APPLE__) || !defined(__MACH__) @@ -70,9 +94,13 @@ void ServerStartCmd::Exec() { execl(p.c_str(), exe.c_str(), "--start-server", (char*)0); } else { // Parent process + if(!TryConnectToServer(host, port)) { + return false; + } std::cout << "Server started" << std::endl; } #endif + return true; } }; // namespace commands \ No newline at end of file diff --git a/engine/commands/server_start_cmd.h b/engine/commands/server_start_cmd.h index 9eeb28b50..cb74c5ebc 100644 --- a/engine/commands/server_start_cmd.h +++ b/engine/commands/server_start_cmd.h @@ -1,11 +1,21 @@ #pragma once #include +#include "httplib.h" namespace commands { -class ServerStartCmd{ +inline bool IsServerAlive(const std::string& host, int port) { + httplib::Client cli(host + ":" + std::to_string(port)); + auto res = cli.Get("/healthz"); + if (res && res->status == httplib::StatusCode::OK_200) { + return true; + } + return false; +} + +class ServerStartCmd { public: ServerStartCmd(); - void Exec(); + bool Exec(const std::string& host, int port); }; } // namespace commands \ No newline at end of file diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index e05e825b0..3f8b01136 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -187,7 +187,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { config_yaml_utils::DumpYamlConfig(config, config_path.string()); } commands::ServerStartCmd ssc; - ssc.Exec(); + ssc.Exec(config.apiServerHost, std::stoi(config.apiServerPort)); }); auto stop_cmd = app_.add_subcommand("stop", "Stop the API server");