diff --git a/engine/commands/cortex_upd_cmd.cc b/engine/commands/cortex_upd_cmd.cc index b8edede53..9947b1da0 100644 --- a/engine/commands/cortex_upd_cmd.cc +++ b/engine/commands/cortex_upd_cmd.cc @@ -12,7 +12,181 @@ namespace commands { -void CortexUpdCmd::Exec(std::string v) { +namespace { +std::chrono::seconds GetUpdateIntervalCheck() { + if (CORTEX_VARIANT == file_manager_utils::kNightlyVariant) { + return std::chrono::seconds(10 * 60); + } else if (CORTEX_VARIANT == file_manager_utils::kBetaVariant) { + return std::chrono::seconds(60 * 60); + } else { + return std::chrono::seconds(24 * 60 * 60); + } +} + +std::chrono::seconds GetTimeSinceEpochMillisec() { + using namespace std::chrono; + return duration_cast(system_clock::now().time_since_epoch()); +} +} // namespace + +std::optional CheckNewUpdate( + std::optional timeout) { + // Get info from .cortexrc + auto should_check_update = false; + auto config = file_manager_utils::GetCortexConfig(); + auto now = GetTimeSinceEpochMillisec(); + if (auto t = now - std::chrono::seconds(config.checkedForUpdateAt); + t > GetUpdateIntervalCheck()) { + should_check_update = true; + config.checkedForUpdateAt = now.count(); + CTL_INF("Will check for new update, time from last check: " << t.count() + << " seconds"); + } + + if (!should_check_update) { + CTL_INF("Will not check for new update, return the cache latest: " + << config.latestRelease); + return config.latestRelease; + } + + auto host_name = GetHostName(); + auto release_path = GetReleasePath(); + CTL_INF("Engine release path: " << host_name << release_path); + + httplib::Client cli(host_name); + if (timeout.has_value()) { + cli.set_connection_timeout(*timeout); + cli.set_read_timeout(*timeout); + } + if (auto res = cli.Get(release_path)) { + if (res->status == httplib::StatusCode::OK_200) { + try { + auto get_latest = [](const nlohmann::json& data) -> std::string { + if (data.empty()) { + return ""; + } + + if (CORTEX_VARIANT == file_manager_utils::kBetaVariant) { + for (auto& d : data) { + if (auto tag = d["tag_name"].get(); + tag.find(kBetaComp) != std::string::npos) { + return tag; + } + } + return data[0]["tag_name"].get(); + } else { + return data["tag_name"].get(); + } + return ""; + }; + + auto json_res = nlohmann::json::parse(res->body); + std::string latest_version = get_latest(json_res); + if (latest_version.empty()) { + CTL_WRN("Release not found!"); + return std::nullopt; + } + std::string current_version = CORTEX_CPP_VERSION; + CTL_INF("Got the latest release, update to the config file: " + << latest_version) + config.latestRelease = latest_version; + config_yaml_utils::DumpYamlConfig( + config, file_manager_utils::GetConfigurationPath().string()); + if (current_version != latest_version) { + return latest_version; + } + } catch (const nlohmann::json::parse_error& e) { + CTL_INF("JSON parse error: " << e.what()); + return std::nullopt; + } + } else { + CTL_INF("HTTP error: " << res->status); + return std::nullopt; + } + } else { + auto err = res.error(); + CTL_INF("HTTP error: " << httplib::to_string(err)); + return std::nullopt; + } + return std::nullopt; +} + +bool ReplaceBinaryInflight(const std::filesystem::path& src, + const std::filesystem::path& dst) { + if (src == dst) { + // Already has the newest + return true; + } + + std::filesystem::path temp = dst.parent_path() / "cortex_temp"; + auto restore_binary = [&temp, &dst]() { + if (std::filesystem::exists(temp)) { + std::rename(temp.string().c_str(), dst.string().c_str()); + CLI_LOG("Restored binary file"); + } + }; + + try { + if (std::filesystem::exists(temp)) { + std::filesystem::remove(temp); + } +#if !defined(_WIN32) + // Get permissions of the executable file + struct stat dst_file_stat; + if (stat(dst.string().c_str(), &dst_file_stat) != 0) { + CTL_ERR("Error getting permissions of executable file: " << dst.string()); + return false; + } + + // Get owner and group of the executable file + uid_t dst_file_owner = dst_file_stat.st_uid; + gid_t dst_file_group = dst_file_stat.st_gid; +#endif + + std::rename(dst.string().c_str(), temp.string().c_str()); + std::filesystem::copy_file( + src, dst, std::filesystem::copy_options::overwrite_existing); + +#if !defined(_WIN32) + // Set permissions of the executable file + if (chmod(dst.string().c_str(), dst_file_stat.st_mode) != 0) { + CTL_ERR("Error setting permissions of executable file: " << dst.string()); + restore_binary(); + return false; + } + + // Set owner and group of the executable file + if (chown(dst.string().c_str(), dst_file_owner, dst_file_group) != 0) { + CTL_ERR( + "Error setting owner and group of executable file: " << dst.string()); + restore_binary(); + return false; + } + + // Remove cortex_temp + if (unlink(temp.string().c_str()) != 0) { + CTL_ERR("Error deleting self: " << strerror(errno)); + restore_binary(); + return false; + } +#endif + } catch (const std::exception& e) { + CTL_ERR("Something went wrong: " << e.what()); + restore_binary(); + return false; + } + + return true; +} + +void CortexUpdCmd::Exec(const std::string& v) { + // Check for update, if current version is the latest, notify to user + if (auto latest_version = commands::CheckNewUpdate(std::nullopt); + latest_version.has_value() && *latest_version == CORTEX_CPP_VERSION) { + CLI_LOG("cortex is up to date"); + return; + } + { auto config = file_manager_utils::GetCortexConfig(); httplib::Client cli(config.apiServerHost + ":" + config.apiServerPort); diff --git a/engine/commands/cortex_upd_cmd.h b/engine/commands/cortex_upd_cmd.h index 142f7c9c9..c332807d7 100644 --- a/engine/commands/cortex_upd_cmd.h +++ b/engine/commands/cortex_upd_cmd.h @@ -63,131 +63,11 @@ inline std::string GetReleasePath() { } } -inline void CheckNewUpdate() { - auto host_name = GetHostName(); - auto release_path = GetReleasePath(); - CTL_INF("Engine release path: " << host_name << release_path); +std::optional CheckNewUpdate( + std::optional timeout); - httplib::Client cli(host_name); - cli.set_connection_timeout(kTimeoutCheckUpdate); - cli.set_read_timeout(kTimeoutCheckUpdate); - if (auto res = cli.Get(release_path)) { - if (res->status == httplib::StatusCode::OK_200) { - try { - auto get_latest = [](const nlohmann::json& data) -> std::string { - if (data.empty()) { - return ""; - } - - if (CORTEX_VARIANT == file_manager_utils::kBetaVariant) { - for (auto& d : data) { - if (auto tag = d["tag_name"].get(); - tag.find(kBetaComp) != std::string::npos) { - return tag; - } - } - return data[0]["tag_name"].get(); - } else { - return data["tag_name"].get(); - } - return ""; - }; - - auto json_res = nlohmann::json::parse(res->body); - std::string latest_version = get_latest(json_res); - if (latest_version.empty()) { - CTL_WRN("Release not found!"); - return; - } - std::string current_version = CORTEX_CPP_VERSION; - if (current_version != latest_version) { - CLI_LOG("\nA new release of cortex is available: " - << current_version << " -> " << latest_version); - CLI_LOG("To upgrade, run: " << GetRole() << GetCortexBinary() - << " update"); - if (CORTEX_VARIANT == file_manager_utils::kProdVariant) { - CLI_LOG(json_res["html_url"].get()); - } - } - } catch (const nlohmann::json::parse_error& e) { - CTL_INF("JSON parse error: " << e.what()); - } - } else { - CTL_INF("HTTP error: " << res->status); - } - } else { - auto err = res.error(); - CTL_INF("HTTP error: " << httplib::to_string(err)); - } -} - -inline bool ReplaceBinaryInflight(const std::filesystem::path& src, - const std::filesystem::path& dst) { - if (src == dst) { - // Already has the newest - return true; - } - - std::filesystem::path temp = dst.parent_path() / "cortex_temp"; - auto restore_binary = [&temp, &dst]() { - if (std::filesystem::exists(temp)) { - std::rename(temp.string().c_str(), dst.string().c_str()); - CLI_LOG("Restored binary file"); - } - }; - - try { - if (std::filesystem::exists(temp)) { - std::filesystem::remove(temp); - } -#if !defined(_WIN32) - // Get permissions of the executable file - struct stat dst_file_stat; - if (stat(dst.string().c_str(), &dst_file_stat) != 0) { - CTL_ERR("Error getting permissions of executable file: " << dst.string()); - return false; - } - - // Get owner and group of the executable file - uid_t dst_file_owner = dst_file_stat.st_uid; - gid_t dst_file_group = dst_file_stat.st_gid; -#endif - - std::rename(dst.string().c_str(), temp.string().c_str()); - std::filesystem::copy_file( - src, dst, std::filesystem::copy_options::overwrite_existing); - -#if !defined(_WIN32) - // Set permissions of the executable file - if (chmod(dst.string().c_str(), dst_file_stat.st_mode) != 0) { - CTL_ERR("Error setting permissions of executable file: " << dst.string()); - restore_binary(); - return false; - } - - // Set owner and group of the executable file - if (chown(dst.string().c_str(), dst_file_owner, dst_file_group) != 0) { - CTL_ERR( - "Error setting owner and group of executable file: " << dst.string()); - restore_binary(); - return false; - } - - // Remove cortex_temp - if (unlink(temp.string().c_str()) != 0) { - CTL_ERR("Error deleting self: " << strerror(errno)); - restore_binary(); - return false; - } -#endif - } catch (const std::exception& e) { - CTL_ERR("Something went wrong: " << e.what()); - restore_binary(); - return false; - } - - return true; -} +bool ReplaceBinaryInflight(const std::filesystem::path& src, + const std::filesystem::path& dst); // This class manages the 'cortex update' command functionality // There are three release types available: @@ -196,7 +76,7 @@ inline bool ReplaceBinaryInflight(const std::filesystem::path& src, // - Nightly: Enables retrieval of the latest nightly build and specific versions using the -v flag class CortexUpdCmd { public: - void Exec(std::string version); + void Exec(const std::string& v); private: bool GetStable(const std::string& v); diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index f9b6a04e6..da59d9d5d 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -72,7 +72,13 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { // Check new update, only check for stable release for now #ifdef CORTEX_CPP_VERSION if (cml_data_.check_upd) { - commands::CheckNewUpdate(); + if (auto latest_version = commands::CheckNewUpdate(commands::kTimeoutCheckUpdate); + latest_version.has_value() && *latest_version != CORTEX_CPP_VERSION) { + CLI_LOG("\nA new release of cortex is available: " + << CORTEX_CPP_VERSION << " -> " << *latest_version); + CLI_LOG("To upgrade, run: " << commands::GetRole() + << commands::GetCortexBinary() << " update"); + } } #endif diff --git a/engine/test/components/CMakeLists.txt b/engine/test/components/CMakeLists.txt index f89881118..13bb9c526 100644 --- a/engine/test/components/CMakeLists.txt +++ b/engine/test/components/CMakeLists.txt @@ -3,17 +3,31 @@ project(test-components) enable_testing() -add_executable(${PROJECT_NAME} ${SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/modellist_utils.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../config/yaml_config.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../config/gguf_parser.cc) +add_executable(${PROJECT_NAME} + ${SRCS} + ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/modellist_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../config/yaml_config.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../config/gguf_parser.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../commands/cortex_upd_cmd.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../commands/server_stop_cmd.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../services/download_service.cc +) find_package(Drogon CONFIG REQUIRED) find_package(GTest CONFIG REQUIRED) find_package(yaml-cpp CONFIG REQUIRED) find_package(httplib CONFIG REQUIRED) +find_package(unofficial-minizip CONFIG REQUIRED) +find_package(LibArchive REQUIRED) +find_package(CURL REQUIRED) target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon GTest::gtest GTest::gtest_main yaml-cpp::yaml-cpp ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(${PROJECT_NAME} PRIVATE httplib::httplib) +target_link_libraries(${PROJECT_NAME} PRIVATE unofficial::minizip::minizip) +target_link_libraries(${PROJECT_NAME} PRIVATE LibArchive::LibArchive) +target_link_libraries(${PROJECT_NAME} PRIVATE CURL::libcurl) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../) add_test(NAME ${PROJECT_NAME} diff --git a/engine/test/components/test_cortex_config.cc b/engine/test/components/test_cortex_config.cc new file mode 100644 index 000000000..513716940 --- /dev/null +++ b/engine/test/components/test_cortex_config.cc @@ -0,0 +1,96 @@ +#include "gtest/gtest.h" +#include "utils/config_yaml_utils.h" + +namespace config_yaml_utils { +class CortexConfigTest : public ::testing::Test { + protected: + const std::string test_file_path = "test_config.yaml"; + CortexConfig default_config; + + void SetUp() override { + // Set up default configuration + default_config = { + "default_log_path", "default_data_path", 1000, + kDefaultHost, kDefaultPort, kDefaultCheckedForUpdateAt, + kDefaultLatestRelease}; + } + + void TearDown() override { + // Clean up: remove the test file if it exists + if (std::filesystem::exists(test_file_path)) { + std::filesystem::remove(test_file_path); + } + } +}; + +TEST_F(CortexConfigTest, DumpYamlConfig_WritesCorrectly) { + CortexConfig config = {"log_path", "data_path", 5000, "localhost", + "8080", 123456789, "v1.0.0"}; + + DumpYamlConfig(config, test_file_path); + + // Verify that the file was created and contains the expected data + YAML::Node node = YAML::LoadFile(test_file_path); + EXPECT_EQ(node["logFolderPath"].as(), config.logFolderPath); + EXPECT_EQ(node["dataFolderPath"].as(), config.dataFolderPath); + EXPECT_EQ(node["maxLogLines"].as(), config.maxLogLines); + EXPECT_EQ(node["apiServerHost"].as(), config.apiServerHost); + EXPECT_EQ(node["apiServerPort"].as(), config.apiServerPort); + EXPECT_EQ(node["checkedForUpdateAt"].as(), + config.checkedForUpdateAt); + EXPECT_EQ(node["latestRelease"].as(), config.latestRelease); +} + +TEST_F(CortexConfigTest, FromYaml_ReadsCorrectly) { + // First, create a valid YAML configuration file + CortexConfig config = {"log_path", "data_path", 5000, "localhost", + "8080", 123456789, "v1.0.0"}; + + DumpYamlConfig(config, test_file_path); + + // Now read from the YAML file + CortexConfig loaded_config = FromYaml(test_file_path, default_config); + + // Verify that the loaded configuration matches what was written + EXPECT_EQ(loaded_config.logFolderPath, config.logFolderPath); + EXPECT_EQ(loaded_config.dataFolderPath, config.dataFolderPath); + EXPECT_EQ(loaded_config.maxLogLines, config.maxLogLines); + EXPECT_EQ(loaded_config.apiServerHost, config.apiServerHost); + EXPECT_EQ(loaded_config.apiServerPort, config.apiServerPort); + EXPECT_EQ(loaded_config.checkedForUpdateAt, config.checkedForUpdateAt); + EXPECT_EQ(loaded_config.latestRelease, config.latestRelease); +} + +TEST_F(CortexConfigTest, FromYaml_FileNotFound) { + std::filesystem::remove(test_file_path); // Ensure the file does not exist + + EXPECT_THROW( + { FromYaml(test_file_path, default_config); }, + std::runtime_error); // Expect a runtime error due to missing file +} + +TEST_F(CortexConfigTest, FromYaml_IncompleteConfigUsesDefaults) { + // Create an incomplete YAML configuration file + std::ofstream out_file(test_file_path); + out_file << "logFolderPath: log_path\n"; // Missing other fields + out_file.close(); + + CortexConfig loaded_config = FromYaml(test_file_path, default_config); + + // Verify that defaults are used where values are missing + EXPECT_EQ(loaded_config.logFolderPath, "log_path"); + EXPECT_EQ(loaded_config.dataFolderPath, + default_config.dataFolderPath); // Default value + EXPECT_EQ(loaded_config.maxLogLines, + default_config.maxLogLines); // Default value + EXPECT_EQ(loaded_config.apiServerHost, + default_config.apiServerHost); // Default value + EXPECT_EQ(loaded_config.apiServerPort, + default_config.apiServerPort); // Default value + EXPECT_EQ(loaded_config.checkedForUpdateAt, + default_config.checkedForUpdateAt); // Default value + EXPECT_EQ(loaded_config.latestRelease, + default_config.latestRelease); // Default value +} + +} // namespace config_yaml_utils \ No newline at end of file diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 8e3668292..22672b5ac 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -13,12 +13,16 @@ struct CortexConfig { int maxLogLines; std::string apiServerHost; std::string apiServerPort; + uint64_t checkedForUpdateAt; + std::string latestRelease; }; const std::string kCortexFolderName = "cortexcpp"; const std::string kDefaultHost{"127.0.0.1"}; const std::string kDefaultPort{"3928"}; const int kDefaultMaxLines{100000}; +constexpr const uint64_t kDefaultCheckedForUpdateAt = 0u; +constexpr const auto kDefaultLatestRelease = "default_version"; inline void DumpYamlConfig(const CortexConfig& config, const std::string& path) { @@ -35,6 +39,8 @@ inline void DumpYamlConfig(const CortexConfig& config, node["maxLogLines"] = config.maxLogLines; node["apiServerHost"] = config.apiServerHost; node["apiServerPort"] = config.apiServerPort; + node["checkedForUpdateAt"] = config.checkedForUpdateAt; + node["latestRelease"] = config.latestRelease; out_file << node; out_file.close(); @@ -45,7 +51,7 @@ inline void DumpYamlConfig(const CortexConfig& config, } inline CortexConfig FromYaml(const std::string& path, - const std::string& variant) { + const CortexConfig& default_cfg) { std::filesystem::path config_file_path{path}; if (!std::filesystem::exists(config_file_path)) { throw std::runtime_error("File not found: " + path); @@ -53,19 +59,37 @@ inline CortexConfig FromYaml(const std::string& path, try { auto node = YAML::LoadFile(config_file_path.string()); - int max_lines; - if (!node["maxLogLines"]) { - max_lines = kDefaultMaxLines; - } else { - max_lines = node["maxLogLines"].as(); - } + bool should_update_config = + (!node["logFolderPath"] || !node["dataFolderPath"] || + !node["maxLogLines"] || !node["apiServerHost"] || + !node["apiServerPort"] || !node["checkedForUpdateAt"] || + !node["latestRelease"]); + CortexConfig config = { - .logFolderPath = node["logFolderPath"].as(), - .dataFolderPath = node["dataFolderPath"].as(), - .maxLogLines = max_lines, - .apiServerHost = node["apiServerHost"].as(), - .apiServerPort = node["apiServerPort"].as(), + .logFolderPath = node["logFolderPath"] + ? node["logFolderPath"].as() + : default_cfg.logFolderPath, + .dataFolderPath = node["dataFolderPath"] + ? node["dataFolderPath"].as() + : default_cfg.dataFolderPath, + .maxLogLines = node["maxLogLines"] ? node["maxLogLines"].as() + : default_cfg.maxLogLines, + .apiServerHost = node["apiServerHost"] + ? node["apiServerHost"].as() + : default_cfg.apiServerHost, + .apiServerPort = node["apiServerPort"] + ? node["apiServerPort"].as() + : default_cfg.apiServerPort, + .checkedForUpdateAt = node["checkedForUpdateAt"] + ? node["checkedForUpdateAt"].as() + : default_cfg.checkedForUpdateAt, + .latestRelease = node["latestRelease"] + ? node["latestRelease"].as() + : default_cfg.latestRelease, }; + if (should_update_config) { + DumpYamlConfig(config, path); + } return config; } catch (const YAML::BadFile& e) { CTL_ERR("Failed to read file: " << e.what()); diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index 57ce2f917..d849d550c 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -109,17 +109,11 @@ inline std::filesystem::path GetConfigurationPath() { return configuration_path; } -inline void CreateConfigFileIfNotExist() { - auto config_path = GetConfigurationPath(); - if (std::filesystem::exists(config_path)) { - // already exists - return; - } +inline std::string GetDefaultDataFolderName() { #ifndef CORTEX_VARIANT #define CORTEX_VARIANT "prod" #endif std::string default_data_folder_name{config_yaml_utils::kCortexFolderName}; - std::string variant{CORTEX_VARIANT}; std::string env_postfix{""}; if (variant == kBetaVariant) { @@ -128,6 +122,17 @@ inline void CreateConfigFileIfNotExist() { env_postfix.append("-").append(kNightlyVariant); } default_data_folder_name.append(env_postfix); + return default_data_folder_name; +} + +inline void CreateConfigFileIfNotExist() { + auto config_path = GetConfigurationPath(); + if (std::filesystem::exists(config_path)) { + // already exists + return; + } + + auto default_data_folder_name = GetDefaultDataFolderName(); CLI_LOG("Config file not found. Creating one at " + config_path.string()); auto defaultDataFolderPath = @@ -146,8 +151,20 @@ inline void CreateConfigFileIfNotExist() { inline config_yaml_utils::CortexConfig GetCortexConfig() { auto config_path = GetConfigurationPath(); - std::string variant = ""; // TODO: empty for now - return config_yaml_utils::FromYaml(config_path.string(), variant); + auto default_data_folder_name = GetDefaultDataFolderName(); + auto default_data_folder_path = + file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name; + auto default_cfg = config_yaml_utils::CortexConfig{ + .logFolderPath = default_data_folder_path.string(), + .dataFolderPath = default_data_folder_path.string(), + .maxLogLines = config_yaml_utils::kDefaultMaxLines, + .apiServerHost = config_yaml_utils::kDefaultHost, + .apiServerPort = config_yaml_utils::kDefaultPort, + .checkedForUpdateAt = config_yaml_utils::kDefaultCheckedForUpdateAt, + .latestRelease = config_yaml_utils::kDefaultLatestRelease, + }; + + return config_yaml_utils::FromYaml(config_path.string(), default_cfg); } inline std::filesystem::path GetCortexDataPath() {