diff --git a/engine/commands/engine_get_cmd.cc b/engine/commands/engine_get_cmd.cc index aa4cbabf2..67818a537 100644 --- a/engine/commands/engine_get_cmd.cc +++ b/engine/commands/engine_get_cmd.cc @@ -8,15 +8,17 @@ namespace commands { void EngineGetCmd::Exec(const std::string& engine_name) const { auto engine = engine_service_.GetEngineInfo(engine_name); - if (engine == std::nullopt) { - CLI_LOG("Engine " + engine_name + " is not supported!"); + if (engine.has_error()) { + CLI_LOG(engine.error()); return; } + auto version = engine->version.value_or(""); + auto variant = engine->variant.value_or(""); tabulate::Table table; - table.add_row({"Name", "Supported Formats", "Version", "Status"}); + table.add_row({"Name", "Supported Formats", "Version", "Variant", "Status"}); table.add_row( - {engine->product_name, engine->format, engine->version, engine->status}); + {engine->product_name, engine->format, version, variant, engine->status}); std::cout << table << std::endl; } }; // namespace commands diff --git a/engine/commands/engine_list_cmd.cc b/engine/commands/engine_list_cmd.cc index 4fe1b1321..0492a2852 100644 --- a/engine/commands/engine_list_cmd.cc +++ b/engine/commands/engine_list_cmd.cc @@ -9,12 +9,15 @@ bool EngineListCmd::Exec() { auto status_list = engine_service.GetEngineInfoList(); tabulate::Table table; - table.add_row({"#", "Name", "Supported Formats", "Version", "Status"}); + table.add_row( + {"#", "Name", "Supported Formats", "Version", "Variant", "Status"}); for (int i = 0; i < status_list.size(); i++) { - auto status = status_list[i]; + auto engine_status = status_list[i]; std::string index = std::to_string(i + 1); - table.add_row({index, status.product_name, status.format, status.version, - status.status}); + auto variant = engine_status.variant.value_or(""); + auto version = engine_status.version.value_or(""); + table.add_row({index, engine_status.product_name, engine_status.format, + version, variant, engine_status.status}); } std::cout << table << std::endl; diff --git a/engine/commands/engine_uninstall_cmd.cc b/engine/commands/engine_uninstall_cmd.cc index 2500a0ab7..1f2095edb 100644 --- a/engine/commands/engine_uninstall_cmd.cc +++ b/engine/commands/engine_uninstall_cmd.cc @@ -10,7 +10,7 @@ void EngineUninstallCmd::Exec(const std::string& engine) { if (result.has_error()) { CLI_LOG(result.error()); } else { - CLI_LOG("Engine uninstalled successfully"); + CLI_LOG("Engine " + engine + " uninstalled successfully!"); } } }; // namespace commands diff --git a/engine/commands/model_del_cmd.cc b/engine/commands/model_del_cmd.cc index 7f6b6d32a..f955a2591 100644 --- a/engine/commands/model_del_cmd.cc +++ b/engine/commands/model_del_cmd.cc @@ -1,48 +1,13 @@ #include "model_del_cmd.h" -#include "cmd_info.h" -#include "config/yaml_config.h" -#include "utils/file_manager_utils.h" -#include "utils/modellist_utils.h" +#include "utils/logging_utils.h" namespace commands { -bool ModelDelCmd::Exec(const std::string& model_handle) { - modellist_utils::ModelListUtils modellist_handler; - config::YamlHandler yaml_handler; - - try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); - yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml); - auto mc = yaml_handler.GetModelConfig(); - // Remove yaml file - std::filesystem::remove(model_entry.path_to_model_yaml); - // Remove model files if they are not imported locally - if (model_entry.branch_name != "imported") { - if (mc.files.size() > 0) { - if (mc.engine == "cortex.llamacpp") { - for (auto& file : mc.files) { - std::filesystem::path gguf_p(file); - std::filesystem::remove(gguf_p); - } - } else { - std::filesystem::path f(mc.files[0]); - std::filesystem::remove_all(f); - } - } else { - CTL_WRN("model config files are empty!"); - } - } - - // update model.list - if (modellist_handler.DeleteModelEntry(model_handle)) { - CLI_LOG("The model " << model_handle << " was deleted"); - return true; - } else { - CTL_ERR("Could not delete model: " << model_handle); - return false; - } - } catch (const std::exception& e) { - CLI_LOG("Fail to delete model with ID '" + model_handle + "': " + e.what()); - false; +void ModelDelCmd::Exec(const std::string& model_handle) { + auto result = model_service_.DeleteModel(model_handle); + if (result.has_error()) { + CLI_LOG(result.error()); + } else { + CLI_LOG("Model " + model_handle + " deleted successfully"); } } -} // namespace commands \ No newline at end of file +} // namespace commands diff --git a/engine/commands/model_del_cmd.h b/engine/commands/model_del_cmd.h index 437564208..878afa384 100644 --- a/engine/commands/model_del_cmd.h +++ b/engine/commands/model_del_cmd.h @@ -1,11 +1,17 @@ #pragma once #include +#include "services/model_service.h" namespace commands { class ModelDelCmd { public: - bool Exec(const std::string& model_handle); + explicit ModelDelCmd() : model_service_{ModelService()} {}; + + void Exec(const std::string& model_handle); + + private: + ModelService model_service_; }; -} \ No newline at end of file +} // namespace commands diff --git a/engine/commands/model_list_cmd.cc b/engine/commands/model_list_cmd.cc index 6e3990eb6..d9a84ec3b 100644 --- a/engine/commands/model_list_cmd.cc +++ b/engine/commands/model_list_cmd.cc @@ -1,5 +1,4 @@ #include "model_list_cmd.h" -#include #include #include #include diff --git a/engine/commands/model_pull_cmd.cc b/engine/commands/model_pull_cmd.cc index e46d78981..88a474e86 100644 --- a/engine/commands/model_pull_cmd.cc +++ b/engine/commands/model_pull_cmd.cc @@ -1,8 +1,13 @@ #include "model_pull_cmd.h" -#include "utils/cortexso_parser.h" +#include "utils/logging_utils.h" namespace commands { void ModelPullCmd::Exec(const std::string& input) { - model_service_.DownloadModel(input); + auto result = model_service_.DownloadModel(input); + if (result.has_error()) { + CLI_LOG(result.error()); + } else { + CLI_LOG("Model downloaded successfully!"); + } } }; // namespace commands diff --git a/engine/commands/run_cmd.cc b/engine/commands/run_cmd.cc index ea551987f..75f2b440f 100644 --- a/engine/commands/run_cmd.cc +++ b/engine/commands/run_cmd.cc @@ -19,13 +19,12 @@ void RunCmd::Exec() { // Download model if it does not exist { if (!modellist_handler.HasModel(model_handle_)) { - model_id = model_service_.DownloadModel(model_handle_); - if (!model_id.has_value()) { - CTL_ERR("Error: Could not get model_id from handle: " << model_handle_); + auto result = model_service_.DownloadModel(model_handle_); + if (result.has_error()) { + CTL_ERR("Error: " << result.error()); return; - } else { - CTL_INF("model_id: " << model_id.value()); } + CTL_INF("model_id: " << model_id.value()); } } diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index da59d9d5d..8a28f6d9e 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -233,14 +233,13 @@ void CommandLineParser::SetupModelCommands() { " models delete [model_id]"); model_del_cmd->group(kSubcommands); model_del_cmd->add_option("model_id", cml_data_.model_id, ""); - model_del_cmd->callback([this, model_del_cmd]() { + model_del_cmd->callback([&]() { if (cml_data_.model_id.empty()) { CLI_LOG("[model_id] is required\n"); CLI_LOG(model_del_cmd->help()); return; }; - commands::ModelDelCmd mdc; - mdc.Exec(cml_data_.model_id); + commands::ModelDelCmd().Exec(cml_data_.model_id); }); std::string model_alias; @@ -518,4 +517,4 @@ void CommandLineParser::ModelUpdate(CLI::App* parent) { commands::ModelUpdCmd command(cml_data_.model_id); command.Exec(cml_data_.model_update_options); }); -} \ No newline at end of file +} diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index 1c1466e5e..59ccd35ec 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -1,18 +1,15 @@ #include "engines.h" -#include -#include -#include #include #include "services/engine_service.h" #include "utils/archive_utils.h" #include "utils/cortex_utils.h" -#include "utils/system_info_utils.h" +#include "utils/logging_utils.h" void Engines::InstallEngine( const HttpRequestPtr& req, std::function&& callback, - const std::string& engine) const { - LOG_DEBUG << "InitEngine, Engine: " << engine; + const std::string& engine) { + if (engine.empty()) { Json::Value res; res["message"] = "Engine name is required"; @@ -23,98 +20,27 @@ void Engines::InstallEngine( return; } - auto system_info = system_info_utils::GetSystemInfo(); auto version{"latest"}; - constexpr auto gitHubHost = "https://api.github.com"; - - std::ostringstream engineReleasePath; - engineReleasePath << "/repos/janhq/" << engine << "/releases/" << version; - - httplib::Client cli(gitHubHost); - using namespace nlohmann; - if (auto res = cli.Get(engineReleasePath.str())) { - if (res->status == httplib::StatusCode::OK_200) { - try { - auto jsonResponse = json::parse(res->body); - auto assets = jsonResponse["assets"]; - - auto os_arch{system_info->os + "-" + system_info->arch}; - for (auto& asset : assets) { - auto assetName = asset["name"].get(); - if (assetName.find(os_arch) != std::string::npos) { - auto download_url = - asset["browser_download_url"].get(); - auto name = asset["name"].get(); - LOG_INFO << "Download url: " << download_url; - - std::filesystem::path engine_folder_path = - file_manager_utils::GetContainerFolderPath( - file_manager_utils::DownloadTypeToString( - DownloadType::Engine)) / - engine; - - if (!std::filesystem::exists(engine_folder_path)) { - CTL_INF("Creating " << engine_folder_path.string()); - std::filesystem::create_directories(engine_folder_path); - } - auto local_path = engine_folder_path / assetName; - auto downloadTask{DownloadTask{.id = engine, - .type = DownloadType::Engine, - .items = {DownloadItem{ - .id = engine, - .downloadUrl = download_url, - .localPath = local_path, - }}}}; - - DownloadService().AddAsyncDownloadTask( - downloadTask, [](const DownloadTask& finishedTask) { - // try to unzip the downloaded file - archive_utils::ExtractArchive( - finishedTask.items[0].localPath.string(), - finishedTask.items[0] - .localPath.parent_path() - .parent_path() - .string()); - - // remove the downloaded file - try { - std::filesystem::remove(finishedTask.items[0].localPath); - } catch (const std::exception& e) { - LOG_WARN << "Could not delete file: " << e.what(); - } - LOG_INFO << "Finished!"; - }); - - Json::Value res; - res["message"] = "Engine download started"; - res["result"] = "OK"; - auto resp = cortex_utils::CreateCortexHttpJsonResponse(res); - resp->setStatusCode(k200OK); - callback(resp); - return; - } - } - Json::Value res; - res["message"] = "Engine not found"; - res["result"] = "Error"; - auto resp = cortex_utils::CreateCortexHttpJsonResponse(res); - resp->setStatusCode(k404NotFound); - callback(resp); - } catch (const json::parse_error& e) { - std::cerr << "JSON parse error: " << e.what() << std::endl; - } - } + auto result = engine_service_.InstallEngine(engine, version); + if (result.has_error()) { + Json::Value res; + res["message"] = result.error(); + auto resp = cortex_utils::CreateCortexHttpJsonResponse(res); + resp->setStatusCode(k400BadRequest); + callback(resp); } else { - auto err = res.error(); - LOG_ERROR << "HTTP error: " << httplib::to_string(err); + Json::Value res; + res["message"] = "Engine " + engine + " installed successfully!"; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(res); + resp->setStatusCode(k200OK); + callback(resp); } } void Engines::ListEngine( const HttpRequestPtr& req, std::function&& callback) const { - auto engine_service = EngineService(); - auto status_list = engine_service.GetEngineInfoList(); + auto status_list = engine_service_.GetEngineInfoList(); Json::Value ret; ret["object"] = "list"; @@ -123,7 +49,8 @@ void Engines::ListEngine( Json::Value ret; ret["name"] = status.name; ret["description"] = status.description; - ret["version"] = status.version; + ret["version"] = status.version.value_or(""); + ret["variant"] = status.variant.value_or(""); ret["productName"] = status.product_name; ret["status"] = status.status; @@ -140,61 +67,47 @@ void Engines::ListEngine( void Engines::GetEngine(const HttpRequestPtr& req, std::function&& callback, const std::string& engine) const { - auto engine_service = EngineService(); - try { - auto status = engine_service.GetEngineInfo(engine); - Json::Value ret; + auto status = engine_service_.GetEngineInfo(engine); + Json::Value ret; + if (status.has_value()) { ret["name"] = status->name; ret["description"] = status->description; - ret["version"] = status->version; + ret["version"] = status->version.value_or(""); + ret["variant"] = status->variant.value_or(""); ret["productName"] = status->product_name; ret["status"] = status->status; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); resp->setStatusCode(k200OK); callback(resp); - } catch (const std::runtime_error e) { + } else { Json::Value ret; - ret["message"] = e.what(); + ret["message"] = "Engine not found"; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); resp->setStatusCode(k400BadRequest); callback(resp); - } catch (const std::exception& e) { - Json::Value ret; - ret["message"] = e.what(); - auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k500InternalServerError); - callback(resp); } } void Engines::UninstallEngine( const HttpRequestPtr& req, std::function&& callback, - const std::string& engine) const { - LOG_INFO << "[Http] Uninstall engine " << engine; - auto engine_service = EngineService(); + const std::string& engine) { + auto result = engine_service_.UninstallEngine(engine); Json::Value ret; - try { - // TODO: Unload the model which is currently running on engine_ - // TODO: Unload engine if is loaded - engine_service.UninstallEngine(engine); - ret["message"] = "Engine " + engine + " uninstalled successfully!"; - auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k200OK); - callback(resp); - } catch (const std::runtime_error& e) { - CLI_LOG("Runtime exception"); - ret["message"] = "Engine " + engine + " is not installed!"; + if (result.has_error()) { + CTL_INF(result.error()); + ret["message"] = result.error(); auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); resp->setStatusCode(k400BadRequest); callback(resp); - } catch (const std::exception& e) { - ret["message"] = "Engine " + engine + " failed to uninstall: " + e.what(); + } else { + CTL_INF("Engine uninstalled successfully"); + ret["message"] = "Engine " + engine + " uninstalled successfully!"; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k400BadRequest); + resp->setStatusCode(k200OK); callback(resp); } } diff --git a/engine/controllers/engines.h b/engine/controllers/engines.h index 91127e5e0..f1d76ff82 100644 --- a/engine/controllers/engines.h +++ b/engine/controllers/engines.h @@ -3,14 +3,16 @@ #include #include #include -#include "utils/cortexso_parser.h" +#include "services/engine_service.h" using namespace drogon; class Engines : public drogon::HttpController { public: + Engines() : engine_service_{EngineService()} {}; + METHOD_LIST_BEGIN - METHOD_ADD(Engines::InstallEngine, "/{1}/init", Post); + METHOD_ADD(Engines::InstallEngine, "/{1}/install", Post); METHOD_ADD(Engines::UninstallEngine, "/{1}", Delete); METHOD_ADD(Engines::ListEngine, "", Get); METHOD_ADD(Engines::GetEngine, "/{1}", Get); @@ -18,7 +20,7 @@ class Engines : public drogon::HttpController { void InstallEngine(const HttpRequestPtr& req, std::function&& callback, - const std::string& engine) const; + const std::string& engine); void ListEngine(const HttpRequestPtr& req, std::function&& callback) const; @@ -29,5 +31,8 @@ class Engines : public drogon::HttpController { void UninstallEngine(const HttpRequestPtr& req, std::function&& callback, - const std::string& engine) const; + const std::string& engine); + + private: + EngineService engine_service_; }; diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 4660b50e5..72706b2b1 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -1,21 +1,23 @@ #include "models.h" -#include "commands/model_del_cmd.h" +#include +#include "config/gguf_parser.h" #include "config/yaml_config.h" #include "trantor/utils/Logger.h" #include "utils/cortex_utils.h" #include "utils/file_manager_utils.h" -#include "utils/model_callback_utils.h" +#include "utils/http_util.h" +#include "utils/logging_utils.h" #include "utils/modellist_utils.h" +#include "utils/string_utils.h" -void Models::PullModel( - const HttpRequestPtr& req, - std::function&& callback) const { +void Models::PullModel(const HttpRequestPtr& req, + std::function&& callback) { if (!http_util::HasFieldInReq(req, callback, "modelId")) { return; } - auto modelHandle = (*(req->getJsonObject())).get("modelId", "").asString(); - LOG_DEBUG << "PullModel, Model handle: " << modelHandle; - if (modelHandle.empty()) { + + auto model_handle = (*(req->getJsonObject())).get("modelId", "").asString(); + if (model_handle.empty()) { Json::Value ret; ret["result"] = "Bad Request"; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); @@ -24,24 +26,32 @@ void Models::PullModel( return; } - auto downloadTask = cortexso_parser::getDownloadTask(modelHandle); - if (downloadTask.has_value()) { - DownloadService downloadService; - downloadService.AddAsyncDownloadTask(downloadTask.value(), - model_callback_utils::DownloadModelCb); + auto handle_model_input = + [&, model_handle]() -> cpp::result { + CTL_INF("Handle model input, model handle: " + model_handle); + if (string_utils::StartsWith(model_handle, "https")) { + return model_service_.HandleUrl(model_handle, true); + } else if (model_handle.find(":") == std::string::npos) { + auto model_and_branch = string_utils::SplitBy(model_handle, ":"); + return model_service_.DownloadModelFromCortexso( + model_and_branch[0], model_and_branch[1], true); + } + + return cpp::fail("Invalid model handle or not supported!"); + }; + auto result = handle_model_input(); + if (result.has_error()) { Json::Value ret; - ret["result"] = "OK"; - ret["modelHandle"] = modelHandle; + ret["message"] = result.error(); auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k200OK); + resp->setStatusCode(k400BadRequest); callback(resp); } else { Json::Value ret; - ret["result"] = "Not Found"; - ret["modelHandle"] = modelHandle; + ret["message"] = "Model start downloading!"; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k404NotFound); + resp->setStatusCode(k200OK); callback(resp); } } @@ -136,25 +146,23 @@ void Models::GetModel( void Models::DeleteModel(const HttpRequestPtr& req, std::function&& callback, - const std::string& model_id) const { - LOG_DEBUG << "DeleteModel, Model handle: " << model_id; - commands::ModelDelCmd mdc; - if (mdc.Exec(model_id)) { + const std::string& model_id) { + auto result = model_service_.DeleteModel(model_id); + if (result.has_error()) { Json::Value ret; - ret["result"] = "OK"; - ret["modelHandle"] = model_id; + ret["message"] = result.error(); auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k200OK); + resp->setStatusCode(drogon::k400BadRequest); callback(resp); } else { Json::Value ret; - ret["result"] = "Not Found"; - ret["modelHandle"] = model_id; + ret["message"] = "Deleted successfully!"; auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); - resp->setStatusCode(k404NotFound); + resp->setStatusCode(k200OK); callback(resp); } } + void Models::UpdateModel( const HttpRequestPtr& req, std::function&& callback) const { @@ -319,4 +327,4 @@ void Models::SetModelAlias( resp->setStatusCode(k400BadRequest); callback(resp); } -} \ No newline at end of file +} diff --git a/engine/controllers/models.h b/engine/controllers/models.h index 8d652c86a..9c67ff7dc 100644 --- a/engine/controllers/models.h +++ b/engine/controllers/models.h @@ -2,18 +2,17 @@ #include #include -#include "services/download_service.h" -#include "utils/cortex_utils.h" -#include "utils/cortexso_parser.h" -#include "utils/http_util.h" +#include "services/model_service.h" using namespace drogon; class Models : public drogon::HttpController { public: + explicit Models() : model_service_{ModelService()} {}; + METHOD_LIST_BEGIN METHOD_ADD(Models::PullModel, "/pull", Post); - METHOD_ADD(Models::ListModel, "/list", Get); + METHOD_ADD(Models::ListModel, "", Get); METHOD_ADD(Models::GetModel, "/get", Post); METHOD_ADD(Models::UpdateModel, "/update/", Post); METHOD_ADD(Models::ImportModel, "/import", Post); @@ -22,20 +21,24 @@ class Models : public drogon::HttpController { METHOD_LIST_END void PullModel(const HttpRequestPtr& req, - std::function&& callback) const; + std::function&& callback); void ListModel(const HttpRequestPtr& req, std::function&& callback) const; void GetModel(const HttpRequestPtr& req, std::function&& callback) const; - void UpdateModel(const HttpRequestPtr& req, - std::function&& callback) const; + void UpdateModel( + const HttpRequestPtr& req, + std::function&& callback) const; void ImportModel( const HttpRequestPtr& req, std::function&& callback) const; void DeleteModel(const HttpRequestPtr& req, std::function&& callback, - const std::string& model_id) const; + const std::string& model_id); void SetModelAlias( const HttpRequestPtr& req, std::function&& callback) const; -}; \ No newline at end of file + + private: + ModelService model_service_; +}; diff --git a/engine/controllers/server.h b/engine/controllers/server.h index ed9e35933..58615517d 100644 --- a/engine/controllers/server.h +++ b/engine/controllers/server.h @@ -17,11 +17,8 @@ #include #include "common/base.h" -#include "config/gguf_parser.h" -#include "config/yaml_config.h" #include "cortex-common/EngineI.h" #include "cortex-common/cortexpythoni.h" -#include "trantor/utils/SerialTaskQueue.h" #include "utils/dylib.h" #include "utils/json.hpp" @@ -161,4 +158,4 @@ class server : public drogon::HttpController, std::unordered_map engines_; std::string cur_engine_type_; }; -}; // namespace inferences \ No newline at end of file +}; // namespace inferences diff --git a/engine/cortex-common/EngineI.h b/engine/cortex-common/EngineI.h index 08789fc5d..e35c3c3f6 100644 --- a/engine/cortex-common/EngineI.h +++ b/engine/cortex-common/EngineI.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include "json/value.h" diff --git a/engine/e2e-test/test_cli_model_delete.py b/engine/e2e-test/test_cli_model_delete.py index 8bbe95b35..23534dd4c 100644 --- a/engine/e2e-test/test_cli_model_delete.py +++ b/engine/e2e-test/test_cli_model_delete.py @@ -9,7 +9,8 @@ def setup_and_teardown(self): # Setup # Pull model - stdout, stderr, return_code = popen(["pull", "tinyllama"], "1\n") + # TODO: using pull with branch for easy testing tinyllama:gguf for example + popen(["pull", "tinyllama"], "1\n") yield @@ -19,7 +20,7 @@ def setup_and_teardown(self): def test_models_delete_should_be_successful(self): exit_code, output, error = run( - "Delete model", ["models", "delete", "tinyllama"] + "Delete model", ["models", "delete", "tinyllama:gguf"] ) - assert "The model tinyllama was deleted" in output + assert "Model tinyllama:gguf deleted successfully" in output assert exit_code == 0, f"Model does not exist: {error}" diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index 904a2323a..d985636f2 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -27,12 +27,13 @@ size_t WriteCallback(void* ptr, size_t size, size_t nmemb, FILE* stream) { } } // namespace -cpp::result DownloadService::AddDownloadTask( - DownloadTask& task, std::optional callback) { +cpp::result DownloadService::VerifyDownloadTask( + DownloadTask& task) const noexcept { CLI_LOG("Validating download items, please wait.."); - // preprocess to check if all the item are valid + auto total_download_size{0}; std::optional err_msg = std::nullopt; + for (auto& item : task.items) { auto file_size = GetFileSize(item.downloadUrl); if (file_size.has_error()) { @@ -49,6 +50,17 @@ cpp::result DownloadService::AddDownloadTask( return cpp::fail(err_msg.value()); } + return {}; +} + +cpp::result DownloadService::AddDownloadTask( + DownloadTask& task, + std::optional callback) noexcept { + auto validating_result = VerifyDownloadTask(task); + if (validating_result.has_error()) { + return cpp::fail(validating_result.error()); + } + // all items are valid, start downloading // if any item from the task failed to download, the whole task will be // considered failed @@ -97,15 +109,40 @@ cpp::result DownloadService::GetFileSize( return content_length; } -void DownloadService::AddAsyncDownloadTask( - const DownloadTask& task, - std::optional callback) { - // just start one thread and handle all the download items - for (const auto& item : task.items) { - std::thread([this, task, &callback, item]() { - this->Download(task.id, item, false); - }).detach(); +cpp::result DownloadService::AddAsyncDownloadTask( + DownloadTask& task, + std::optional callback) noexcept { + auto verifying_result = VerifyDownloadTask(task); + if (verifying_result.has_error()) { + return cpp::fail(verifying_result.error()); } + + auto execute_download_async = [&, task, callback]() { + std::optional dl_err_msg = std::nullopt; + for (const auto& item : task.items) { + CTL_INF("Start downloading: " + item.localPath.filename().string()); + auto result = Download(task.id, item, false); + if (result.has_error()) { + dl_err_msg = result.error(); + break; + } + } + + if (dl_err_msg.has_value()) { + CTL_ERR(dl_err_msg.value()); + return; + } + + if (callback.has_value()) { + CTL_INF("Download success, executing post download lambda!"); + callback.value()(task); + } + }; + + std::thread t(execute_download_async); + t.detach(); + + return {}; } cpp::result DownloadService::Download( diff --git a/engine/services/download_service.h b/engine/services/download_service.h index b807a1179..7d44185f5 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -58,12 +58,12 @@ class DownloadService { std::function; cpp::result AddDownloadTask( - DownloadTask& task, - std::optional callback = std::nullopt); + DownloadTask& task, std::optional callback = + std::nullopt) noexcept; - void AddAsyncDownloadTask( - const DownloadTask& task, - std::optional callback = std::nullopt); + cpp::result AddAsyncDownloadTask( + DownloadTask& task, std::optional callback = + std::nullopt) noexcept; /** * Getting file size for a provided url. Can be used to validating the download url. @@ -74,6 +74,9 @@ class DownloadService { const std::string& url) const noexcept; private: + cpp::result VerifyDownloadTask( + DownloadTask& task) const noexcept; + cpp::result Download(const std::string& download_id, const DownloadItem& download_item, bool allow_resume) noexcept; diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index 048495494..d809e831e 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -1,5 +1,6 @@ #include "engine_service.h" #include +#include #include "algorithm" #include "utils/archive_utils.h" #include "utils/engine_matcher_utils.h" @@ -38,11 +39,12 @@ EngineService::EngineService() .cuda_driver_version = system_info_utils::GetCudaVersion()} {} EngineService::~EngineService() {} -std::optional EngineService::GetEngineInfo( +cpp::result EngineService::GetEngineInfo( const std::string& engine) const { + if (std::find(kSupportEngines.begin(), kSupportEngines.end(), engine) == kSupportEngines.end()) { - return std::nullopt; + return cpp::fail("Engine " + engine + " is not supported!"); } auto engine_status_list = GetEngineInfoList(); @@ -76,25 +78,41 @@ std::vector EngineService::GetEngineInfoList() const { .description = "This extension enables chat completion API calls using " "the Onnx engine", .format = "ONNX", - .version = "0.0.1", .product_name = "ONNXRuntime", .status = onnx_status}, {.name = "cortex.llamacpp", .description = "This extension enables chat completion API calls using " "the LlamaCPP engine", .format = "GGUF", - .version = "0.0.1", .product_name = "llama.cpp", .status = llamacpp_status}, {.name = "cortex.tensorrt-llm", .description = "This extension enables chat completion API calls using " "the TensorrtLLM engine", .format = "TensorRT Engines", - .version = "0.0.1", .product_name = "TensorRT-LLM", .status = tensorrt_status}, }; + for (auto& engine : engines) { + if (engine.status == kReady) { + // try to read the version.txt + auto engine_info_path = file_manager_utils::GetEnginesContainerPath() / + engine.name / "version.txt"; + if (!std::filesystem::exists(engine_info_path)) { + continue; + } + try { + auto node = YAML::LoadFile(engine_info_path.string()); + engine.version = node["version"].as(); + engine.variant = node["name"].as(); + } catch (const YAML::Exception& e) { + CTL_ERR("Error reading version.txt: " << e.what()); + continue; + } + } + } + return engines; } diff --git a/engine/services/engine_service.h b/engine/services/engine_service.h index fd48ad8e3..8b6417019 100644 --- a/engine/services/engine_service.h +++ b/engine/services/engine_service.h @@ -12,9 +12,10 @@ struct EngineInfo { std::string name; std::string description; std::string format; - std::string version; + std::optional version; std::string product_name; std::string status; + std::optional variant; }; namespace system_info_utils { @@ -32,7 +33,8 @@ class EngineService { EngineService(); ~EngineService(); - std::optional GetEngineInfo(const std::string& engine) const; + cpp::result GetEngineInfo( + const std::string& engine) const; std::vector GetEngineInfoList() const; diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index 36a5e7aa5..bb5b02a96 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -5,64 +5,155 @@ #include "config/gguf_parser.h" #include "config/yaml_config.h" #include "utils/cli_selection_utils.h" -#include "utils/cortexso_parser.h" #include "utils/file_manager_utils.h" #include "utils/huggingface_utils.h" #include "utils/logging_utils.h" #include "utils/modellist_utils.h" +#include "utils/result.hpp" #include "utils/string_utils.h" -std::optional ModelService::DownloadModel( - const std::string& input) { +namespace { +void ParseGguf(const DownloadItem& ggufDownloadItem, + std::optional author) { + + config::GGUFHandler gguf_handler; + config::YamlHandler yaml_handler; + gguf_handler.Parse(ggufDownloadItem.localPath.string()); + config::ModelConfig model_config = gguf_handler.GetModelConfig(); + model_config.id = + ggufDownloadItem.localPath.parent_path().filename().string(); + model_config.files = {ggufDownloadItem.localPath.string()}; + yaml_handler.UpdateModelConfig(model_config); + + auto yaml_path{ggufDownloadItem.localPath}; + auto yaml_name = yaml_path.replace_extension(".yml"); + + if (!std::filesystem::exists(yaml_path)) { + yaml_handler.WriteYamlFile(yaml_path.string()); + } + + auto url_obj = url_parser::FromUrlString(ggufDownloadItem.downloadUrl); + auto branch = url_obj.pathParams[3]; + CTL_INF("Adding model to modellist with branch: " << branch); + + auto author_id = author.has_value() ? author.value() : "cortexso"; + modellist_utils::ModelListUtils modellist_utils_obj; + modellist_utils::ModelEntry model_entry{ + .model_id = ggufDownloadItem.id, + .author_repo_id = author_id, + .branch_name = branch, + .path_to_model_yaml = yaml_name.string(), + .model_alias = ggufDownloadItem.id, + .status = modellist_utils::ModelStatus::READY}; + modellist_utils_obj.AddModelEntry(model_entry, true); +} + +cpp::result GetDownloadTask( + const std::string& modelId, const std::string& branch = "main") { + using namespace nlohmann; + url_parser::Url url = { + .protocol = "https", + .host = ModelService::kHuggingFaceHost, + .pathParams = {"api", "models", "cortexso", modelId, "tree", branch}}; + + httplib::Client cli(url.GetProtocolAndHost()); + auto res = cli.Get(url.GetPathAndQuery()); + if (res->status != httplib::StatusCode::OK_200) { + auto err = res.error(); + return cpp::fail("HTTP error: " + httplib::to_string(err)); + } + auto jsonResponse = json::parse(res->body); + + std::vector download_items{}; + auto model_container_path = file_manager_utils::GetModelsContainerPath() / + "cortex.so" / modelId / branch; + file_manager_utils::CreateDirectoryRecursively(model_container_path.string()); + + for (const auto& [key, value] : jsonResponse.items()) { + auto path = value["path"].get(); + if (path == ".gitattributes" || path == ".gitignore" || + path == "README.md") { + continue; + } + url_parser::Url download_url = { + .protocol = "https", + .host = ModelService::kHuggingFaceHost, + .pathParams = {"cortexso", modelId, "resolve", branch, path}}; + + auto local_path = model_container_path / path; + download_items.push_back( + DownloadItem{.id = path, + .downloadUrl = download_url.ToFullPath(), + .localPath = local_path}); + } + + DownloadTask download_tasks{ + .id = branch == "main" ? modelId : modelId + "-" + branch, + .type = DownloadType::Model, + .items = download_items}; + + return download_tasks; +} +} // namespace + +cpp::result ModelService::DownloadModel( + const std::string& input, bool async) { if (input.empty()) { - throw std::runtime_error( + return cpp::fail( "Input must be Cortex Model Hub handle or HuggingFace url!"); } if (string_utils::StartsWith(input, "https://")) { - return DownloadModelByDirectUrl(input); + // TODO: better name, for example handle url + return HandleUrl(input, async); + } + + if (input.find(":") != std::string::npos) { + auto parsed = string_utils::SplitBy(input, ":"); + if (parsed.size() != 2) { + return cpp::fail("Invalid model handle: " + input); + } + return DownloadModelFromCortexso(parsed[0], parsed[1], false); } if (input.find("/") != std::string::npos) { auto parsed = string_utils::SplitBy(input, "/"); if (parsed.size() != 2) { - throw std::runtime_error("Invalid model handle: " + input); + return cpp::fail("Invalid model handle: " + input); } auto author = parsed[0]; auto model_name = parsed[1]; if (author == "cortexso") { - return DownloadModelByModelName(model_name); + return HandleCortexsoModel(model_name); } - CLI_LOG("Model " << model_name << " downloaded successfully!") - return DownloadHuggingFaceGgufModel(author, model_name, std::nullopt); + return DownloadHuggingFaceGgufModel(author, model_name, std::nullopt, + async); } - return DownloadModelByModelName(input); + return HandleCortexsoModel(input); } -std::optional ModelService::DownloadModelByModelName( +cpp::result ModelService::HandleCortexsoModel( const std::string& modelName) { - try { - auto branches = - huggingface_utils::GetModelRepositoryBranches("cortexso", modelName); - std::vector options{}; - for (const auto& branch : branches) { - if (branch.name != "main") { - options.emplace_back(branch.name); - } - } - if (options.empty()) { - CLI_LOG("No variant found"); - return std::nullopt; + auto branches = + huggingface_utils::GetModelRepositoryBranches("cortexso", modelName); + if (branches.has_error()) { + return cpp::fail(branches.error()); + } + + std::vector options{}; + for (const auto& branch : branches.value()) { + if (branch.name != "main") { + options.emplace_back(branch.name); } - auto selection = cli_selection_utils::PrintSelection(options); - return DownloadModelFromCortexso(modelName, selection.value()); - } catch (const std::runtime_error& e) { - CLI_LOG("Error downloading model, " << e.what()); - return std::nullopt; } + if (options.empty()) { + return cpp::fail("No variant available"); + } + auto selection = cli_selection_utils::PrintSelection(options); + return DownloadModelFromCortexso(modelName, selection.value()); } std::optional ModelService::GetDownloadedModel( @@ -91,8 +182,8 @@ std::optional ModelService::GetDownloadedModel( return std::nullopt; } -std::optional ModelService::DownloadModelByDirectUrl( - const std::string& url) { +cpp::result ModelService::HandleUrl( + const std::string& url, bool async) { auto url_obj = url_parser::FromUrlString(url); if (url_obj.host == kHuggingFaceHost) { @@ -105,6 +196,7 @@ std::optional ModelService::DownloadModelByDirectUrl( auto file_name{url_obj.pathParams.back()}; if (author == "cortexso") { + // TODO: try to get the branch return DownloadModelFromCortexso(model_id); } @@ -138,78 +230,93 @@ std::optional ModelService::DownloadModelByDirectUrl( ParseGguf(gguf_download_item, author); }; - auto result = download_service_.AddDownloadTask(downloadTask, on_finished); - if (result.has_error()) { - CTL_ERR(result.error()); - return std::nullopt; + if (async) { + auto result = + download_service_.AddAsyncDownloadTask(downloadTask, on_finished); + if (result.has_error()) { + CTL_ERR(result.error()); + return cpp::fail(result.error()); + } + return unique_model_id; + } else { + auto result = download_service_.AddDownloadTask(downloadTask, on_finished); + if (result.has_error()) { + CTL_ERR(result.error()); + return cpp::fail(result.error()); + } + return unique_model_id; } - return unique_model_id; } -std::optional ModelService::DownloadModelFromCortexso( - const std::string& name, const std::string& branch) { - - auto downloadTask = cortexso_parser::getDownloadTask(name, branch); - if (downloadTask.has_value()) { - std::string model_id{name + ":" + branch}; - auto result = DownloadService().AddDownloadTask( - downloadTask.value(), [&](const DownloadTask& finishedTask) { - const DownloadItem* model_yml_item = nullptr; - auto need_parse_gguf = true; - - for (const auto& item : finishedTask.items) { - if (item.localPath.filename().string() == "model.yml") { - model_yml_item = &item; - } - } +cpp::result ModelService::DownloadModelFromCortexso( + const std::string& name, const std::string& branch, bool async) { - if (model_yml_item != nullptr) { - auto url_obj = - url_parser::FromUrlString(model_yml_item->downloadUrl); - CTL_INF("Adding model to modellist with branch: " << branch); - config::YamlHandler yaml_handler; - yaml_handler.ModelConfigFromFile( - model_yml_item->localPath.string()); - auto mc = yaml_handler.GetModelConfig(); - - modellist_utils::ModelListUtils modellist_utils_obj; - modellist_utils::ModelEntry model_entry{ - .model_id = model_id, - .author_repo_id = "cortexso", - .branch_name = branch, - .path_to_model_yaml = model_yml_item->localPath.string(), - .model_alias = model_id, - .status = modellist_utils::ModelStatus::READY}; - modellist_utils_obj.AddModelEntry(model_entry); - } - }); - if (result.has_error()) { - CTL_ERR(result.error()); - return std::nullopt; + auto download_task = GetDownloadTask(name, branch); + if (download_task.has_error()) { + return cpp::fail(download_task.error()); + } + + std::string model_id{name + ":" + branch}; + auto on_finished = [&](const DownloadTask& finishedTask) { + const DownloadItem* model_yml_item = nullptr; + auto need_parse_gguf = true; + + for (const auto& item : finishedTask.items) { + if (item.localPath.filename().string() == "model.yml") { + model_yml_item = &item; + } } - CLI_LOG("Model " << model_id << " downloaded successfully!") - return model_id; - } else { - CTL_ERR("Model not found"); - return std::nullopt; + if (model_yml_item == nullptr) { + CTL_WRN("model.yml not found in the downloaded files for " + model_id); + return; + } + auto url_obj = url_parser::FromUrlString(model_yml_item->downloadUrl); + CTL_INF("Adding model to modellist with branch: " << branch); + config::YamlHandler yaml_handler; + yaml_handler.ModelConfigFromFile(model_yml_item->localPath.string()); + auto mc = yaml_handler.GetModelConfig(); + + modellist_utils::ModelListUtils modellist_utils_obj; + modellist_utils::ModelEntry model_entry{ + .model_id = model_id, + .author_repo_id = "cortexso", + .branch_name = branch, + .path_to_model_yaml = model_yml_item->localPath.string(), + .model_alias = model_id, + .status = modellist_utils::ModelStatus::READY}; + modellist_utils_obj.AddModelEntry(model_entry); + }; + + auto result = async ? DownloadService().AddAsyncDownloadTask( + download_task.value(), on_finished) + : DownloadService().AddDownloadTask(download_task.value(), + on_finished); + + if (result.has_error()) { + return cpp::fail(result.error()); } + + CLI_LOG("Model " << model_id << " downloaded successfully!") + return model_id; } -std::optional ModelService::DownloadHuggingFaceGgufModel( - const std::string& author, const std::string& modelName, - std::optional fileName) { +cpp::result +ModelService::DownloadHuggingFaceGgufModel(const std::string& author, + const std::string& modelName, + std::optional fileName, + bool async) { auto repo_info = huggingface_utils::GetHuggingFaceModelRepoInfo(author, modelName); + if (!repo_info.has_value()) { - // throw is better? - CTL_ERR("Model not found"); - return std::nullopt; + return cpp::fail("Model not found"); } if (!repo_info->gguf.has_value()) { - throw std::runtime_error( - "Not a GGUF model. Currently, only GGUF single file is supported."); + return cpp::fail( + "Not a GGUF model. Currently, only GGUF single file is " + "supported."); } std::vector options{}; @@ -223,40 +330,46 @@ std::optional ModelService::DownloadHuggingFaceGgufModel( auto download_url = huggingface_utils::GetDownloadableUrl(author, modelName, selection.value()); - return DownloadModelByDirectUrl(download_url); + return HandleUrl(download_url); } -void ModelService::ParseGguf(const DownloadItem& ggufDownloadItem, - std::optional author) const { +cpp::result ModelService::DeleteModel( + const std::string& model_handle) { - config::GGUFHandler gguf_handler; + modellist_utils::ModelListUtils modellist_handler; config::YamlHandler yaml_handler; - gguf_handler.Parse(ggufDownloadItem.localPath.string()); - config::ModelConfig model_config = gguf_handler.GetModelConfig(); - model_config.id = - ggufDownloadItem.localPath.parent_path().filename().string(); - model_config.files = {ggufDownloadItem.localPath.string()}; - yaml_handler.UpdateModelConfig(model_config); - auto yaml_path{ggufDownloadItem.localPath}; - auto yaml_name = yaml_path.replace_extension(".yml"); + try { + auto model_entry = modellist_handler.GetModelInfo(model_handle); + yaml_handler.ModelConfigFromFile(model_entry.path_to_model_yaml); + auto mc = yaml_handler.GetModelConfig(); + // Remove yaml file + std::filesystem::remove(model_entry.path_to_model_yaml); + // Remove model files if they are not imported locally + if (model_entry.branch_name != "imported") { + if (mc.files.size() > 0) { + if (mc.engine == "cortex.llamacpp") { + for (auto& file : mc.files) { + std::filesystem::path gguf_p(file); + std::filesystem::remove(gguf_p); + } + } else { + std::filesystem::path f(mc.files[0]); + std::filesystem::remove_all(f); + } + } else { + CTL_WRN("model config files are empty!"); + } + } - if (!std::filesystem::exists(yaml_path)) { - yaml_handler.WriteYamlFile(yaml_path.string()); + // update model.list + if (modellist_handler.DeleteModelEntry(model_handle)) { + return {}; + } else { + return cpp::fail("Could not delete model: " + model_handle); + } + } catch (const std::exception& e) { + return cpp::fail("Fail to delete model with ID '" + model_handle + + "': " + e.what()); } - - auto url_obj = url_parser::FromUrlString(ggufDownloadItem.downloadUrl); - auto branch = url_obj.pathParams[3]; - CTL_INF("Adding model to modellist with branch: " << branch); - - auto author_id = author.has_value() ? author.value() : "cortexso"; - modellist_utils::ModelListUtils modellist_utils_obj; - modellist_utils::ModelEntry model_entry{ - .model_id = ggufDownloadItem.id, - .author_repo_id = author_id, - .branch_name = branch, - .path_to_model_yaml = yaml_name.string(), - .model_alias = ggufDownloadItem.id, - .status = modellist_utils::ModelStatus::READY}; - modellist_utils_obj.AddModelEntry(model_entry, true); } diff --git a/engine/services/model_service.h b/engine/services/model_service.h index 4237f1b17..6ddc00d7c 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -6,36 +6,46 @@ class ModelService { public: + constexpr auto static kHuggingFaceHost = "huggingface.co"; + ModelService() : download_service_{DownloadService()} {}; /** * Return model id if download successfully */ - std::optional DownloadModel(const std::string& input); + cpp::result DownloadModel(const std::string& input, + bool async = false); + + cpp::result DownloadModelFromCortexso( + const std::string& name, const std::string& branch = "main", + bool async = false); std::optional GetDownloadedModel( const std::string& modelId) const; - private: - std::optional DownloadModelByDirectUrl(const std::string& url); + /** + * Delete a model from local. If a model is an import model, we only delete + * in our database/model.list. + */ + cpp::result DeleteModel(const std::string& model_handle); - std::optional DownloadModelFromCortexso( - const std::string& name, const std::string& branch = "main"); + cpp::result HandleUrl(const std::string& url, + bool async = false); + private: /** * Handle downloading model which have following pattern: author/model_name */ - std::optional DownloadHuggingFaceGgufModel( + cpp::result DownloadHuggingFaceGgufModel( const std::string& author, const std::string& modelName, - std::optional fileName); + std::optional fileName, bool async = false); - std::optional DownloadModelByModelName( + /** + * Handling cortexso models. Will look through cortexso's HF repository and + * listing all the branches, except main. Then print out the selection for user. + */ + cpp::result HandleCortexsoModel( const std::string& modelName); DownloadService download_service_; - - void ParseGguf(const DownloadItem& ggufDownloadItem, - std::optional author = nullptr) const; - - constexpr auto static kHuggingFaceHost = "huggingface.co"; }; diff --git a/engine/test/components/test_huggingface_utils.cc b/engine/test/components/test_huggingface_utils.cc index b1d949d22..c790020c3 100644 --- a/engine/test/components/test_huggingface_utils.cc +++ b/engine/test/components/test_huggingface_utils.cc @@ -7,13 +7,13 @@ TEST_F(HuggingFaceUtilTestSuite, TestGetModelRepositoryBranches) { auto branches = huggingface_utils::GetModelRepositoryBranches("cortexso", "tinyllama"); - EXPECT_EQ(branches.size(), 3); - EXPECT_EQ(branches[0].name, "gguf"); - EXPECT_EQ(branches[0].ref, "refs/heads/gguf"); - EXPECT_EQ(branches[1].name, "1b-gguf"); - EXPECT_EQ(branches[1].ref, "refs/heads/1b-gguf"); - EXPECT_EQ(branches[2].name, "main"); - EXPECT_EQ(branches[2].ref, "refs/heads/main"); + EXPECT_EQ(branches.value().size(), 3); + EXPECT_EQ(branches.value()[0].name, "gguf"); + EXPECT_EQ(branches.value()[0].ref, "refs/heads/gguf"); + EXPECT_EQ(branches.value()[1].name, "1b-gguf"); + EXPECT_EQ(branches.value()[1].ref, "refs/heads/1b-gguf"); + EXPECT_EQ(branches.value()[2].name, "main"); + EXPECT_EQ(branches.value()[2].ref, "refs/heads/main"); } TEST_F(HuggingFaceUtilTestSuite, TestGetHuggingFaceModelRepoInfoSuccessfully) { diff --git a/engine/utils/cortexso_parser.h b/engine/utils/cortexso_parser.h deleted file mode 100644 index af3372022..000000000 --- a/engine/utils/cortexso_parser.h +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include -#include - -#include -#include -#include "httplib.h" -#include "utils/file_manager_utils.h" -#include "utils/huggingface_utils.h" -#include "utils/logging_utils.h" - -namespace cortexso_parser { -constexpr static auto kHuggingFaceHost = "huggingface.co"; - -inline std::optional getDownloadTask( - const std::string& modelId, const std::string& branch = "main") { - using namespace nlohmann; - url_parser::Url url = { - .protocol = "https", - .host = kHuggingFaceHost, - .pathParams = {"api", "models", "cortexso", modelId, "tree", branch}}; - - httplib::Client cli(url.GetProtocolAndHost()); - if (auto res = cli.Get(url.GetPathAndQuery())) { - if (res->status == httplib::StatusCode::OK_200) { - try { - auto jsonResponse = json::parse(res->body); - - std::vector download_items{}; - auto model_container_path = - file_manager_utils::GetModelsContainerPath() / "cortex.so" / - modelId / branch; - file_manager_utils::CreateDirectoryRecursively( - model_container_path.string()); - - for (const auto& [key, value] : jsonResponse.items()) { - auto path = value["path"].get(); - if (path == ".gitattributes" || path == ".gitignore" || - path == "README.md") { - continue; - } - url_parser::Url download_url = { - .protocol = "https", - .host = kHuggingFaceHost, - .pathParams = {"cortexso", modelId, "resolve", branch, path}}; - - auto local_path = model_container_path / path; - download_items.push_back( - DownloadItem{.id = path, - .downloadUrl = download_url.ToFullPath(), - .localPath = local_path}); - } - - DownloadTask download_tasks{ - .id = branch == "main" ? modelId : modelId + "-" + branch, - .type = DownloadType::Model, - .items = download_items}; - - return download_tasks; - } catch (const json::parse_error& e) { - CTL_ERR("JSON parse error: {}" << e.what()); - } - } - } else { - auto err = res.error(); - LOG_ERROR << "HTTP error: " << httplib::to_string(err); - } - return std::nullopt; -} -} // namespace cortexso_parser diff --git a/engine/utils/huggingface_utils.h b/engine/utils/huggingface_utils.h index 3699c38b8..ae264fae6 100644 --- a/engine/utils/huggingface_utils.h +++ b/engine/utils/huggingface_utils.h @@ -2,10 +2,10 @@ #include #include -#include #include #include #include "utils/json.hpp" +#include "utils/result.hpp" #include "utils/url_parser.h" namespace huggingface_utils { @@ -47,10 +47,11 @@ struct HuggingFaceModelRepoInfo { std::string createdAt; }; -inline std::vector GetModelRepositoryBranches( - const std::string& author, const std::string& modelName) { +inline cpp::result, std::string> +GetModelRepositoryBranches(const std::string& author, + const std::string& modelName) { if (author.empty() || modelName.empty()) { - throw std::runtime_error("Author and model name cannot be empty"); + return cpp::fail("Author and model name cannot be empty"); } auto url_obj = url_parser::Url{ .protocol = "https", @@ -60,8 +61,8 @@ inline std::vector GetModelRepositoryBranches( httplib::Client cli(url_obj.GetProtocolAndHost()); auto res = cli.Get(url_obj.GetPathAndQuery()); if (res->status != httplib::StatusCode::OK_200) { - throw std::runtime_error( - "Failed to get model repository branches: " + author + "/" + modelName); + return cpp::fail("Failed to get model repository branches: " + author + + "/" + modelName); } using json = nlohmann::json; @@ -82,10 +83,11 @@ inline std::vector GetModelRepositoryBranches( } // only support gguf for now -inline std::optional GetHuggingFaceModelRepoInfo( - const std::string& author, const std::string& modelName) { +inline cpp::result +GetHuggingFaceModelRepoInfo(const std::string& author, + const std::string& modelName) { if (author.empty() || modelName.empty()) { - throw std::runtime_error("Author and model name cannot be empty"); + return cpp::fail("Author and model name cannot be empty"); } auto url_obj = url_parser::Url{.protocol = "https", @@ -95,8 +97,8 @@ inline std::optional GetHuggingFaceModelRepoInfo( httplib::Client cli(url_obj.GetProtocolAndHost()); auto res = cli.Get(url_obj.GetPathAndQuery()); if (res->status != httplib::StatusCode::OK_200) { - throw std::runtime_error("Failed to get model repository info: " + author + - "/" + modelName); + return cpp::fail("Failed to get model repository info: " + author + "/" + + modelName); } using json = nlohmann::json; diff --git a/engine/utils/model_callback_utils.h b/engine/utils/model_callback_utils.h deleted file mode 100644 index c6e98dd48..000000000 --- a/engine/utils/model_callback_utils.h +++ /dev/null @@ -1,90 +0,0 @@ -#pragma once - -#include -#include - -#include "config/gguf_parser.h" -#include "config/yaml_config.h" -#include "services/download_service.h" -#include "utils/huggingface_utils.h" -#include "utils/logging_utils.h" -#include "utils/modellist_utils.h" - -namespace model_callback_utils { - -inline void ParseGguf(const DownloadItem& ggufDownloadItem, - std::optional author = nullptr) { - config::GGUFHandler gguf_handler; - config::YamlHandler yaml_handler; - gguf_handler.Parse(ggufDownloadItem.localPath.string()); - config::ModelConfig model_config = gguf_handler.GetModelConfig(); - model_config.id = - ggufDownloadItem.localPath.parent_path().filename().string(); - model_config.files = {ggufDownloadItem.localPath.string()}; - yaml_handler.UpdateModelConfig(model_config); - - auto yaml_path{ggufDownloadItem.localPath}; - auto yaml_name = yaml_path.replace_extension(".yml"); - - if (!std::filesystem::exists(yaml_path)) { - yaml_handler.WriteYamlFile(yaml_path.string()); - } - - auto url_obj = url_parser::FromUrlString(ggufDownloadItem.downloadUrl); - auto branch = url_obj.pathParams[3]; - CTL_INF("Adding model to modellist with branch: " << branch); - - auto author_id = author.has_value() ? author.value() : "cortexso"; - modellist_utils::ModelListUtils modellist_utils_obj; - modellist_utils::ModelEntry model_entry{ - .model_id = model_config.id, - .author_repo_id = author_id, - .branch_name = branch, - .path_to_model_yaml = yaml_name.string(), - .model_alias = model_config.id, - .status = modellist_utils::ModelStatus::READY}; - modellist_utils_obj.AddModelEntry(model_entry); -} - -inline void DownloadModelCb(const DownloadTask& finishedTask) { - const DownloadItem* model_yml_di = nullptr; - const DownloadItem* gguf_di = nullptr; - auto need_parse_gguf = true; - - for (const auto& item : finishedTask.items) { - if (item.localPath.filename().string() == "model.yml") { - model_yml_di = &item; - } - if (item.localPath.extension().string() == ".gguf") { - gguf_di = &item; - } - if (item.downloadUrl.find("cortexso") != std::string::npos) { - // if downloading from cortexso, we dont need to parse gguf - need_parse_gguf = false; - } - } - - if (need_parse_gguf && gguf_di != nullptr) { - ParseGguf(*gguf_di); - } - - if (model_yml_di != nullptr) { - auto url_obj = url_parser::FromUrlString(model_yml_di->downloadUrl); - auto branch = url_obj.pathParams[3]; - CTL_INF("Adding model to modellist with branch: " << branch); - config::YamlHandler yaml_handler; - yaml_handler.ModelConfigFromFile(model_yml_di->localPath.string()); - auto mc = yaml_handler.GetModelConfig(); - - modellist_utils::ModelListUtils modellist_utils_obj; - modellist_utils::ModelEntry model_entry{ - .model_id = mc.name, - .author_repo_id = "cortexso", - .branch_name = branch, - .path_to_model_yaml = model_yml_di->localPath.string(), - .model_alias = mc.name, - .status = modellist_utils::ModelStatus::READY}; - modellist_utils_obj.AddModelEntry(model_entry); - } -} -} // namespace model_callback_utils