diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 25c0783b1..e82e07aab 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -157,7 +157,7 @@ target_link_libraries(${TARGET_NAME} PRIVATE JsonCpp::JsonCpp Drogon::Drogon Ope target_link_libraries(${TARGET_NAME} PRIVATE SQLiteCpp) target_link_libraries(${TARGET_NAME} PRIVATE eventpp::eventpp) target_link_libraries(${TARGET_NAME} PRIVATE lfreist-hwinfo::hwinfo) - + # ############################################################################## if(CMAKE_CXX_STANDARD LESS 17) diff --git a/engine/cli/CMakeLists.txt b/engine/cli/CMakeLists.txt index df4f1a76b..eb29460a7 100644 --- a/engine/cli/CMakeLists.txt +++ b/engine/cli/CMakeLists.txt @@ -83,6 +83,7 @@ add_executable(${TARGET_NAME} main.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/model_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/inference_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/hardware_service.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../services/database_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../extensions/remote-engine/remote_engine.cc ${CMAKE_CURRENT_SOURCE_DIR}/../extensions/remote-engine/template_renderer.cc ${CMAKE_CURRENT_SOURCE_DIR}/utils/easywsclient.cc diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc index 825780895..6f8f227e6 100644 --- a/engine/cli/command_line_parser.cc +++ b/engine/cli/command_line_parser.cc @@ -49,8 +49,9 @@ CommandLineParser::CommandLineParser() : app_("\nCortex.cpp CLI\n"), download_service_{std::make_shared()}, dylib_path_manager_{std::make_shared()}, - engine_service_{std::make_shared(download_service_, - dylib_path_manager_)} { + db_service_{std::make_shared()}, + engine_service_{std::make_shared( + download_service_, dylib_path_manager_, db_service_)} { supported_engines_ = engine_service_->GetSupportedEngineNames().value(); } @@ -177,7 +178,7 @@ void CommandLineParser::SetupCommonCommands() { return; commands::RunCmd rc(cml_data_.config.apiServerHost, std::stoi(cml_data_.config.apiServerPort), - cml_data_.model_id, engine_service_); + cml_data_.model_id, db_service_, engine_service_); rc.Exec(cml_data_.run_detach, run_settings_); }); } @@ -216,9 +217,10 @@ void CommandLineParser::SetupModelCommands() { CLI_LOG(model_start_cmd->help()); return; }; - commands::ModelStartCmd().Exec(cml_data_.config.apiServerHost, - std::stoi(cml_data_.config.apiServerPort), - cml_data_.model_id, run_settings_); + commands::ModelStartCmd(db_service_) + .Exec(cml_data_.config.apiServerHost, + std::stoi(cml_data_.config.apiServerPort), cml_data_.model_id, + run_settings_); }); auto stop_model_cmd = diff --git a/engine/cli/command_line_parser.h b/engine/cli/command_line_parser.h index 14e10e420..5b64f7f4d 100644 --- a/engine/cli/command_line_parser.h +++ b/engine/cli/command_line_parser.h @@ -45,6 +45,7 @@ class CommandLineParser { CLI::App app_; std::shared_ptr download_service_; std::shared_ptr dylib_path_manager_; + std::shared_ptr db_service_; std::shared_ptr engine_service_; std::vector supported_engines_; diff --git a/engine/cli/commands/chat_completion_cmd.cc b/engine/cli/commands/chat_completion_cmd.cc index 77d222176..77ee4fca3 100644 --- a/engine/cli/commands/chat_completion_cmd.cc +++ b/engine/cli/commands/chat_completion_cmd.cc @@ -56,10 +56,9 @@ void ChatCompletionCmd::Exec(const std::string& host, int port, const std::string& model_handle, std::string msg) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CLI_LOG("Error: " + model_entry.error()); return; diff --git a/engine/cli/commands/chat_completion_cmd.h b/engine/cli/commands/chat_completion_cmd.h index a784b4604..44de5d256 100644 --- a/engine/cli/commands/chat_completion_cmd.h +++ b/engine/cli/commands/chat_completion_cmd.h @@ -3,16 +3,20 @@ #include #include #include "config/model_config.h" +#include "services/database_service.h" namespace commands { class ChatCompletionCmd { public: + explicit ChatCompletionCmd(std::shared_ptr db_service) + : db_service_(db_service) {} void Exec(const std::string& host, int port, const std::string& model_handle, std::string msg); void Exec(const std::string& host, int port, const std::string& model_handle, const config::ModelConfig& mc, std::string msg); private: + std::shared_ptr db_service_; std::vector histories_; }; } // namespace commands diff --git a/engine/cli/commands/model_start_cmd.cc b/engine/cli/commands/model_start_cmd.cc index 12aec944d..ef5d5c1f2 100644 --- a/engine/cli/commands/model_start_cmd.cc +++ b/engine/cli/commands/model_start_cmd.cc @@ -13,7 +13,7 @@ bool ModelStartCmd::Exec( const std::unordered_map& options, bool print_success_log) { std::optional model_id = - SelectLocalModel(host, port, model_handle); + SelectLocalModel(host, port, model_handle, *db_service_); if (!model_id.has_value()) { return false; diff --git a/engine/cli/commands/model_start_cmd.h b/engine/cli/commands/model_start_cmd.h index 124ef463d..c69bfc32a 100644 --- a/engine/cli/commands/model_start_cmd.h +++ b/engine/cli/commands/model_start_cmd.h @@ -3,16 +3,23 @@ #include #include #include "json/json.h" +#include "services/database_service.h" namespace commands { class ModelStartCmd { public: + explicit ModelStartCmd(std::shared_ptr db_service) + : db_service_(db_service) {} bool Exec(const std::string& host, int port, const std::string& model_handle, const std::unordered_map& options, bool print_success_log = true); - private: + + private: bool UpdateConfig(Json::Value& data, const std::string& key, const std::string& value); + + private: + std::shared_ptr db_service_; }; } // namespace commands diff --git a/engine/cli/commands/run_cmd.cc b/engine/cli/commands/run_cmd.cc index 91a813d64..c01d3d806 100644 --- a/engine/cli/commands/run_cmd.cc +++ b/engine/cli/commands/run_cmd.cc @@ -14,12 +14,11 @@ namespace commands { std::optional SelectLocalModel(std::string host, int port, - const std::string& model_handle) { + const std::string& model_handle, + DatabaseService& db_service) { std::optional model_id = model_handle; - cortex::db::Models modellist_handler; - if (model_handle.empty()) { - auto all_local_models = modellist_handler.LoadModelList(); + auto all_local_models = db_service.LoadModelList(); if (all_local_models.has_error() || all_local_models.value().empty()) { CLI_LOG("No local models available!"); return std::nullopt; @@ -42,7 +41,7 @@ std::optional SelectLocalModel(std::string host, int port, CLI_LOG("Selected: " << selection.value()); } } else { - auto related_models_ids = modellist_handler.FindRelatedModel(model_handle); + auto related_models_ids = db_service.FindRelatedModel(model_handle); if (related_models_ids.has_error() || related_models_ids.value().empty()) { auto result = ModelPullCmd().Exec(host, port, model_handle); if (!result) { @@ -69,19 +68,18 @@ std::optional SelectLocalModel(std::string host, int port, void RunCmd::Exec(bool run_detach, const std::unordered_map& options) { std::optional model_id = - SelectLocalModel(host_, port_, model_handle_); + SelectLocalModel(host_, port_, model_handle_, *db_service_); if (!model_id.has_value()) { return; } - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; auto address = host_ + ":" + std::to_string(port_); try { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - auto model_entry = modellist_handler.GetModelInfo(*model_id); + auto model_entry = db_service_->GetModelInfo(*model_id); if (model_entry.has_error()) { CLI_LOG("Error: " + model_entry.error()); return; @@ -128,7 +126,7 @@ void RunCmd::Exec(bool run_detach, mc.engine.find(kLlamaEngine) == std::string::npos) || !commands::ModelStatusCmd().IsLoaded(host_, port_, *model_id)) { - auto res = commands::ModelStartCmd() + auto res = commands::ModelStartCmd(db_service_) .Exec(host_, port_, *model_id, options, false /*print_success_log*/); if (!res) { @@ -144,7 +142,7 @@ void RunCmd::Exec(bool run_detach, << commands::GetCortexBinary() << " run " << *model_id << "` for interactive chat shell"); } else { - ChatCompletionCmd().Exec(host_, port_, *model_id, mc, ""); + ChatCompletionCmd(db_service_).Exec(host_, port_, *model_id, mc, ""); } } } catch (const std::exception& e) { diff --git a/engine/cli/commands/run_cmd.h b/engine/cli/commands/run_cmd.h index b22b064f9..ec5c61fd3 100644 --- a/engine/cli/commands/run_cmd.h +++ b/engine/cli/commands/run_cmd.h @@ -2,20 +2,24 @@ #include #include +#include "services/database_service.h" #include "services/engine_service.h" namespace commands { std::optional SelectLocalModel(std::string host, int port, - const std::string& model_handle); + const std::string& model_handle, + DatabaseService& db_service); class RunCmd { public: explicit RunCmd(std::string host, int port, std::string model_handle, + std::shared_ptr db_service, std::shared_ptr engine_service) : host_{std::move(host)}, port_{port}, model_handle_{std::move(model_handle)}, + db_service_(db_service), engine_service_{engine_service} {}; void Exec(bool chat_flag, @@ -25,6 +29,7 @@ class RunCmd { std::string host_; int port_; std::string model_handle_; + std::shared_ptr db_service_; std::shared_ptr engine_service_; }; } // namespace commands diff --git a/engine/cli/commands/server_start_cmd.cc b/engine/cli/commands/server_start_cmd.cc index 3d6045cd5..4268f6362 100644 --- a/engine/cli/commands/server_start_cmd.cc +++ b/engine/cli/commands/server_start_cmd.cc @@ -114,7 +114,8 @@ bool ServerStartCmd::Exec(const std::string& host, int port, // Some engines requires to add lib search path before process being created auto download_srv = std::make_shared(); auto dylib_path_mng = std::make_shared(); - EngineService(download_srv, dylib_path_mng).RegisterEngineLibPath(); + auto db_srv = std::make_shared(); + EngineService(download_srv, dylib_path_mng, db_srv).RegisterEngineLibPath(); std::string p = cortex_utils::GetCurrentPath() + "/" + exe; execl(p.c_str(), exe.c_str(), "--start-server", "--config_file_path", diff --git a/engine/controllers/hardware.h b/engine/controllers/hardware.h index 6cca4fd2a..8b2b551ce 100644 --- a/engine/controllers/hardware.h +++ b/engine/controllers/hardware.h @@ -9,7 +9,7 @@ using namespace drogon; class Hardware : public drogon::HttpController { public: explicit Hardware(std::shared_ptr engine_svc, - std::shared_ptr hw_svc) + std::shared_ptr hw_svc) : engine_svc_(engine_svc), hw_svc_(hw_svc) {} METHOD_LIST_BEGIN METHOD_ADD(Hardware::GetHardwareInfo, "/hardware", Get); @@ -27,5 +27,5 @@ class Hardware : public drogon::HttpController { private: std::shared_ptr engine_svc_ = nullptr; - std::shared_ptr hw_svc_= nullptr; + std::shared_ptr hw_svc_= nullptr; }; \ No newline at end of file diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 1c33ab1dc..1a501287d 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -165,10 +165,9 @@ void Models::ListModel( model_service_->ForceIndexingModelList(); // Iterate through directory - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto list_entry = modellist_handler.LoadModelList(); + auto list_entry = db_service_->LoadModelList(); if (list_entry) { for (const auto& model_entry : list_entry.value()) { try { @@ -256,9 +255,8 @@ void Models::GetModel(const HttpRequestPtr& req, Json::Value ret; try { - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto model_entry = modellist_handler.GetModelInfo(model_id); + auto model_entry = db_service_->GetModelInfo(model_id); if (model_entry.has_error()) { ret["id"] = model_id; ret["object"] = "model"; @@ -337,8 +335,7 @@ void Models::UpdateModel(const HttpRequestPtr& req, namespace fmu = file_manager_utils; auto json_body = *(req->getJsonObject()); try { - cortex::db::Models model_list_utils; - auto model_entry = model_list_utils.GetModelInfo(model_id); + auto model_entry = db_service_->GetModelInfo(model_id); config::YamlHandler yaml_handler; auto yaml_fp = fmu::ToAbsoluteCortexDataPath( fs::path(model_entry.value().path_to_model_yaml)); @@ -401,7 +398,6 @@ void Models::ImportModel( auto option = (*(req->getJsonObject())).get("option", "symlink").asString(); config::GGUFHandler gguf_handler; config::YamlHandler yaml_handler; - cortex::db::Models modellist_utils_obj; std::string model_yaml_path = (file_manager_utils::GetModelsContainerPath() / std::filesystem::path("imported") / std::filesystem::path(modelHandle + ".yml")) @@ -440,7 +436,7 @@ void Models::ImportModel( model_config.name = modelName.empty() ? model_config.name : modelName; yaml_handler.UpdateModelConfig(model_config); - if (modellist_utils_obj.AddModelEntry(model_entry).value()) { + if (db_service_->AddModelEntry(model_entry).value()) { yaml_handler.WriteYamlFile(model_yaml_path); std::string success_message = "Model is imported successfully!"; LOG_INFO << success_message; @@ -667,7 +663,6 @@ void Models::AddRemoteModel( config::RemoteModelConfig model_config; model_config.LoadFromJson(*(req->getJsonObject())); - cortex::db::Models modellist_utils_obj; std::string model_yaml_path = (file_manager_utils::GetModelsContainerPath() / std::filesystem::path("remote") / std::filesystem::path(model_handle + ".yml")) @@ -683,7 +678,7 @@ void Models::AddRemoteModel( "openai"}; std::filesystem::create_directories( std::filesystem::path(model_yaml_path).parent_path()); - if (modellist_utils_obj.AddModelEntry(model_entry).value()) { + if (db_service_->AddModelEntry(model_entry).value()) { model_config.SaveToYamlFile(model_yaml_path); std::string success_message = "Model is imported successfully!"; LOG_INFO << success_message; diff --git a/engine/controllers/models.h b/engine/controllers/models.h index d3200f33a..60053acdb 100644 --- a/engine/controllers/models.h +++ b/engine/controllers/models.h @@ -45,10 +45,12 @@ class Models : public drogon::HttpController { ADD_METHOD_TO(Models::GetModelSources, "/v1/models/sources", Get); METHOD_LIST_END - explicit Models(std::shared_ptr model_service, + explicit Models(std::shared_ptr db_service, + std::shared_ptr model_service, std::shared_ptr engine_service, - std::shared_ptr mss) - : model_service_{model_service}, + std::shared_ptr mss) + : db_service_(db_service), + model_service_{model_service}, engine_service_{engine_service}, model_src_svc_(mss) {} @@ -105,7 +107,8 @@ class Models : public drogon::HttpController { std::function&& callback); private: + std::shared_ptr db_service_; std::shared_ptr model_service_; std::shared_ptr engine_service_; - std::shared_ptr model_src_svc_; + std::shared_ptr model_src_svc_; }; diff --git a/engine/controllers/server.cc b/engine/controllers/server.cc index 19842bcdb..d8e29eb1b 100644 --- a/engine/controllers/server.cc +++ b/engine/controllers/server.cc @@ -8,7 +8,7 @@ using namespace inferences; namespace inferences { -server::server(std::shared_ptr inference_service, +server::server(std::shared_ptr inference_service, std::shared_ptr engine_service) : inference_svc_(inference_service), engine_service_(engine_service) { #if defined(_WIN32) @@ -45,7 +45,7 @@ void server::ChatCompletion( }(); LOG_DEBUG << "request body: " << json_body->toStyledString(); - auto q = std::make_shared(); + auto q = std::make_shared(); auto ir = inference_svc_->HandleChatCompletion(q, json_body); if (ir.has_error()) { auto err = ir.error(); @@ -67,7 +67,7 @@ void server::ChatCompletion( void server::Embedding(const HttpRequestPtr& req, std::function&& callback) { LOG_TRACE << "Start embedding"; - auto q = std::make_shared(); + auto q = std::make_shared(); auto ir = inference_svc_->HandleEmbedding(q, req->getJsonObject()); if (ir.has_error()) { auto err = ir.error(); @@ -138,7 +138,7 @@ void server::LoadModel(const HttpRequestPtr& req, } void server::ProcessStreamRes(std::function cb, - std::shared_ptr q, + std::shared_ptr q, const std::string& engine_type, const std::string& model_id) { auto err_or_done = std::make_shared(false); @@ -178,7 +178,7 @@ void server::ProcessStreamRes(std::function cb, } void server::ProcessNonStreamRes(std::function cb, - services::SyncQueue& q) { + SyncQueue& q) { auto [status, res] = q.wait_and_pop(); function_calling_utils::PostProcessResponse(res); LOG_DEBUG << "response: " << res.toStyledString(); diff --git a/engine/controllers/server.h b/engine/controllers/server.h index 22ea86c30..ef8a32f5d 100644 --- a/engine/controllers/server.h +++ b/engine/controllers/server.h @@ -27,7 +27,7 @@ class server : public drogon::HttpController, public BaseChatCompletion, public BaseEmbedding { public: - server(std::shared_ptr inference_service, + server(std::shared_ptr inference_service, std::shared_ptr engine_service); ~server(); METHOD_LIST_BEGIN @@ -72,14 +72,14 @@ class server : public drogon::HttpController, private: void ProcessStreamRes(std::function cb, - std::shared_ptr q, + std::shared_ptr q, const std::string& engine_type, const std::string& model_id); void ProcessNonStreamRes(std::function cb, - services::SyncQueue& q); + SyncQueue& q); private: - std::shared_ptr inference_svc_; + std::shared_ptr inference_svc_; std::shared_ptr engine_service_; }; }; // namespace inferences diff --git a/engine/main.cc b/engine/main.cc index 938392bf0..77f51c7fa 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -21,6 +21,7 @@ #include "repositories/thread_fs_repository.h" #include "services/assistant_service.h" #include "services/config_service.h" +#include "services/database_service.h" #include "services/file_watcher_service.h" #include "services/message_service.h" #include "services/model_service.h" @@ -120,7 +121,8 @@ void RunServer(std::optional host, std::optional port, LOG_INFO << "cortex.cpp version: undefined"; #endif - auto hw_service = std::make_shared(); + auto db_service = std::make_shared(); + auto hw_service = std::make_shared(db_service); hw_service->UpdateHardwareInfos(); if (hw_service->ShouldRestart()) { CTL_INF("Restart to update hardware configuration"); @@ -140,7 +142,8 @@ void RunServer(std::optional host, std::optional port, // utils auto dylib_path_manager = std::make_shared(); - auto file_repo = std::make_shared(data_folder_path); + auto file_repo = + std::make_shared(data_folder_path, db_service); auto msg_repo = std::make_shared(data_folder_path); auto thread_repo = std::make_shared(data_folder_path); auto assistant_repo = @@ -156,13 +159,12 @@ void RunServer(std::optional host, std::optional port, auto config_service = std::make_shared(); auto download_service = std::make_shared(event_queue_ptr, config_service); - auto engine_service = - std::make_shared(download_service, dylib_path_manager); - auto inference_svc = - std::make_shared(engine_service); - auto model_src_svc = std::make_shared(); + auto engine_service = std::make_shared( + download_service, dylib_path_manager, db_service); + auto inference_svc = std::make_shared(engine_service); + auto model_src_svc = std::make_shared(db_service); auto model_service = std::make_shared( - download_service, inference_svc, engine_service); + db_service, hw_service, download_service, inference_svc, engine_service); inference_svc->SetModelService(model_service); auto file_watcher_srv = std::make_shared( @@ -177,8 +179,8 @@ void RunServer(std::optional host, std::optional port, auto thread_ctl = std::make_shared(thread_srv, message_srv); auto message_ctl = std::make_shared(message_srv); auto engine_ctl = std::make_shared(engine_service); - auto model_ctl = - std::make_shared(model_service, engine_service, model_src_svc); + auto model_ctl = std::make_shared(db_service, model_service, + engine_service, model_src_svc); auto event_ctl = std::make_shared(event_queue_ptr); auto pm_ctl = std::make_shared(); auto hw_ctl = std::make_shared(engine_service, hw_service); diff --git a/engine/repositories/file_fs_repository.cc b/engine/repositories/file_fs_repository.cc index a209d33c3..e6c28b38e 100644 --- a/engine/repositories/file_fs_repository.cc +++ b/engine/repositories/file_fs_repository.cc @@ -17,7 +17,6 @@ cpp::result FileFsRepository::StoreFile( std::filesystem::create_directories(file_container_path); } - cortex::db::File db; auto original_filename = file_metadata.filename; auto file_full_path = file_container_path / original_filename; @@ -53,7 +52,7 @@ cpp::result FileFsRepository::StoreFile( file.flush(); file.close(); - auto result = db.AddFileEntry(file_metadata); + auto result = db_service_->AddFileEntry(file_metadata); if (result.has_error()) { std::filesystem::remove(file_full_path); return cpp::fail(result.error()); @@ -70,8 +69,7 @@ cpp::result FileFsRepository::StoreFile( cpp::result, std::string> FileFsRepository::ListFiles( const std::string& purpose, uint8_t limit, const std::string& order, const std::string& after) const { - cortex::db::File db; - auto res = db.GetFileList(); + auto res = db_service_->GetFileList(); if (res.has_error()) { return cpp::fail(res.error()); } @@ -101,8 +99,7 @@ cpp::result FileFsRepository::RetrieveFile( CTL_INF("Retrieving file: " + file_id); auto file_container_path = GetFilePath(); - cortex::db::File db; - auto res = db.GetFileById(file_id); + auto res = db_service_->GetFileById(file_id); if (res.has_error()) { return cpp::fail(res.error()); } @@ -158,15 +155,14 @@ cpp::result FileFsRepository::DeleteFileLocal( const std::string& file_id) { CTL_INF("Deleting file: " + file_id); auto file_container_path = GetFilePath(); - cortex::db::File db; - auto file_metadata = db.GetFileById(file_id); + auto file_metadata = db_service_->GetFileById(file_id); if (file_metadata.has_error()) { return cpp::fail(file_metadata.error()); } auto file_path = file_container_path / file_metadata->filename; - auto res = db.DeleteFileEntry(file_id); + auto res = db_service_->DeleteFileEntry(file_id); if (res.has_error()) { CTL_ERR("Failed to delete file entry: " << res.error()); return cpp::fail(res.error()); diff --git a/engine/repositories/file_fs_repository.h b/engine/repositories/file_fs_repository.h index 77af60dfc..e2ad424a7 100644 --- a/engine/repositories/file_fs_repository.h +++ b/engine/repositories/file_fs_repository.h @@ -2,6 +2,7 @@ #include #include "common/repository/file_repository.h" +#include "services/database_service.h" #include "utils/logging_utils.h" class FileFsRepository : public FileRepository { @@ -28,8 +29,9 @@ class FileFsRepository : public FileRepository { cpp::result DeleteFileLocal( const std::string& file_id) override; - explicit FileFsRepository(const std::filesystem::path& data_folder_path) - : data_folder_path_{data_folder_path} { + explicit FileFsRepository(const std::filesystem::path& data_folder_path, + std::shared_ptr db_service) + : data_folder_path_{data_folder_path}, db_service_(db_service) { CTL_INF("Constructing FileFsRepository.."); auto file_container_path = data_folder_path_ / kFileContainerFolderName; @@ -47,4 +49,5 @@ class FileFsRepository : public FileRepository { * The path to the data folder. */ std::filesystem::path data_folder_path_; + std::shared_ptr db_service_ = nullptr; }; diff --git a/engine/services/database_service.cc b/engine/services/database_service.cc new file mode 100644 index 000000000..d4cd977a9 --- /dev/null +++ b/engine/services/database_service.cc @@ -0,0 +1,130 @@ +#include "database_service.h" + +// begin engines +std::optional DatabaseService::UpsertEngine( + const std::string& engine_name, const std::string& type, + const std::string& api_key, const std::string& url, + const std::string& version, const std::string& variant, + const std::string& status, const std::string& metadata) { + return cortex::db::Engines().UpsertEngine(engine_name, type, api_key, url, + version, variant, status, metadata); +} + +std::optional> DatabaseService::GetEngines() const { + return cortex::db::Engines().GetEngines(); +} + +std::optional DatabaseService::GetEngineById(int id) const { + return cortex::db::Engines().GetEngineById(id); +} + +std::optional DatabaseService::GetEngineByNameAndVariant( + const std::string& engine_name, + const std::optional variant) const { + return cortex::db::Engines().GetEngineByNameAndVariant(engine_name, variant); +} + +std::optional DatabaseService::DeleteEngineById(int id) { + return cortex::db::Engines().DeleteEngineById(id); +} +// end engines + +// begin file +cpp::result, std::string> +DatabaseService::GetFileList() const { + return cortex::db::File().GetFileList(); +} + +cpp::result DatabaseService::GetFileById( + const std::string& file_id) const { + return cortex::db::File().GetFileById(file_id); +} + +cpp::result DatabaseService::AddFileEntry( + OpenAi::File& file) { + return cortex::db::File().AddFileEntry(file); +} + +cpp::result DatabaseService::DeleteFileEntry( + const std::string& file_id) { + return cortex::db::File().DeleteFileEntry(file_id); +} +// end file + +// begin hardware +cpp::result, std::string> +DatabaseService::LoadHardwareList() const { + return cortex::db::Hardware().LoadHardwareList(); +} + +cpp::result DatabaseService::AddHardwareEntry( + const HardwareEntry& new_entry) { + return cortex::db::Hardware().AddHardwareEntry(new_entry); +} + +cpp::result DatabaseService::UpdateHardwareEntry( + const std::string& id, const HardwareEntry& updated_entry) { + return cortex::db::Hardware().UpdateHardwareEntry(id, updated_entry); +} + +cpp::result DatabaseService::DeleteHardwareEntry( + const std::string& id) { + return cortex::db::Hardware().DeleteHardwareEntry(id); +} +// end hardware + +// begin models +cpp::result, std::string> +DatabaseService::LoadModelList() const { + return cortex::db::Models().LoadModelList(); +} + +cpp::result DatabaseService::GetModelInfo( + const std::string& identifier) const { + return cortex::db::Models().GetModelInfo(identifier); +} + +cpp::result DatabaseService::AddModelEntry( + ModelEntry new_entry) { + return cortex::db::Models().AddModelEntry(new_entry); +} + +cpp::result DatabaseService::UpdateModelEntry( + const std::string& identifier, const ModelEntry& updated_entry) { + return cortex::db::Models().UpdateModelEntry(identifier, updated_entry); +} + +cpp::result DatabaseService::DeleteModelEntry( + const std::string& identifier) { + return cortex::db::Models().DeleteModelEntry(identifier); +} + +cpp::result DatabaseService::DeleteModelEntryWithOrg( + const std::string& src) { + return cortex::db::Models().DeleteModelEntryWithOrg(src); +} + +cpp::result DatabaseService::DeleteModelEntryWithRepo( + const std::string& src) { + return cortex::db::Models().DeleteModelEntryWithRepo(src); +} + +cpp::result, std::string> +DatabaseService::FindRelatedModel(const std::string& identifier) const { + return cortex::db::Models().FindRelatedModel(identifier); +} + +bool DatabaseService::HasModel(const std::string& identifier) const { + return cortex::db::Models().HasModel(identifier); +} + +cpp::result, std::string> +DatabaseService::GetModelSources() const { + return cortex::db::Models().GetModelSources(); +} + +cpp::result, std::string> DatabaseService::GetModels( + const std::string& model_src) const { + return cortex::db::Models().GetModels(model_src); +} +// end models \ No newline at end of file diff --git a/engine/services/database_service.h b/engine/services/database_service.h new file mode 100644 index 000000000..4fb4f7be0 --- /dev/null +++ b/engine/services/database_service.h @@ -0,0 +1,68 @@ +#pragma once +#include "database/engines.h" +#include "database/file.h" +#include "database/hardware.h" +#include "database/models.h" + +using EngineEntry = cortex::db::EngineEntry; +using HardwareEntry = cortex::db::HardwareEntry; +using ModelEntry = cortex::db::ModelEntry; + +class DatabaseService { + public: + // engines + std::optional UpsertEngine( + const std::string& engine_name, const std::string& type, + const std::string& api_key, const std::string& url, + const std::string& version, const std::string& variant, + const std::string& status, const std::string& metadata); + + std::optional> GetEngines() const; + std::optional GetEngineById(int id) const; + std::optional GetEngineByNameAndVariant( + const std::string& engine_name, + const std::optional variant = std::nullopt) const; + + std::optional DeleteEngineById(int id); + + // file + cpp::result, std::string> GetFileList() const; + + cpp::result GetFileById( + const std::string& file_id) const; + + cpp::result AddFileEntry(OpenAi::File& file); + + cpp::result DeleteFileEntry(const std::string& file_id); + + // hardware + cpp::result, std::string> LoadHardwareList() const; + cpp::result AddHardwareEntry( + const HardwareEntry& new_entry); + cpp::result UpdateHardwareEntry( + const std::string& id, const HardwareEntry& updated_entry); + cpp::result DeleteHardwareEntry(const std::string& id); + + // models + cpp::result, std::string> LoadModelList() const; + cpp::result GetModelInfo( + const std::string& identifier) const; + void PrintModelInfo(const ModelEntry& entry) const; + cpp::result AddModelEntry(ModelEntry new_entry); + cpp::result UpdateModelEntry( + const std::string& identifier, const ModelEntry& updated_entry); + cpp::result DeleteModelEntry( + const std::string& identifier); + cpp::result DeleteModelEntryWithOrg( + const std::string& src); + cpp::result DeleteModelEntryWithRepo( + const std::string& src); + cpp::result, std::string> FindRelatedModel( + const std::string& identifier) const; + bool HasModel(const std::string& identifier) const; + cpp::result, std::string> GetModelSources() const; + cpp::result, std::string> GetModels( + const std::string& model_src) const; + + private: +}; \ No newline at end of file diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index 93311f98b..bb0727bcc 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -1032,8 +1032,8 @@ cpp::result EngineService::UpdateEngine( cpp::result, std::string> EngineService::GetEngines() { - cortex::db::Engines engines; - auto get_res = engines.GetEngines(); + assert(db_service_); + auto get_res = db_service_->GetEngines(); if (!get_res.has_value()) { return cpp::fail("Failed to get engine entries"); @@ -1044,8 +1044,8 @@ EngineService::GetEngines() { cpp::result EngineService::GetEngineById( int id) { - cortex::db::Engines engines; - auto get_res = engines.GetEngineById(id); + assert(db_service_); + auto get_res = db_service_->GetEngineById(id); if (!get_res.has_value()) { return cpp::fail("Engine with ID " + std::to_string(id) + " not found"); @@ -1058,8 +1058,8 @@ cpp::result EngineService::GetEngineByNameAndVariant( const std::string& engine_name, const std::optional variant) { - cortex::db::Engines engines; - auto get_res = engines.GetEngineByNameAndVariant(engine_name, variant); + assert(db_service_); + auto get_res = db_service_->GetEngineByNameAndVariant(engine_name, variant); if (!get_res.has_value()) { if (variant.has_value()) { @@ -1078,9 +1078,9 @@ cpp::result EngineService::UpsertEngine( const std::string& api_key, const std::string& url, const std::string& version, const std::string& variant, const std::string& status, const std::string& metadata) { - cortex::db::Engines engines; - auto upsert_res = engines.UpsertEngine(engine_name, type, api_key, url, - version, variant, status, metadata); + assert(db_service_); + auto upsert_res = db_service_->UpsertEngine( + engine_name, type, api_key, url, version, variant, status, metadata); if (upsert_res.has_value()) { return upsert_res.value(); } else { @@ -1089,8 +1089,8 @@ cpp::result EngineService::UpsertEngine( } std::string EngineService::DeleteEngine(int id) { - cortex::db::Engines engines; - auto delete_res = engines.DeleteEngineById(id); + assert(db_service_); + auto delete_res = db_service_->DeleteEngineById(id); if (delete_res.has_value()) { return delete_res.value(); } else { diff --git a/engine/services/engine_service.h b/engine/services/engine_service.h index 8ead4f6d6..fcd3fdda9 100644 --- a/engine/services/engine_service.h +++ b/engine/services/engine_service.h @@ -12,6 +12,7 @@ #include "cortex-common/cortexpythoni.h" #include "cortex-common/remote_enginei.h" #include "database/engines.h" +#include "services/database_service.h" #include "services/download_service.h" #include "utils/cpuid/cpu_info.h" #include "utils/dylib.h" @@ -59,16 +60,19 @@ class EngineService : public EngineServiceI { std::string cuda_driver_version; }; HardwareInfo hw_inf_; + std::shared_ptr db_service_ = nullptr; public: explicit EngineService( std::shared_ptr download_service, - std::shared_ptr dylib_path_manager) + std::shared_ptr dylib_path_manager, + std::shared_ptr db_service) : download_service_{download_service}, dylib_path_manager_{dylib_path_manager}, hw_inf_{.sys_inf = system_info_utils::GetSystemInfo(), .cuda_driver_version = - system_info_utils::GetDriverAndCudaVersion().second} {} + system_info_utils::GetDriverAndCudaVersion().second}, + db_service_(db_service) {} std::vector GetEngineInfoList() const; diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc index ca2bd8ed9..5552aca56 100644 --- a/engine/services/hardware_service.cc +++ b/engine/services/hardware_service.cc @@ -11,8 +11,6 @@ #include "database/hardware.h" #include "utils/cortex_utils.h" -namespace services { - namespace { bool TryConnectToServer(const std::string& host, int port) { constexpr const auto kMaxRetry = 4u; @@ -34,9 +32,8 @@ bool TryConnectToServer(const std::string& host, int port) { HardwareInfo HardwareService::GetHardwareInfo() { // append active state - cortex::db::Hardware hw_db; auto gpus = cortex::hw::GetGPUInfo(); - auto res = hw_db.LoadHardwareList(); + auto res = db_service_->LoadHardwareList(); if (res.has_value()) { // Only a few elements, brute-force is enough for (auto& entry : res.value()) { @@ -210,7 +207,6 @@ bool HardwareService::SetActivateHardwareConfig( const cortex::hw::ActivateHardwareConfig& ahc) { // Note: need to map software_id and hardware_id // Update to db - cortex::db::Hardware hw_db; // copy all gpu information to new vector auto ahc_gpus = ahc.gpus; auto activate = [&ahc](int software_id) { @@ -225,7 +221,7 @@ bool HardwareService::SetActivateHardwareConfig( return INT_MAX; }; - auto res = hw_db.LoadHardwareList(); + auto res = db_service_->LoadHardwareList(); if (res.has_value()) { bool need_update = false; std::vector> activated_ids; @@ -258,7 +254,7 @@ bool HardwareService::SetActivateHardwareConfig( for (auto& e : res.value()) { e.activated = activate(e.software_id); e.priority = priority(e.software_id); - auto res = hw_db.UpdateHardwareEntry(e.uuid, e); + auto res = db_service_->UpdateHardwareEntry(e.uuid, e); if (res.has_error()) { CTL_WRN(res.error()); } @@ -271,8 +267,7 @@ bool HardwareService::SetActivateHardwareConfig( void HardwareService::UpdateHardwareInfos() { using HwEntry = cortex::db::HardwareEntry; auto gpus = cortex::hw::GetGPUInfo(); - cortex::db::Hardware hw_db; - auto b = hw_db.LoadHardwareList(); + auto b = db_service_->LoadHardwareList(); std::vector> activated_gpu_bf; std::string debug_b; for (auto const& he : b.value()) { @@ -285,7 +280,8 @@ void HardwareService::UpdateHardwareInfos() { for (auto const& gpu : gpus) { // ignore error // Note: only support NVIDIA for now, so hardware_id = software_id - auto res = hw_db.AddHardwareEntry(HwEntry{.uuid = gpu.uuid, + auto res = + db_service_->AddHardwareEntry(HwEntry{.uuid = gpu.uuid, .type = "gpu", .hardware_id = std::stoi(gpu.id), .software_id = std::stoi(gpu.id), @@ -296,7 +292,7 @@ void HardwareService::UpdateHardwareInfos() { } } - auto a = hw_db.LoadHardwareList(); + auto a = db_service_->LoadHardwareList(); std::vector a_gpu; std::vector> activated_gpu_af; std::string debug_a; @@ -350,11 +346,10 @@ bool HardwareService::IsValidConfig( const cortex::hw::ActivateHardwareConfig& ahc) { if (ahc.gpus.empty()) return true; - cortex::db::Hardware hw_db; auto is_valid = [&ahc](int software_id) { return std::count(ahc.gpus.begin(), ahc.gpus.end(), software_id) > 0; }; - auto res = hw_db.LoadHardwareList(); + auto res = db_service_->LoadHardwareList(); if (res.has_value()) { for (auto const& e : res.value()) { if (is_valid(e.software_id)) { @@ -364,4 +359,3 @@ bool HardwareService::IsValidConfig( } return false; } -} // namespace services diff --git a/engine/services/hardware_service.h b/engine/services/hardware_service.h index 48ab7a4b1..ad9d70233 100644 --- a/engine/services/hardware_service.h +++ b/engine/services/hardware_service.h @@ -4,6 +4,7 @@ #include #include "common/hardware_config.h" +#include "database_service.h" #include "utils/hardware/cpu_info.h" #include "utils/hardware/gpu_info.h" #include "utils/hardware/os_info.h" @@ -11,8 +12,6 @@ #include "utils/hardware/ram_info.h" #include "utils/hardware/storage_info.h" -namespace services { - struct HardwareInfo { cortex::hw::CPU cpu; cortex::hw::OS os; @@ -24,6 +23,8 @@ struct HardwareInfo { class HardwareService { public: + explicit HardwareService(std::shared_ptr db_service) + : db_service_(db_service) {} HardwareInfo GetHardwareInfo(); bool Restart(const std::string& host, int port); bool SetActivateHardwareConfig(const cortex::hw::ActivateHardwareConfig& ahc); @@ -32,6 +33,6 @@ class HardwareService { bool IsValidConfig(const cortex::hw::ActivateHardwareConfig& ahc); private: + std::shared_ptr db_service_ = nullptr; std::optional ahc_; -}; -} // namespace services +}; \ No newline at end of file diff --git a/engine/services/inference_service.cc b/engine/services/inference_service.cc index 08107562b..9d8e9f4f8 100644 --- a/engine/services/inference_service.cc +++ b/engine/services/inference_service.cc @@ -4,7 +4,6 @@ #include "utils/function_calling/common.h" #include "utils/jinja_utils.h" -namespace services { cpp::result InferenceService::HandleChatCompletion( std::shared_ptr q, std::shared_ptr json_body) { std::string engine_type; @@ -337,4 +336,3 @@ bool InferenceService::HasFieldInReq(std::shared_ptr json_body, } return true; } -} // namespace services diff --git a/engine/services/inference_service.h b/engine/services/inference_service.h index 54bc9dc29..75b07b1a3 100644 --- a/engine/services/inference_service.h +++ b/engine/services/inference_service.h @@ -7,8 +7,6 @@ #include "services/model_service.h" #include "utils/result.hpp" -namespace services { - // Status and result using InferResult = std::pair; @@ -68,4 +66,3 @@ class InferenceService { std::shared_ptr engine_service_; std::weak_ptr model_service_; }; -} // namespace services diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index be0eb12a7..2d69e0f17 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -21,7 +21,8 @@ #include "utils/widechar_conv.h" namespace { -void ParseGguf(const DownloadItem& ggufDownloadItem, +void ParseGguf(DatabaseService& db_service, + const DownloadItem& ggufDownloadItem, std::optional author, std::optional name, std::optional size) { @@ -64,8 +65,7 @@ void ParseGguf(const DownloadItem& ggufDownloadItem, CTL_INF("path_to_model_yaml: " << rel.string()); auto author_id = author.has_value() ? author.value() : "cortexso"; - cortex::db::Models modellist_utils_obj; - if (!modellist_utils_obj.HasModel(ggufDownloadItem.id)) { + if (!db_service.HasModel(ggufDownloadItem.id)) { cortex::db::ModelEntry model_entry{ .model = ggufDownloadItem.id, .author_repo_id = author_id, @@ -73,18 +73,17 @@ void ParseGguf(const DownloadItem& ggufDownloadItem, .path_to_model_yaml = rel.string(), .model_alias = ggufDownloadItem.id, .status = cortex::db::ModelStatus::Downloaded}; - auto result = modellist_utils_obj.AddModelEntry(model_entry); + auto result = db_service.AddModelEntry(model_entry); if (result.has_error()) { CTL_ERR("Error adding model to modellist: " + result.error()); } } else { - if (auto m = modellist_utils_obj.GetModelInfo(ggufDownloadItem.id); + if (auto m = db_service.GetModelInfo(ggufDownloadItem.id); m.has_value()) { auto upd_m = m.value(); upd_m.status = cortex::db::ModelStatus::Downloaded; - if (auto r = - modellist_utils_obj.UpdateModelEntry(ggufDownloadItem.id, upd_m); + if (auto r = db_service.UpdateModelEntry(ggufDownloadItem.id, upd_m); r.has_error()) { CTL_ERR(r.error()); } @@ -137,10 +136,9 @@ cpp::result GetDownloadTask( void ModelService::ForceIndexingModelList() { CTL_INF("Force indexing model list"); - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto list_entry = modellist_handler.LoadModelList(); + auto list_entry = db_service_->LoadModelList(); if (list_entry.has_error()) { CTL_ERR("Failed to load model list: " << list_entry.error()); return; @@ -164,8 +162,7 @@ void ModelService::ForceIndexingModelList() { yaml_handler.Reset(); } catch (const std::exception& e) { // remove in db - auto remove_result = - modellist_handler.DeleteModelEntry(model_entry.model); + auto remove_result = db_service_->DeleteModelEntry(model_entry.model); // silently ignore result } } @@ -218,10 +215,8 @@ cpp::result ModelService::HandleCortexsoModel( auto default_model_branch = huggingface_utils::GetDefaultBranch(modelName); - cortex::db::Models modellist_handler; - auto downloaded_model_ids = - modellist_handler.FindRelatedModel(modelName).value_or( - std::vector{}); + auto downloaded_model_ids = db_service_->FindRelatedModel(modelName).value_or( + std::vector{}); std::vector avai_download_opts{}; for (const auto& branch : branches.value()) { @@ -261,9 +256,8 @@ cpp::result ModelService::HandleCortexsoModel( std::optional ModelService::GetDownloadedModel( const std::string& modelId) const { - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto model_entry = modellist_handler.GetModelInfo(modelId); + auto model_entry = db_service_->GetModelInfo(modelId); if (!model_entry.has_value()) { return std::nullopt; } @@ -310,7 +304,6 @@ cpp::result ModelService::HandleDownloadUrlAsync( } std::string huggingFaceHost{kHuggingFaceHost}; - cortex::db::Models modellist_handler; std::string unique_model_id = ""; if (temp_model_id.has_value()) { unique_model_id = temp_model_id.value(); @@ -318,7 +311,7 @@ cpp::result ModelService::HandleDownloadUrlAsync( unique_model_id = author + ":" + model_id + ":" + file_name; } - auto model_entry = modellist_handler.GetModelInfo(unique_model_id); + auto model_entry = db_service_->GetModelInfo(unique_model_id); if (model_entry.has_value() && model_entry->status == cortex::db::ModelStatus::Downloaded) { CLI_LOG("Model already downloaded: " << unique_model_id); @@ -346,14 +339,15 @@ cpp::result ModelService::HandleDownloadUrlAsync( .localPath = local_path, }}}}; - auto on_finished = [author, temp_name](const DownloadTask& finishedTask) { + auto on_finished = [this, author, + temp_name](const DownloadTask& finishedTask) { // Sum downloadedBytes from all items uint64_t model_size = 0; for (const auto& item : finishedTask.items) { model_size = model_size + item.bytes.value_or(0); } auto gguf_download_item = finishedTask.items[0]; - ParseGguf(gguf_download_item, author, temp_name, model_size); + ParseGguf(*db_service_, gguf_download_item, author, temp_name, model_size); }; downloadTask.id = unique_model_id; @@ -366,11 +360,10 @@ ModelService::GetEstimation(const std::string& model_handle, int n_ubatch) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -384,8 +377,8 @@ ModelService::GetEstimation(const std::string& model_handle, fs::path(model_entry.value().path_to_model_yaml)) .string()); auto mc = yaml_handler.GetModelConfig(); - services::HardwareService hw_svc; - auto hw_info = hw_svc.GetHardwareInfo(); + assert(hw_service_); + auto hw_info = hw_service_->GetHardwareInfo(); auto free_vram_MiB = 0u; for (const auto& gpu : hw_info.gpus) { free_vram_MiB += gpu.free_vram; @@ -438,8 +431,7 @@ cpp::result ModelService::HandleUrl( std::string huggingFaceHost{kHuggingFaceHost}; std::string unique_model_id{author + ":" + model_id + ":" + file_name}; - cortex::db::Models modellist_handler; - auto model_entry = modellist_handler.GetModelInfo(unique_model_id); + auto model_entry = db_service_->GetModelInfo(unique_model_id); if (model_entry.has_value()) { CLI_LOG("Model already downloaded: " << unique_model_id); @@ -467,14 +459,14 @@ cpp::result ModelService::HandleUrl( .localPath = local_path, }}}}; - auto on_finished = [author](const DownloadTask& finishedTask) { + auto on_finished = [this, author](const DownloadTask& finishedTask) { // Sum downloadedBytes from all items uint64_t model_size = 0; for (const auto& item : finishedTask.items) { model_size = model_size + item.bytes.value_or(0); } auto gguf_download_item = finishedTask.items[0]; - ParseGguf(gguf_download_item, author, std::nullopt, model_size); + ParseGguf(*db_service_, gguf_download_item, author, std::nullopt, model_size); }; auto result = download_service_->AddDownloadTask(downloadTask, on_finished); @@ -488,7 +480,7 @@ cpp::result ModelService::HandleUrl( } bool ModelService::HasModel(const std::string& id) const { - return cortex::db::Models().HasModel(id); + return db_service_->HasModel(id); } cpp::result @@ -501,7 +493,6 @@ ModelService::DownloadModelFromCortexsoAsync( return cpp::fail(download_task.error()); } - cortex::db::Models modellist_handler; std::string unique_model_id = ""; if (temp_model_id.has_value()) { unique_model_id = temp_model_id.value(); @@ -509,13 +500,13 @@ ModelService::DownloadModelFromCortexsoAsync( unique_model_id = name + ":" + branch; } - auto model_entry = modellist_handler.GetModelInfo(unique_model_id); + auto model_entry = db_service_->GetModelInfo(unique_model_id); if (model_entry.has_value() && model_entry->status == cortex::db::ModelStatus::Downloaded) { return cpp::fail("Please delete the model before downloading again"); } - auto on_finished = [unique_model_id, + auto on_finished = [this, unique_model_id, branch](const DownloadTask& finishedTask) { const DownloadItem* model_yml_item = nullptr; auto need_parse_gguf = true; @@ -551,8 +542,7 @@ ModelService::DownloadModelFromCortexsoAsync( file_manager_utils::ToRelativeCortexDataPath(model_yml_item->localPath); CTL_INF("path_to_model_yaml: " << rel.string()); - cortex::db::Models modellist_utils_obj; - if (!modellist_utils_obj.HasModel(unique_model_id)) { + if (!db_service_->HasModel(unique_model_id)) { cortex::db::ModelEntry model_entry{ .model = unique_model_id, .author_repo_id = "cortexso", @@ -560,18 +550,16 @@ ModelService::DownloadModelFromCortexsoAsync( .path_to_model_yaml = rel.string(), .model_alias = unique_model_id, .status = cortex::db::ModelStatus::Downloaded}; - auto result = modellist_utils_obj.AddModelEntry(model_entry); + auto result = db_service_->AddModelEntry(model_entry); if (result.has_error()) { CTL_ERR("Error adding model to modellist: " + result.error()); } } else { - if (auto m = modellist_utils_obj.GetModelInfo(unique_model_id); - m.has_value()) { + if (auto m = db_service_->GetModelInfo(unique_model_id); m.has_value()) { auto upd_m = m.value(); upd_m.status = cortex::db::ModelStatus::Downloaded; - if (auto r = - modellist_utils_obj.UpdateModelEntry(unique_model_id, upd_m); + if (auto r = db_service_->UpdateModelEntry(unique_model_id, upd_m); r.has_error()) { CTL_ERR(r.error()); } @@ -595,7 +583,7 @@ cpp::result ModelService::DownloadModelFromCortexso( } std::string model_id{name + ":" + branch}; - auto on_finished = [branch, model_id](const DownloadTask& finishedTask) { + auto on_finished = [this, branch, model_id](const DownloadTask& finishedTask) { const DownloadItem* model_yml_item = nullptr; auto need_parse_gguf = true; @@ -622,8 +610,7 @@ cpp::result ModelService::DownloadModelFromCortexso( file_manager_utils::ToRelativeCortexDataPath(model_yml_item->localPath); CTL_INF("path_to_model_yaml: " << rel.string()); - cortex::db::Models modellist_utils_obj; - if (!modellist_utils_obj.HasModel(model_id)) { + if (!db_service_->HasModel(model_id)) { cortex::db::ModelEntry model_entry{ .model = model_id, .author_repo_id = "cortexso", @@ -631,16 +618,16 @@ cpp::result ModelService::DownloadModelFromCortexso( .path_to_model_yaml = rel.string(), .model_alias = model_id, .status = cortex::db::ModelStatus::Downloaded}; - auto result = modellist_utils_obj.AddModelEntry(model_entry); + auto result = db_service_->AddModelEntry(model_entry); if (result.has_error()) { CTL_ERR("Error adding model to modellist: " + result.error()); } } else { - if (auto m = modellist_utils_obj.GetModelInfo(model_id); m.has_value()) { + if (auto m = db_service_->GetModelInfo(model_id); m.has_value()) { auto upd_m = m.value(); upd_m.status = cortex::db::ModelStatus::Downloaded; - if (auto r = modellist_utils_obj.UpdateModelEntry(model_id, upd_m); + if (auto r = db_service_->UpdateModelEntry(model_id, upd_m); r.has_error()) { CTL_ERR(r.error()); } @@ -694,7 +681,6 @@ cpp::result ModelService::DeleteModel( const std::string& model_handle) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; auto result = StopModel(model_handle); @@ -706,7 +692,7 @@ cpp::result ModelService::DeleteModel( } try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -737,7 +723,7 @@ cpp::result ModelService::DeleteModel( } // update model.list - if (modellist_handler.DeleteModelEntry(model_handle)) { + if (db_service_->DeleteModelEntry(model_handle)) { return {}; } else { return cpp::fail("Could not delete model: " + model_handle); @@ -753,7 +739,6 @@ cpp::result ModelService::StartModel( bool bypass_model_check) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; std::optional custom_prompt_template; std::optional ctx_len; @@ -771,7 +756,7 @@ cpp::result ModelService::StartModel( Json::Value json_data; // Currently we don't support download vision models, so we need to bypass check if (!bypass_model_check) { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -910,7 +895,6 @@ cpp::result ModelService::StopModel( const std::string& model_handle) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { @@ -918,7 +902,7 @@ cpp::result ModelService::StopModel( bypass_stop_check_set_.end()); std::string engine_name = ""; if (!bypass_check) { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -958,11 +942,10 @@ cpp::result ModelService::GetModelStatus( const std::string& model_handle) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -1083,8 +1066,7 @@ cpp::result ModelService::GetModelPullInfo( auto default_model_branch = huggingface_utils::GetDefaultBranch(model_name); - cortex::db::Models modellist_handler; - auto downloaded_model_ids = modellist_handler.FindRelatedModel(model_name) + auto downloaded_model_ids = db_service_->FindRelatedModel(model_name) .value_or(std::vector{}); std::vector avai_download_opts{}; @@ -1128,8 +1110,8 @@ cpp::result, std::string> ModelService::MayFallbackToCpu(const std::string& model_path, int ngl, int ctx_len, int n_batch, int n_ubatch, const std::string& kv_cache_type) { - services::HardwareService hw_svc; - auto hw_info = hw_svc.GetHardwareInfo(); + assert(hw_service_); + auto hw_info = hw_service_->GetHardwareInfo(); assert(!!engine_svc_); auto default_engine = engine_svc_->GetDefaultEngineVariant(kLlamaEngine); bool is_cuda = false; diff --git a/engine/services/model_service.h b/engine/services/model_service.h index ab3596812..cc659fea5 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -6,12 +6,12 @@ #include "common/engine_servicei.h" #include "common/model_metadata.h" #include "config/model_config.h" +#include "services/database_service.h" #include "services/download_service.h" +#include "services/hardware_service.h" #include "utils/hardware/gguf/gguf_file_estimate.h" -namespace services { class InferenceService; -} struct ModelPullInfo { std::string id; @@ -31,14 +31,14 @@ class ModelService { public: void ForceIndexingModelList(); - explicit ModelService(std::shared_ptr download_service) - : download_service_{download_service} {}; - - explicit ModelService( - std::shared_ptr download_service, - std::shared_ptr inference_service, - std::shared_ptr engine_svc) - : download_service_{download_service}, + explicit ModelService(std::shared_ptr db_service, + std::shared_ptr hw_service, + std::shared_ptr download_service, + std::shared_ptr inference_service, + std::shared_ptr engine_svc) + : db_service_(db_service), + hw_service_(hw_service), + download_service_{download_service}, inference_svc_(inference_service), engine_svc_(engine_svc) {}; @@ -115,8 +115,10 @@ class ModelService { const std::string& model_path, int ngl, int ctx_len, int n_batch = 2048, int n_ubatch = 2048, const std::string& kv_cache_type = "f16"); + std::shared_ptr db_service_; + std::shared_ptr hw_service_; std::shared_ptr download_service_; - std::shared_ptr inference_svc_; + std::shared_ptr inference_svc_; std::unordered_set bypass_stop_check_set_; std::shared_ptr engine_svc_ = nullptr; diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc index a7d9d5e6e..7fc0ef5b2 100644 --- a/engine/services/model_source_service.cc +++ b/engine/services/model_source_service.cc @@ -9,7 +9,6 @@ #include "utils/string_utils.h" #include "utils/url_parser.h" -namespace services { namespace hu = huggingface_utils; namespace { @@ -61,10 +60,13 @@ std::vector ParseJsonString(const std::string& json_str) { } // namespace -ModelSourceService::ModelSourceService() { +ModelSourceService::ModelSourceService( + std::shared_ptr db_service) + : db_service_(db_service) { sync_db_thread_ = std::thread(&ModelSourceService::SyncModelSource, this); running_ = true; } + ModelSourceService::~ModelSourceService() { running_ = false; if (sync_db_thread_.joinable()) { @@ -106,8 +108,7 @@ cpp::result ModelSourceService::AddModelSource( cpp::result ModelSourceService::RemoveModelSource( const std::string& model_source) { - cortex::db::Models model_db; - auto srcs = model_db.GetModelSources(); + auto srcs = db_service_->GetModelSources(); if (srcs.has_error()) { return cpp::fail(srcs.error()); } else { @@ -127,13 +128,13 @@ cpp::result ModelSourceService::RemoveModelSource( } if (r.pathParams.size() == 1) { - if (auto del_res = model_db.DeleteModelEntryWithOrg(model_source); + if (auto del_res = db_service_->DeleteModelEntryWithOrg(model_source); del_res.has_error()) { CTL_INF(del_res.error()); return cpp::fail(del_res.error()); } } else { - if (auto del_res = model_db.DeleteModelEntryWithRepo(model_source); + if (auto del_res = db_service_->DeleteModelEntryWithRepo(model_source); del_res.has_error()) { CTL_INF(del_res.error()); return cpp::fail(del_res.error()); @@ -145,8 +146,7 @@ cpp::result ModelSourceService::RemoveModelSource( cpp::result, std::string> ModelSourceService::GetModelSources() { - cortex::db::Models model_db; - return model_db.GetModelSources(); + return db_service_->GetModelSources(); } cpp::result ModelSourceService::AddHfOrg( @@ -156,10 +156,9 @@ cpp::result ModelSourceService::AddHfOrg( if (res.has_value()) { auto models = ParseJsonString(res.value()); // Get models from db - cortex::db::Models model_db; - auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + auto model_list_before = db_service_->GetModels(model_source) + .value_or(std::vector{}); std::unordered_set updated_model_list; // Add new models for (auto const& m : models) { @@ -179,7 +178,7 @@ cpp::result ModelSourceService::AddHfOrg( // Clean up for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); + if (auto del_res = db_service_->DeleteModelEntry(mid); del_res.has_error()) { CTL_INF(del_res.error()); } @@ -195,10 +194,9 @@ cpp::result ModelSourceService::AddHfRepo( const std::string& model_source, const std::string& author, const std::string& model_name) { // Get models from db - cortex::db::Models model_db; auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + db_service_->GetModels(model_source).value_or(std::vector{}); std::unordered_set updated_model_list; auto add_res = AddRepoSiblings(model_source, author, model_name); if (add_res.has_error()) { @@ -208,7 +206,8 @@ cpp::result ModelSourceService::AddHfRepo( } for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); del_res.has_error()) { + if (auto del_res = db_service_->DeleteModelEntry(mid); + del_res.has_error()) { CTL_INF(del_res.error()); } } @@ -234,7 +233,6 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source, for (const auto& sibling : repo_info->siblings) { if (string_utils::EndsWith(sibling.rfilename, ".gguf")) { - cortex::db::Models model_db; std::string model_id = author + ":" + model_name + ":" + sibling.rfilename; cortex::db::ModelEntry e = { @@ -248,15 +246,15 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source, .status = cortex::db::ModelStatus::Downloadable, .engine = "llama-cpp", .metadata = repo_info->metadata}; - if (!model_db.HasModel(model_id)) { - if (auto add_res = model_db.AddModelEntry(e); add_res.has_error()) { + if (!db_service_->HasModel(model_id)) { + if (auto add_res = db_service_->AddModelEntry(e); add_res.has_error()) { CTL_INF(add_res.error()); } } else { - if (auto m = model_db.GetModelInfo(model_id); + if (auto m = db_service_->GetModelInfo(model_id); m.has_value() && m->status == cortex::db::ModelStatus::Downloadable) { - if (auto upd_res = model_db.UpdateModelEntry(model_id, e); + if (auto upd_res = db_service_->UpdateModelEntry(model_id, e); upd_res.has_error()) { CTL_INF(upd_res.error()); } @@ -276,10 +274,9 @@ cpp::result ModelSourceService::AddCortexsoOrg( if (res.has_value()) { auto models = ParseJsonString(res.value()); // Get models from db - cortex::db::Models model_db; - auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + auto model_list_before = db_service_->GetModels(model_source) + .value_or(std::vector{}); std::unordered_set updated_model_list; for (auto const& m : models) { CTL_INF(m.id); @@ -313,7 +310,7 @@ cpp::result ModelSourceService::AddCortexsoOrg( // Clean up for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); + if (auto del_res = db_service_->DeleteModelEntry(mid); del_res.has_error()) { CTL_INF(del_res.error()); } @@ -340,10 +337,9 @@ cpp::result ModelSourceService::AddCortexsoRepo( return cpp::fail(repo_info.error()); } // Get models from db - cortex::db::Models model_db; auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + db_service_->GetModels(model_source).value_or(std::vector{}); std::unordered_set updated_model_list; for (auto const& [branch, _] : branches.value()) { @@ -359,7 +355,8 @@ cpp::result ModelSourceService::AddCortexsoRepo( // Clean up for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); del_res.has_error()) { + if (auto del_res = db_service_->DeleteModelEntry(mid); + del_res.has_error()) { CTL_INF(del_res.error()); } } @@ -397,7 +394,6 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, CTL_INF("Only support gguf file format! - branch: " << branch); return {}; } else { - cortex::db::Models model_db; std::string model_id = model_name + ":" + branch; cortex::db::ModelEntry e = {.model = model_id, .author_repo_id = author, @@ -409,16 +405,16 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, .status = cortex::db::ModelStatus::Downloadable, .engine = "llama-cpp", .metadata = metadata}; - if (!model_db.HasModel(model_id)) { + if (!db_service_->HasModel(model_id)) { CTL_INF("Adding model to db: " << model_name << ":" << branch); - if (auto res = model_db.AddModelEntry(e); + if (auto res = db_service_->AddModelEntry(e); res.has_error() || !res.value()) { CTL_DBG("Cannot add model to db: " << model_id); } } else { - if (auto m = model_db.GetModelInfo(model_id); + if (auto m = db_service_->GetModelInfo(model_id); m.has_value() && m->status == cortex::db::ModelStatus::Downloadable) { - if (auto upd_res = model_db.UpdateModelEntry(model_id, e); + if (auto upd_res = db_service_->UpdateModelEntry(model_id, e); upd_res.has_error()) { CTL_INF(upd_res.error()); } @@ -444,8 +440,7 @@ void ModelSourceService::SyncModelSource() { CTL_DBG("Start to sync cortex.db"); start_time = current_time; - cortex::db::Models model_db; - auto res = model_db.GetModelSources(); + auto res = db_service_->GetModelSources(); if (res.has_error()) { CTL_INF(res.error()); } else { @@ -489,5 +484,3 @@ void ModelSourceService::SyncModelSource() { } } } - -} // namespace services \ No newline at end of file diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h index aa0b37259..7227267d3 100644 --- a/engine/services/model_source_service.h +++ b/engine/services/model_source_service.h @@ -2,14 +2,14 @@ #include #include #include +#include "services/database_service.h" #include "utils/result.hpp" -namespace services { class ModelSourceService { public: - explicit ModelSourceService(); + explicit ModelSourceService(std::shared_ptr db_service); ~ModelSourceService(); - + cpp::result AddModelSource( const std::string& model_source); @@ -22,9 +22,9 @@ class ModelSourceService { cpp::result AddHfOrg(const std::string& model_source, const std::string& author); - cpp::result AddHfRepo( - const std::string& model_source, const std::string& author, - const std::string& model_name); + cpp::result AddHfRepo(const std::string& model_source, + const std::string& author, + const std::string& model_name); cpp::result, std::string> AddRepoSiblings( const std::string& model_source, const std::string& author, @@ -41,13 +41,12 @@ class ModelSourceService { AddCortexsoRepoBranch(const std::string& model_source, const std::string& author, const std::string& model_name, - const std::string& branch, - const std::string& metadata); + const std::string& branch, const std::string& metadata); void SyncModelSource(); private: + std::shared_ptr db_service_ = nullptr; std::thread sync_db_thread_; std::atomic running_; -}; -} // namespace services \ No newline at end of file +}; \ No newline at end of file