From 8a70e7c3da7d3c6145b59ec07f34f8f3616831e3 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 12:12:02 +0700 Subject: [PATCH 1/3] fix: bypass check model id if llama_model_path exists --- engine/controllers/models.cc | 8 +++++++- engine/services/model_service.h | 11 +++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 796f70d16..087401747 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -442,6 +442,12 @@ void Models::StartModel( // model_path has higher priority if (auto& o = (*(req->getJsonObject()))["llama_model_path"]; !o.isNull()) { params_override.model_path = o.asString(); + if (auto& mp = (*(req->getJsonObject()))["model_path"]; !mp.isNull()) { + if (std::filesystem::exists(params_override.model_path.value())) { + CTL_INF("llama_model_path exists, bypass check model id"); + params_override.bypass_llama_model_path = true; + } + } } if (auto& o = (*(req->getJsonObject()))["model_path"]; !o.isNull()) { @@ -489,7 +495,7 @@ void Models::StartModel( auto& v = result.value(); Json::Value ret; ret["message"] = "Started successfully!"; - if(v.warning) { + if (v.warning) { ret["warning"] = *(v.warning); } auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); diff --git a/engine/services/model_service.h b/engine/services/model_service.h index 47d61c154..66b3198c8 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -3,10 +3,10 @@ #include #include #include +#include "common/engine_servicei.h" #include "config/model_config.h" #include "services/download_service.h" #include "services/inference_service.h" -#include "common/engine_servicei.h" struct ModelPullInfo { std::string id; @@ -26,12 +26,15 @@ struct StartParameterOverride { std::optional cache_type; std::optional mmproj; std::optional model_path; - bool bypass_model_check() const { return mmproj.has_value(); } + bool bypass_llama_model_path = false; + bool bypass_model_check() const { + return mmproj.has_value() || bypass_llama_model_path; + } }; struct StartModelResult { - bool success; - std::optional warning; + bool success; + std::optional warning; }; class ModelService { From 3491b1c60a1eabe58849a572a5ce883782574584 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 12:15:06 +0700 Subject: [PATCH 2/3] fix: correct logic --- engine/controllers/models.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 087401747..edf768c67 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -442,7 +442,7 @@ void Models::StartModel( // model_path has higher priority if (auto& o = (*(req->getJsonObject()))["llama_model_path"]; !o.isNull()) { params_override.model_path = o.asString(); - if (auto& mp = (*(req->getJsonObject()))["model_path"]; !mp.isNull()) { + if (auto& mp = (*(req->getJsonObject()))["model_path"]; mp.isNull()) { if (std::filesystem::exists(params_override.model_path.value())) { CTL_INF("llama_model_path exists, bypass check model id"); params_override.bypass_llama_model_path = true; From 31a5025be8085d6b24928da2b16a9d93a96f1841 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 12:59:01 +0700 Subject: [PATCH 3/3] fix: check db first --- engine/controllers/models.cc | 4 +++- engine/services/model_service.cc | 7 ++++++- engine/services/model_service.h | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index edf768c67..9e4ba1e9f 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -443,7 +443,9 @@ void Models::StartModel( if (auto& o = (*(req->getJsonObject()))["llama_model_path"]; !o.isNull()) { params_override.model_path = o.asString(); if (auto& mp = (*(req->getJsonObject()))["model_path"]; mp.isNull()) { - if (std::filesystem::exists(params_override.model_path.value())) { + // Bypass if model does not exist in DB and llama_model_path exists + if (std::filesystem::exists(params_override.model_path.value()) && + !model_service_->HasModel(model_handle)) { CTL_INF("llama_model_path exists, bypass check model id"); params_override.bypass_llama_model_path = true; } diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index 3a8507c22..793e8ecb5 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -381,6 +381,10 @@ cpp::result ModelService::HandleUrl( return unique_model_id; } +bool ModelService::HasModel(const std::string& id) const { + return cortex::db::Models().HasModel(id); +} + cpp::result ModelService::DownloadModelFromCortexsoAsync( const std::string& name, const std::string& branch, @@ -745,7 +749,8 @@ cpp::result ModelService::StartModel( return cpp::fail( "Not enough VRAM - required: " + std::to_string(vram_needed_MiB) + " MiB, available: " + std::to_string(free_vram_MiB) + - " MiB - Should adjust ngl to " + std::to_string(free_vram_MiB / (vram_needed_MiB / ngl) - 1)); + " MiB - Should adjust ngl to " + + std::to_string(free_vram_MiB / (vram_needed_MiB / ngl) - 1)); } if (ram_needed_MiB > free_ram_MiB) { diff --git a/engine/services/model_service.h b/engine/services/model_service.h index 66b3198c8..7b6375e54 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -92,6 +92,8 @@ class ModelService { const std::string& url, std::optional temp_model_id, std::optional temp_name); + bool HasModel(const std::string& id) const; + private: /** * Handle downloading model which have following pattern: author/model_name