From 37d64681381e647188215955c55f5dbf09f13380 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Wed, 1 Jan 2025 00:07:39 +0700 Subject: [PATCH 1/6] fix: download recursive --- engine/utils/curl_utils.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/engine/utils/curl_utils.cc b/engine/utils/curl_utils.cc index be82b5cfa..d5945e8c8 100644 --- a/engine/utils/curl_utils.cc +++ b/engine/utils/curl_utils.cc @@ -271,8 +271,12 @@ cpp::result SimpleGetJsonRecursive( if (root.isArray()) { for (const auto& value : root) { if (value["type"].asString() == "directory") { - auto temp = SimpleGetJsonRecursive(url + "/" + value["path"].asString(), - timeout); + auto temp = SimpleGetJsonRecursive( + url + "/" + + std::filesystem::path(value["path"].asString()) + .filename() + .string(), + timeout); if (!temp.has_error()) { if (temp.value().isArray()) { for (const auto& item : temp.value()) { From 9d7f4de890277236d1d1c4cb7a13050809f98127 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Wed, 1 Jan 2025 09:32:22 +0700 Subject: [PATCH 2/6] fix: handle model not loaded --- .../extensions/python-engine/python_engine.cc | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/engine/extensions/python-engine/python_engine.cc b/engine/extensions/python-engine/python_engine.cc index 9be369bcf..a62f6526b 100644 --- a/engine/extensions/python-engine/python_engine.cc +++ b/engine/extensions/python-engine/python_engine.cc @@ -531,6 +531,18 @@ void PythonEngine::HandleInference( std::string model = (*json_body)["model"].asString(); Json::Value body = (*json_body)["body"]; + if (models_.find(model) == models_.end()) { + Json::Value error; + error["error"] = "Model '" + model + "' is not loaded!"; + Json::Value status; + status["is_done"] = true; + status["has_error"] = true; + status["is_stream"] = false; + status["status_code"] = k400BadRequest; + callback(std::move(status), std::move(error)); + return; + } + // Transform Request std::string transformed_request; if (!transform_request.empty()) { @@ -699,6 +711,18 @@ void PythonEngine::HandleRouteRequest( std::string model = (*json_body)["model"].asString(); Json::Value body = (*json_body)["body"]; + if (models_.find(model) == models_.end()) { + Json::Value error; + error["error"] = "Model '" + model + "' is not loaded!"; + Json::Value status; + status["is_done"] = true; + status["has_error"] = true; + status["is_stream"] = false; + status["status_code"] = k400BadRequest; + callback(std::move(status), std::move(error)); + return; + } + // Transform Request std::string transformed_request; if (!transform_request.empty()) { From e5ff93517ac39530ffbace455566eb745057b5b8 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Wed, 1 Jan 2025 23:06:12 +0700 Subject: [PATCH 3/6] fix: set permission for all venv folder --- engine/services/model_service.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index c7925360b..f07b0359b 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -584,10 +584,10 @@ ModelService::DownloadModelFromCortexsoAsync( #ifdef _WIN32 set_permission_utils::SetExecutePermissionsRecursive( - venv_path / std::filesystem::path("Scripts")); + venv_path ); #else set_permission_utils::SetExecutePermissionsRecursive( - venv_path / std::filesystem::path("bin")); + venv_path ); #endif } else { From b169359c4b4fe1f731ddd06400cf294b8bc66ebc Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Thu, 2 Jan 2025 08:58:56 +0700 Subject: [PATCH 4/6] format code --- engine/services/model_service.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index f07b0359b..e1d436058 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -582,13 +582,8 @@ ModelService::DownloadModelFromCortexsoAsync( pyvenv_cfg.close(); // Add executable permission to python -#ifdef _WIN32 set_permission_utils::SetExecutePermissionsRecursive( venv_path ); -#else - set_permission_utils::SetExecutePermissionsRecursive( - venv_path ); -#endif } else { CTL_ERR("Failed to extract venv.zip"); From 29d14cef25352476841cce8bfde1bd809527af54 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Thu, 2 Jan 2025 12:47:22 +0700 Subject: [PATCH 5/6] fix: cortex stop will stop all running models --- engine/controllers/process_manager.cc | 13 +++++++++++-- engine/controllers/process_manager.h | 10 ++++++++++ engine/main.cc | 2 +- engine/services/model_service.cc | 3 +-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/engine/controllers/process_manager.cc b/engine/controllers/process_manager.cc index 8373f08fe..69fd74c0a 100644 --- a/engine/controllers/process_manager.cc +++ b/engine/controllers/process_manager.cc @@ -1,12 +1,21 @@ #include "process_manager.h" -#include "utils/cortex_utils.h" - #include #include +#include "json/json.h" +#include "utils/cortex_utils.h" void ProcessManager::destroy( const HttpRequestPtr& req, std::function&& callback) { + auto res = inference_service_->GetModels(std::make_shared()); + // Check if there are any running models + auto running_models = std::get<1>(res); + if (!running_models.get("data", Json::Value()).empty()) { + for (const auto& model : running_models["data"]) { + std::string model_id = model["id"].asString(); + model_service_->StopModel(model_id); + } + } app().quit(); Json::Value ret; diff --git a/engine/controllers/process_manager.h b/engine/controllers/process_manager.h index bded7b103..baef9415e 100644 --- a/engine/controllers/process_manager.h +++ b/engine/controllers/process_manager.h @@ -2,6 +2,8 @@ #include #include +#include "services/inference_service.h" +#include "services/model_service.h" using namespace drogon; @@ -13,4 +15,12 @@ class ProcessManager : public drogon::HttpController { void destroy(const HttpRequestPtr& req, std::function&& callback); + + ProcessManager(std::shared_ptr inference_service, + std::shared_ptr model_service) + : inference_service_(inference_service), model_service_(model_service) {} + + private: + std::shared_ptr inference_service_; + std::shared_ptr model_service_; }; diff --git a/engine/main.cc b/engine/main.cc index 77f51c7fa..b5ce6138d 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -182,7 +182,7 @@ void RunServer(std::optional host, std::optional port, auto model_ctl = std::make_shared(db_service, model_service, engine_service, model_src_svc); auto event_ctl = std::make_shared(event_queue_ptr); - auto pm_ctl = std::make_shared(); + auto pm_ctl = std::make_shared(inference_svc, model_service); auto hw_ctl = std::make_shared(engine_service, hw_service); auto server_ctl = std::make_shared(inference_svc, engine_service); diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index e1d436058..a3d8f8885 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -582,8 +582,7 @@ ModelService::DownloadModelFromCortexsoAsync( pyvenv_cfg.close(); // Add executable permission to python - set_permission_utils::SetExecutePermissionsRecursive( - venv_path ); + set_permission_utils::SetExecutePermissionsRecursive(venv_path); } else { CTL_ERR("Failed to extract venv.zip"); From 82a0ef044969a92cf140a6c8f6f0809b2101301f Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Mon, 6 Jan 2025 12:02:15 +0700 Subject: [PATCH 6/6] Fix: comment --- engine/controllers/process_manager.cc | 12 +++--------- engine/controllers/process_manager.h | 11 ++++------- engine/extensions/python-engine/python_engine.cc | 8 +++++--- engine/main.cc | 2 +- engine/services/model_service.cc | 4 ---- 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/engine/controllers/process_manager.cc b/engine/controllers/process_manager.cc index 69fd74c0a..9d1604754 100644 --- a/engine/controllers/process_manager.cc +++ b/engine/controllers/process_manager.cc @@ -7,16 +7,10 @@ void ProcessManager::destroy( const HttpRequestPtr& req, std::function&& callback) { - auto res = inference_service_->GetModels(std::make_shared()); - // Check if there are any running models - auto running_models = std::get<1>(res); - if (!running_models.get("data", Json::Value()).empty()) { - for (const auto& model : running_models["data"]) { - std::string model_id = model["id"].asString(); - model_service_->StopModel(model_id); - } + auto loaded_engines = engine_service_->GetSupportedEngineNames(); + for (const auto& engine : loaded_engines.value()) { + engine_service_->UnloadEngine(engine); } - app().quit(); Json::Value ret; ret["message"] = "Program is exitting, goodbye!"; diff --git a/engine/controllers/process_manager.h b/engine/controllers/process_manager.h index baef9415e..449e66d21 100644 --- a/engine/controllers/process_manager.h +++ b/engine/controllers/process_manager.h @@ -2,8 +2,7 @@ #include #include -#include "services/inference_service.h" -#include "services/model_service.h" +#include "services/engine_service.h" using namespace drogon; @@ -16,11 +15,9 @@ class ProcessManager : public drogon::HttpController { void destroy(const HttpRequestPtr& req, std::function&& callback); - ProcessManager(std::shared_ptr inference_service, - std::shared_ptr model_service) - : inference_service_(inference_service), model_service_(model_service) {} + ProcessManager(std::shared_ptr engine_service) + : engine_service_(engine_service) {} private: - std::shared_ptr inference_service_; - std::shared_ptr model_service_; + std::shared_ptr engine_service_; }; diff --git a/engine/extensions/python-engine/python_engine.cc b/engine/extensions/python-engine/python_engine.cc index a62f6526b..f9557d70b 100644 --- a/engine/extensions/python-engine/python_engine.cc +++ b/engine/extensions/python-engine/python_engine.cc @@ -18,7 +18,6 @@ static size_t WriteCallback(char* ptr, size_t size, size_t nmemb, PythonEngine::PythonEngine() : q_(4 /*n_parallel*/, "python_engine") {} - PythonEngine::~PythonEngine() { curl_global_cleanup(); } @@ -507,7 +506,6 @@ CurlResponse PythonEngine::MakeStreamPostRequest( return response; } - void PythonEngine::HandleInference( std::shared_ptr json_body, std::function&& callback) { @@ -943,7 +941,11 @@ void PythonEngine::Load(EngineLoadOption opts) { // Develop register model here on loading engine }; -void PythonEngine::Unload(EngineUnloadOption opts) {}; +void PythonEngine::Unload(EngineUnloadOption opts) { + for (const auto& pair : models_) { + TerminateModelProcess(pair.first); + } +}; // extern "C" { // EngineI* get_engine() { diff --git a/engine/main.cc b/engine/main.cc index b5ce6138d..59ec49873 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -182,7 +182,7 @@ void RunServer(std::optional host, std::optional port, auto model_ctl = std::make_shared(db_service, model_service, engine_service, model_src_svc); auto event_ctl = std::make_shared(event_queue_ptr); - auto pm_ctl = std::make_shared(inference_svc, model_service); + auto pm_ctl = std::make_shared(engine_service); auto hw_ctl = std::make_shared(engine_service, hw_service); auto server_ctl = std::make_shared(inference_svc, engine_service); diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index a3d8f8885..74767a9b2 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -567,7 +567,6 @@ ModelService::DownloadModelFromCortexsoAsync( std::filesystem::path("python.exe")) .string() << std::endl; - #else pyvenv_cfg << "home = " << (venv_path / std::filesystem::path("bin/")).string() @@ -577,13 +576,10 @@ ModelService::DownloadModelFromCortexsoAsync( << (venv_path / std::filesystem::path("bin/python")).string() << std::endl; #endif - // Close the file pyvenv_cfg.close(); // Add executable permission to python - set_permission_utils::SetExecutePermissionsRecursive(venv_path); - } else { CTL_ERR("Failed to extract venv.zip"); };