From f503f5918403f6ff2ef4f63dc3a3caded45959c1 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 29 Aug 2024 19:58:16 +0700 Subject: [PATCH 1/3] add engine list cmd --- engine/CMakeLists.txt | 2 + engine/commands/engine_list_cmd.cc | 78 +++++++++++++++++++++++ engine/commands/engine_list_cmd.h | 11 ++++ engine/commands/model_list_cmd.cc | 35 ++++++++-- engine/controllers/command_line_parser.cc | 6 ++ engine/vcpkg.json | 3 +- 6 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 engine/commands/engine_list_cmd.cc create mode 100644 engine/commands/engine_list_cmd.h diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 2ac1947d4..2735501dc 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -95,6 +95,7 @@ find_package(nlohmann_json CONFIG REQUIRED) find_package(CLI11 CONFIG REQUIRED) find_package(unofficial-minizip CONFIG REQUIRED) find_package(LibArchive REQUIRED) +find_package(tabulate CONFIG REQUIRED) # Build using CMAKE-JS if(DEFINED CMAKE_JS_INC) @@ -136,6 +137,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE jinja2cpp) target_link_libraries(${PROJECT_NAME} PRIVATE CLI11::CLI11) target_link_libraries(${PROJECT_NAME} PRIVATE unofficial::minizip::minizip) target_link_libraries(${PROJECT_NAME} PRIVATE LibArchive::LibArchive) +target_link_libraries(${PROJECT_NAME} PRIVATE tabulate::tabulate) # Build using CMAKE-JS if(DEFINED CMAKE_JS_INC) diff --git a/engine/commands/engine_list_cmd.cc b/engine/commands/engine_list_cmd.cc new file mode 100644 index 000000000..403c020e2 --- /dev/null +++ b/engine/commands/engine_list_cmd.cc @@ -0,0 +1,78 @@ +#include "engine_list_cmd.h" +#include +#include +#include +#include "trantor/utils/Logger.h" +#include "utils/cortex_utils.h" + +namespace commands { + +bool EngineListCmd::Exec() { + tabulate::Table table; + table.add_row( + {"(Index)", "name", "description", "version", "product name", "status"}); + table.format().font_color(tabulate::Color::green); +#ifdef _WIN32 + table.add_row( + {"1", "cortex.onnx", + "This extension enables chat completion API calls using the Onnx engine", + "0.0.1", "Onnx Inference Engine", "not_supported"}); +#else + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kOnnxLibPath)) { + table.add_row({"1", "cortex.onnx", + "This extension enables chat completion API calls using the " + "Onnx engine", + "0.0.1", "Onnx Inference Engine", "ready"}); + } else { + table.add_row({"1", "cortex.onnx", + "This extension enables chat completion API calls using the " + "Onnx engine", + "0.0.1", "Onnx Inference Engine", "not_initialized"}); + } +#endif + // lllamacpp + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kLlamaLibPath)) { + table.add_row({"2", "cortex.llamacpp", + "This extension enables chat completion API calls using the " + "LlamaCPP engine", + "0.0.1", "LlamaCPP Inference Engine", "ready"}); + } else { + table.add_row({"2", "cortex.llamacpp", + "This extension enables chat completion API calls using the " + "LlamaCPP engine", + "0.0.1", "LlamaCPP Inference Engine", "not_initialized"}); + } + // tensorrt llm + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kTensorrtLlmPath)) { + table.add_row({"3", "cortex.tensorrt-llm", + "This extension enables chat completion API calls using the " + "TensorrtLLM engine", + "0.0.1", "TensorrtLLM Inference Engine", "ready"}); + } else { + table.add_row({"3", "cortex.tensorrt-llm", + "This extension enables chat completion API calls using the " + "TensorrtLLM engine", + "0.0.1", "TensorrtLLM Inference Engine", "not_initialized"}); + } + for (int i = 0; i < 6; i++) { + table[0][i] + .format() + .font_color(tabulate::Color::white) // Set font color + .font_style({tabulate::FontStyle::bold}) + .font_align(tabulate::FontAlign::center); + } + for (int i = 1; i < 4; i++) { + table[i][0] + .format() + .font_color(tabulate::Color::white) // Set font color + .font_align(tabulate::FontAlign::center); + } + + std::cout << table << std::endl; + return true; +} + +}; // namespace commands \ No newline at end of file diff --git a/engine/commands/engine_list_cmd.h b/engine/commands/engine_list_cmd.h new file mode 100644 index 000000000..2193572b7 --- /dev/null +++ b/engine/commands/engine_list_cmd.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace commands { +class EngineListCmd { + public: + bool Exec() ; +}; + +} // namespace commands \ No newline at end of file diff --git a/engine/commands/model_list_cmd.cc b/engine/commands/model_list_cmd.cc index 2aedd6d24..952467d9a 100644 --- a/engine/commands/model_list_cmd.cc +++ b/engine/commands/model_list_cmd.cc @@ -1,30 +1,55 @@ #include "model_list_cmd.h" #include #include +#include #include -#include "utils/cortex_utils.h" #include "config/yaml_config.h" #include "trantor/utils/Logger.h" +#include "utils/cortex_utils.h" namespace commands { void ModelListCmd::Exec() { if (std::filesystem::exists(cortex_utils::models_folder) && std::filesystem::is_directory(cortex_utils::models_folder)) { + tabulate::Table table; + + table.add_row({"(Index)", "ID", "engine", "version"}); + table.format().font_color(tabulate::Color::green); + int count = 0; // Iterate through directory for (const auto& entry : std::filesystem::directory_iterator(cortex_utils::models_folder)) { if (entry.is_regular_file() && entry.path().extension() == ".yaml") { try { - config::YamlHandler handler; - handler.ModelConfigFromFile(entry.path().string()); - std::cout<<"Model ID: "<< entry.path().stem().string() <<", Engine: "<< handler.GetModelConfig().engine <add_subcommand("list", "List all cortex engines"); + list_engines_cmd->callback([]() { + commands::EngineListCmd command; + command.Exec(); + }); + auto get_engine_cmd = engines_cmd->add_subcommand("get", "Get an engine"); EngineInstall(engines_cmd, "cortex.llamacpp", version); diff --git a/engine/vcpkg.json b/engine/vcpkg.json index 82118eb17..74f89965a 100644 --- a/engine/vcpkg.json +++ b/engine/vcpkg.json @@ -13,6 +13,7 @@ "minizip", "nlohmann-json", "yaml-cpp", - "libarchive" + "libarchive", + "tabulate" ] } \ No newline at end of file From feaf1a327631497460e5e88a78a6c7bbe94440a9 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 29 Aug 2024 21:15:27 +0700 Subject: [PATCH 2/3] add list engines service --- engine/commands/engine_list_cmd.cc | 14 ++++--- engine/controllers/engines.cc | 66 +++++++++++++++++++++++++++++- engine/controllers/engines.h | 3 ++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/engine/commands/engine_list_cmd.cc b/engine/commands/engine_list_cmd.cc index 403c020e2..4ab556853 100644 --- a/engine/commands/engine_list_cmd.cc +++ b/engine/commands/engine_list_cmd.cc @@ -1,10 +1,11 @@ #include "engine_list_cmd.h" #include +// clang-format off #include #include #include "trantor/utils/Logger.h" #include "utils/cortex_utils.h" - +// clang-format on namespace commands { bool EngineListCmd::Exec() { @@ -13,11 +14,6 @@ bool EngineListCmd::Exec() { {"(Index)", "name", "description", "version", "product name", "status"}); table.format().font_color(tabulate::Color::green); #ifdef _WIN32 - table.add_row( - {"1", "cortex.onnx", - "This extension enables chat completion API calls using the Onnx engine", - "0.0.1", "Onnx Inference Engine", "not_supported"}); -#else if (std::filesystem::exists(std::filesystem::current_path().string() + cortex_utils::kOnnxLibPath)) { table.add_row({"1", "cortex.onnx", @@ -30,6 +26,12 @@ bool EngineListCmd::Exec() { "Onnx engine", "0.0.1", "Onnx Inference Engine", "not_initialized"}); } + +#else + table.add_row( + {"1", "cortex.onnx", + "This extension enables chat completion API calls using the Onnx engine", + "0.0.1", "Onnx Inference Engine", "not_supported"}); #endif // lllamacpp if (std::filesystem::exists(std::filesystem::current_path().string() + diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index b10a6b758..9668be441 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -68,7 +68,8 @@ void Engines::InitEngine(const HttpRequestPtr& req, }}}; DownloadService().AddAsyncDownloadTask( - downloadTask, [](const std::string& absolute_path, bool unused) { + downloadTask, + [](const std::string& absolute_path, bool unused) { // try to unzip the downloaded file std::filesystem::path downloadedEnginePath{absolute_path}; LOG_INFO << "Downloaded engine path: " @@ -108,4 +109,67 @@ void Engines::InitEngine(const HttpRequestPtr& req, auto err = res.error(); LOG_ERROR << "HTTP error: " << httplib::to_string(err); } +} + +void Engines::ListEngine( + const HttpRequestPtr& req, + std::function&& callback) const { + Json::Value ret; + ret["object"] = "list"; + Json::Value data(Json::arrayValue); + Json::Value obj_onnx, obj_llamacpp, obj_tensorrt; + obj_onnx["name"] = "cortex.onnx"; + obj_onnx["description"] = + "This extension enables chat completion API calls using the Onnx engine"; + obj_onnx["version"] = "0.0.1"; + obj_onnx["productName"] = "Onnx Inference Engine"; + + obj_llamacpp["name"] = "cortex.llamacpp"; + obj_llamacpp["description"] = + "This extension enables chat completion API calls using the LlamaCPP " + "engine"; + obj_llamacpp["version"] = "0.0.1"; + obj_llamacpp["productName"] = "LlamaCPP Inference Engine"; + + obj_tensorrt["name"] = "cortex.tensorrt-llm"; + obj_tensorrt["description"] = + "This extension enables chat completion API calls using the TensorrtLLM " + "engine"; + obj_tensorrt["version"] = "0.0.1"; + obj_tensorrt["productName"] = "TensorrtLLM Inference Engine"; + +#ifdef _WIN32 + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kOnnxLibPath)) { + obj_onnx["status"] = "ready"; + } else { + obj_onnx["status"] = "not_initialized"; + } +#else + obj_onnx["status"] = "not_supported"; +#endif + // lllamacpp + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kLlamaLibPath)) { + + obj_llamacpp["status"] = "ready"; + } else { + obj_llamacpp["status"] = "not_initialized"; + } + // tensorrt llm + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kTensorrtLlmPath)) { + obj_tensorrt["status"] = "ready"; + } else { + obj_tensorrt["status"] = "not_initialized"; + } + + data.append(std::move(obj_onnx)); + data.append(std::move(obj_llamacpp)); + data.append(std::move(obj_tensorrt)); + ret["data"] = data; + ret["result"] = "OK"; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k200OK); + callback(resp); } \ No newline at end of file diff --git a/engine/controllers/engines.h b/engine/controllers/engines.h index 282e79402..ccc526efe 100644 --- a/engine/controllers/engines.h +++ b/engine/controllers/engines.h @@ -13,9 +13,12 @@ class Engines : public drogon::HttpController { public: METHOD_LIST_BEGIN METHOD_ADD(Engines::InitEngine, "/{1}/init", Post); + METHOD_ADD(Engines::ListEngine, "/list", Get); METHOD_LIST_END void InitEngine(const HttpRequestPtr& req, std::function&& callback, const std::string& engine) const; + void ListEngine(const HttpRequestPtr& req, + std::function&& callback) const; }; From d33ae7d9052dc358964dec1abbc6943db6e3005b Mon Sep 17 00:00:00 2001 From: CameronNg Date: Thu, 29 Aug 2024 23:58:21 +0700 Subject: [PATCH 3/3] fix CI build window --- engine/commands/engine_list_cmd.cc | 9 +++++---- engine/commands/model_list_cmd.cc | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/engine/commands/engine_list_cmd.cc b/engine/commands/engine_list_cmd.cc index 4ab556853..d76d885d7 100644 --- a/engine/commands/engine_list_cmd.cc +++ b/engine/commands/engine_list_cmd.cc @@ -1,11 +1,12 @@ +// clang-format off +#include "utils/cortex_utils.h" +// clang-format on #include "engine_list_cmd.h" #include -// clang-format off #include #include #include "trantor/utils/Logger.h" -#include "utils/cortex_utils.h" -// clang-format on + namespace commands { bool EngineListCmd::Exec() { @@ -77,4 +78,4 @@ bool EngineListCmd::Exec() { return true; } -}; // namespace commands \ No newline at end of file +}; // namespace commands diff --git a/engine/commands/model_list_cmd.cc b/engine/commands/model_list_cmd.cc index 952467d9a..199ca25f8 100644 --- a/engine/commands/model_list_cmd.cc +++ b/engine/commands/model_list_cmd.cc @@ -1,3 +1,6 @@ +// clang-format off +#include "utils/cortex_utils.h" +// clang-format on #include "model_list_cmd.h" #include #include @@ -5,7 +8,6 @@ #include #include "config/yaml_config.h" #include "trantor/utils/Logger.h" -#include "utils/cortex_utils.h" namespace commands { void ModelListCmd::Exec() { @@ -41,14 +43,14 @@ void ModelListCmd::Exec() { .font_align(tabulate::FontAlign::center); } for (int i = 1; i <= count; i++) { - table[i][0] //index value - .format() - .font_color(tabulate::Color::white) // Set font color - .font_align(tabulate::FontAlign::center); - table[i][3] //version value - .format() - .font_align(tabulate::FontAlign::center); - } + table[i][0] //index value + .format() + .font_color(tabulate::Color::white) // Set font color + .font_align(tabulate::FontAlign::center); + table[i][3] //version value + .format() + .font_align(tabulate::FontAlign::center); + } std::cout << table << std::endl; } }