diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 2ac1947d4..2735501dc 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -95,6 +95,7 @@ find_package(nlohmann_json CONFIG REQUIRED) find_package(CLI11 CONFIG REQUIRED) find_package(unofficial-minizip CONFIG REQUIRED) find_package(LibArchive REQUIRED) +find_package(tabulate CONFIG REQUIRED) # Build using CMAKE-JS if(DEFINED CMAKE_JS_INC) @@ -136,6 +137,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE jinja2cpp) target_link_libraries(${PROJECT_NAME} PRIVATE CLI11::CLI11) target_link_libraries(${PROJECT_NAME} PRIVATE unofficial::minizip::minizip) target_link_libraries(${PROJECT_NAME} PRIVATE LibArchive::LibArchive) +target_link_libraries(${PROJECT_NAME} PRIVATE tabulate::tabulate) # Build using CMAKE-JS if(DEFINED CMAKE_JS_INC) diff --git a/engine/commands/engine_list_cmd.cc b/engine/commands/engine_list_cmd.cc new file mode 100644 index 000000000..d76d885d7 --- /dev/null +++ b/engine/commands/engine_list_cmd.cc @@ -0,0 +1,81 @@ +// clang-format off +#include "utils/cortex_utils.h" +// clang-format on +#include "engine_list_cmd.h" +#include +#include +#include +#include "trantor/utils/Logger.h" + +namespace commands { + +bool EngineListCmd::Exec() { + tabulate::Table table; + table.add_row( + {"(Index)", "name", "description", "version", "product name", "status"}); + table.format().font_color(tabulate::Color::green); +#ifdef _WIN32 + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kOnnxLibPath)) { + table.add_row({"1", "cortex.onnx", + "This extension enables chat completion API calls using the " + "Onnx engine", + "0.0.1", "Onnx Inference Engine", "ready"}); + } else { + table.add_row({"1", "cortex.onnx", + "This extension enables chat completion API calls using the " + "Onnx engine", + "0.0.1", "Onnx Inference Engine", "not_initialized"}); + } + +#else + table.add_row( + {"1", "cortex.onnx", + "This extension enables chat completion API calls using the Onnx engine", + "0.0.1", "Onnx Inference Engine", "not_supported"}); +#endif + // lllamacpp + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kLlamaLibPath)) { + table.add_row({"2", "cortex.llamacpp", + "This extension enables chat completion API calls using the " + "LlamaCPP engine", + "0.0.1", "LlamaCPP Inference Engine", "ready"}); + } else { + table.add_row({"2", "cortex.llamacpp", + "This extension enables chat completion API calls using the " + "LlamaCPP engine", + "0.0.1", "LlamaCPP Inference Engine", "not_initialized"}); + } + // tensorrt llm + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kTensorrtLlmPath)) { + table.add_row({"3", "cortex.tensorrt-llm", + "This extension enables chat completion API calls using the " + "TensorrtLLM engine", + "0.0.1", "TensorrtLLM Inference Engine", "ready"}); + } else { + table.add_row({"3", "cortex.tensorrt-llm", + "This extension enables chat completion API calls using the " + "TensorrtLLM engine", + "0.0.1", "TensorrtLLM Inference Engine", "not_initialized"}); + } + for (int i = 0; i < 6; i++) { + table[0][i] + .format() + .font_color(tabulate::Color::white) // Set font color + .font_style({tabulate::FontStyle::bold}) + .font_align(tabulate::FontAlign::center); + } + for (int i = 1; i < 4; i++) { + table[i][0] + .format() + .font_color(tabulate::Color::white) // Set font color + .font_align(tabulate::FontAlign::center); + } + + std::cout << table << std::endl; + return true; +} + +}; // namespace commands diff --git a/engine/commands/engine_list_cmd.h b/engine/commands/engine_list_cmd.h new file mode 100644 index 000000000..2193572b7 --- /dev/null +++ b/engine/commands/engine_list_cmd.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace commands { +class EngineListCmd { + public: + bool Exec() ; +}; + +} // namespace commands \ No newline at end of file diff --git a/engine/commands/model_list_cmd.cc b/engine/commands/model_list_cmd.cc index 2aedd6d24..199ca25f8 100644 --- a/engine/commands/model_list_cmd.cc +++ b/engine/commands/model_list_cmd.cc @@ -1,8 +1,11 @@ +// clang-format off +#include "utils/cortex_utils.h" +// clang-format on #include "model_list_cmd.h" #include #include +#include #include -#include "utils/cortex_utils.h" #include "config/yaml_config.h" #include "trantor/utils/Logger.h" namespace commands { @@ -10,21 +13,45 @@ namespace commands { void ModelListCmd::Exec() { if (std::filesystem::exists(cortex_utils::models_folder) && std::filesystem::is_directory(cortex_utils::models_folder)) { + tabulate::Table table; + + table.add_row({"(Index)", "ID", "engine", "version"}); + table.format().font_color(tabulate::Color::green); + int count = 0; // Iterate through directory for (const auto& entry : std::filesystem::directory_iterator(cortex_utils::models_folder)) { if (entry.is_regular_file() && entry.path().extension() == ".yaml") { try { - config::YamlHandler handler; - handler.ModelConfigFromFile(entry.path().string()); - std::cout<<"Model ID: "<< entry.path().stem().string() <<", Engine: "<< handler.GetModelConfig().engine <add_subcommand("list", "List all cortex engines"); + list_engines_cmd->callback([]() { + commands::EngineListCmd command; + command.Exec(); + }); + auto get_engine_cmd = engines_cmd->add_subcommand("get", "Get an engine"); EngineInstall(engines_cmd, "cortex.llamacpp", version); diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index b10a6b758..9668be441 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -68,7 +68,8 @@ void Engines::InitEngine(const HttpRequestPtr& req, }}}; DownloadService().AddAsyncDownloadTask( - downloadTask, [](const std::string& absolute_path, bool unused) { + downloadTask, + [](const std::string& absolute_path, bool unused) { // try to unzip the downloaded file std::filesystem::path downloadedEnginePath{absolute_path}; LOG_INFO << "Downloaded engine path: " @@ -108,4 +109,67 @@ void Engines::InitEngine(const HttpRequestPtr& req, auto err = res.error(); LOG_ERROR << "HTTP error: " << httplib::to_string(err); } +} + +void Engines::ListEngine( + const HttpRequestPtr& req, + std::function&& callback) const { + Json::Value ret; + ret["object"] = "list"; + Json::Value data(Json::arrayValue); + Json::Value obj_onnx, obj_llamacpp, obj_tensorrt; + obj_onnx["name"] = "cortex.onnx"; + obj_onnx["description"] = + "This extension enables chat completion API calls using the Onnx engine"; + obj_onnx["version"] = "0.0.1"; + obj_onnx["productName"] = "Onnx Inference Engine"; + + obj_llamacpp["name"] = "cortex.llamacpp"; + obj_llamacpp["description"] = + "This extension enables chat completion API calls using the LlamaCPP " + "engine"; + obj_llamacpp["version"] = "0.0.1"; + obj_llamacpp["productName"] = "LlamaCPP Inference Engine"; + + obj_tensorrt["name"] = "cortex.tensorrt-llm"; + obj_tensorrt["description"] = + "This extension enables chat completion API calls using the TensorrtLLM " + "engine"; + obj_tensorrt["version"] = "0.0.1"; + obj_tensorrt["productName"] = "TensorrtLLM Inference Engine"; + +#ifdef _WIN32 + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kOnnxLibPath)) { + obj_onnx["status"] = "ready"; + } else { + obj_onnx["status"] = "not_initialized"; + } +#else + obj_onnx["status"] = "not_supported"; +#endif + // lllamacpp + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kLlamaLibPath)) { + + obj_llamacpp["status"] = "ready"; + } else { + obj_llamacpp["status"] = "not_initialized"; + } + // tensorrt llm + if (std::filesystem::exists(std::filesystem::current_path().string() + + cortex_utils::kTensorrtLlmPath)) { + obj_tensorrt["status"] = "ready"; + } else { + obj_tensorrt["status"] = "not_initialized"; + } + + data.append(std::move(obj_onnx)); + data.append(std::move(obj_llamacpp)); + data.append(std::move(obj_tensorrt)); + ret["data"] = data; + ret["result"] = "OK"; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k200OK); + callback(resp); } \ No newline at end of file diff --git a/engine/controllers/engines.h b/engine/controllers/engines.h index 282e79402..ccc526efe 100644 --- a/engine/controllers/engines.h +++ b/engine/controllers/engines.h @@ -13,9 +13,12 @@ class Engines : public drogon::HttpController { public: METHOD_LIST_BEGIN METHOD_ADD(Engines::InitEngine, "/{1}/init", Post); + METHOD_ADD(Engines::ListEngine, "/list", Get); METHOD_LIST_END void InitEngine(const HttpRequestPtr& req, std::function&& callback, const std::string& engine) const; + void ListEngine(const HttpRequestPtr& req, + std::function&& callback) const; }; diff --git a/engine/vcpkg.json b/engine/vcpkg.json index 82118eb17..74f89965a 100644 --- a/engine/vcpkg.json +++ b/engine/vcpkg.json @@ -13,6 +13,7 @@ "minizip", "nlohmann-json", "yaml-cpp", - "libarchive" + "libarchive", + "tabulate" ] } \ No newline at end of file