From ea16986c90bc67a8b77dbb6e36958d8a9a9dad5a Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 09:06:28 +0700 Subject: [PATCH 1/6] feat: API for configuring huggingface token --- engine/common/api_server_configuration.h | 26 ++++++++++++++++++++++-- engine/services/config_service.cc | 18 ++++++++-------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/engine/common/api_server_configuration.h b/engine/common/api_server_configuration.h index 5bfcbbdc5..03b3022a4 100644 --- a/engine/common/api_server_configuration.h +++ b/engine/common/api_server_configuration.h @@ -90,6 +90,13 @@ static const std::unordered_map .group = "Proxy", .accept_value = "[on|off]", .default_value = "on"}}, + {"huggingface_token", + ApiConfigurationMetadata{.name = "huggingface_token", + .desc = "HuggingFace token to pull models", + .group = "Token", + .accept_value = "string", + .default_value = "", + .allow_empty = true}}, }; class ApiServerConfiguration { @@ -99,7 +106,8 @@ class ApiServerConfiguration { bool verify_proxy_ssl = true, bool verify_proxy_host_ssl = true, const std::string& proxy_url = "", const std::string& proxy_username = "", const std::string& proxy_password = "", const std::string& no_proxy = "", - bool verify_peer_ssl = true, bool verify_host_ssl = true) + bool verify_peer_ssl = true, bool verify_host_ssl = true, + const std::string& hf_token = "") : cors{cors}, allowed_origins{allowed_origins}, verify_proxy_ssl{verify_proxy_ssl}, @@ -109,7 +117,8 @@ class ApiServerConfiguration { proxy_password{proxy_password}, no_proxy{no_proxy}, verify_peer_ssl{verify_peer_ssl}, - verify_host_ssl{verify_host_ssl} {} + verify_host_ssl{verify_host_ssl}, + hf_token{hf_token} {} // cors bool cors{true}; @@ -127,6 +136,9 @@ class ApiServerConfiguration { bool verify_peer_ssl{true}; bool verify_host_ssl{true}; + // token + std::string hf_token{""}; + Json::Value ToJson() const { Json::Value root; root["cors"] = cors; @@ -142,6 +154,7 @@ class ApiServerConfiguration { root["no_proxy"] = no_proxy; root["verify_peer_ssl"] = verify_peer_ssl; root["verify_host_ssl"] = verify_host_ssl; + root["huggingface_token"] = hf_token; return root; } @@ -225,6 +238,15 @@ class ApiServerConfiguration { return true; }}, + {"huggingface_token", + [this](const Json::Value& value) -> bool { + if (!value.isString()) { + return false; + } + hf_token = value.asString(); + return true; + }}, + {"cors", [this](const Json::Value& value) -> bool { if (!value.isBool()) { diff --git a/engine/services/config_service.cc b/engine/services/config_service.cc index 9c794bb38..ce5526090 100644 --- a/engine/services/config_service.cc +++ b/engine/services/config_service.cc @@ -6,10 +6,10 @@ cpp::result ConfigService::UpdateApiServerConfiguration(const Json::Value& json) { auto config = file_manager_utils::GetCortexConfig(); ApiServerConfiguration api_server_config{ - config.enableCors, config.allowedOrigins, config.verifyProxySsl, - config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername, - config.proxyPassword, config.noProxy, config.verifyPeerSsl, - config.verifyHostSsl}; + config.enableCors, config.allowedOrigins, config.verifyProxySsl, + config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername, + config.proxyPassword, config.noProxy, config.verifyPeerSsl, + config.verifyHostSsl, config.huggingFaceToken}; std::vector updated_fields; std::vector invalid_fields; @@ -35,6 +35,8 @@ ConfigService::UpdateApiServerConfiguration(const Json::Value& json) { config.verifyPeerSsl = api_server_config.verify_peer_ssl; config.verifyHostSsl = api_server_config.verify_host_ssl; + config.huggingFaceToken = api_server_config.hf_token; + auto result = file_manager_utils::UpdateCortexConfig(config); return api_server_config; } @@ -43,8 +45,8 @@ cpp::result ConfigService::GetApiServerConfiguration() { auto config = file_manager_utils::GetCortexConfig(); return ApiServerConfiguration{ - config.enableCors, config.allowedOrigins, config.verifyProxySsl, - config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername, - config.proxyPassword, config.noProxy, config.verifyPeerSsl, - config.verifyHostSsl}; + config.enableCors, config.allowedOrigins, config.verifyProxySsl, + config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername, + config.proxyPassword, config.noProxy, config.verifyPeerSsl, + config.verifyHostSsl, config.huggingFaceToken}; } From 546553dc737f79c3c74bd82edeb01fd211c60483 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 09:08:59 +0700 Subject: [PATCH 2/6] chore: API docs --- docs/static/openapi/cortex.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/static/openapi/cortex.json b/docs/static/openapi/cortex.json index 9747a1830..1ac69d78e 100644 --- a/docs/static/openapi/cortex.json +++ b/docs/static/openapi/cortex.json @@ -1810,6 +1810,10 @@ "no_proxy": { "type": "string", "example": "localhost" + }, + "huggingface_token": { + "type": "string", + "example": "your_token" } } }, @@ -1826,7 +1830,8 @@ "verify_proxy_host_ssl": false, "verify_peer_ssl": false, "verify_host_ssl": false, - "no_proxy": "localhost" + "no_proxy": "localhost", + "huggingface_token": "your_token" } } } @@ -1896,6 +1901,11 @@ "type": "string", "description": "List of hosts that should not be proxied.", "example": "localhost" + }, + "huggingface_token": { + "type": "string", + "description": "HuggingFace token to pull models.", + "example": "your_token" } } } @@ -1958,6 +1968,10 @@ "no_proxy": { "type": "string", "example": "localhost" + }, + "huggingface_token": { + "type": "string", + "example": "your_token" } } }, From 77470d449d38798e2e005727eb4036d5b7e07ebd Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 09:17:33 +0700 Subject: [PATCH 3/6] chore: Token docs --- docs/docs/configurations/token.mdx | 120 +++++++++++++++++++++++++++++ docs/sidebars.ts | 5 ++ 2 files changed, 125 insertions(+) create mode 100644 docs/docs/configurations/token.mdx diff --git a/docs/docs/configurations/token.mdx b/docs/docs/configurations/token.mdx new file mode 100644 index 000000000..687879cb4 --- /dev/null +++ b/docs/docs/configurations/token.mdx @@ -0,0 +1,120 @@ +--- +title: Token +description: Setting up token +slug: "token" +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +:::warning +🚧 Cortex.cpp is currently under development. Our documentation outlines the intended behavior of Cortex, which may not yet be fully implemented in the codebase. +::: + +# Token Configuration Guide + +This document describes how to configure HuggingFace token settings for Cortex. + +## Command Line Interface (CLI) + +### Basic Usage + +```bash +cortex config [OPTIONS] [COMMAND] +``` + +### Commands + +- `status`: Display all current configurations + +```bash +cortex config status +``` + +Example Output: + +```bash ++-----------------------+------------------------+ +| Config name | Value | ++-----------------------+------------------------+ +| huggingface_token | | ++-----------------------+------------------------+ +``` + +### Options + +| Option | Description | Example | +| ----------------------------------- | --------------------------- | ------------------------------------------------- | +| `-h, --help` | Print help message and exit | +| `--huggingface_token ` | Set HuggingFace token | `cortex config --huggingface_token token` | + +## Token API Configuration + +### Endpoints + +#### Get Current Configuration + +```http +GET /v1/configs +``` + +Retrieves the current configuration settings. + +##### Response + +```json +{ + "allowed_origins": [ + "http://localhost:39281", + "http://127.0.0.1:39281", + "http://0.0.0.0:39281" + ], + "cors": true, + "huggingface_token": "" +} +``` + +#### Update Configuration + +```http +PATCH /v1/configs +``` + +Updates HuggingFace token configuration settings. + +##### Request Headers + +``` +Content-Type: application/json +``` + +##### Request Body + +```json +{ + "huggingface_token": "token" +} +``` + +##### Parameters + +| Field | Type | Description | +| ----------------------- | ------- | ------------------------------------------| +| `huggingface_token` | string | HuggingFace token to pull models | + +##### Response + +```json +{ + "config": { + "allowed_origins": [ + "http://localhost:39281", + "http://127.0.0.1:39281", + "http://0.0.0.0:39281" + ], + "cors": true, + "huggingface_token": "token" + }, + "message": "Configuration updated successfully" +} +``` \ No newline at end of file diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 126673f1a..b8c6f3387 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -95,6 +95,11 @@ const sidebars: SidebarsConfig = { id: "configurations/cors", label: "CORS", }, + { + type: "doc", + id: "configurations/token", + label: "token", + }, ], }, { From 9ac9e04426ba71a0d75261e58d20801bffc60d78 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 09:59:37 +0700 Subject: [PATCH 4/6] chore: docs --- docs/sidebars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sidebars.ts b/docs/sidebars.ts index b8c6f3387..6bacfe606 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -98,7 +98,7 @@ const sidebars: SidebarsConfig = { { type: "doc", id: "configurations/token", - label: "token", + label: "Token", }, ], }, From bba66dcb9280eeed6b5cc8000ff5f7a5976524a8 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 11:03:44 +0700 Subject: [PATCH 5/6] fix: log level --- engine/services/hardware_service.cc | 11 ++++++++--- engine/utils/logging_utils.h | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc index b9f8fe63b..8510bdd53 100644 --- a/engine/services/hardware_service.cc +++ b/engine/services/hardware_service.cc @@ -117,6 +117,8 @@ bool HardwareService::Restart(const std::string& host, int port) { std::string params = "--ignore_cout"; params += " --config_file_path " + get_config_file_path(); params += " --data_folder_path " + get_data_folder_path(); + params += " --loglevel " + + logging_utils_helper::LogLevelStr(trantor::Logger::logLevel()); std::string cmds = cortex_utils::GetCurrentPath() + "/" + exe + " " + params; // Create child process if (!CreateProcess( @@ -166,9 +168,12 @@ bool HardwareService::Restart(const std::string& host, int port) { CTL_INF("LD_LIBRARY_PATH: " << getenv(name)); #endif std::string p = cortex_utils::GetCurrentPath() + "/" + exe; - execl(p.c_str(), exe.c_str(), "--ignore_cout", "--config_file_path", - get_config_file_path().c_str(), "--data_folder_path", - get_data_folder_path().c_str(), "--loglevel", "INFO", (char*)0); + execl( + p.c_str(), exe.c_str(), "--ignore_cout", "--config_file_path", + get_config_file_path().c_str(), "--data_folder_path", + get_data_folder_path().c_str(), "--loglevel", + logging_utils_helper::LogLevelStr(trantor::Logger::logLevel()).c_str(), + (char*)0); } else { // Parent process if (!TryConnectToServer(host, port)) { diff --git a/engine/utils/logging_utils.h b/engine/utils/logging_utils.h index 2c5affcd4..523b2cd9c 100644 --- a/engine/utils/logging_utils.h +++ b/engine/utils/logging_utils.h @@ -59,4 +59,21 @@ inline void SetLogLevel(const std::string& log_level, bool ignore_cout) { << std::endl; } } + +inline std::string LogLevelStr(const trantor::Logger::LogLevel& log_level) { + switch (log_level) { + case trantor::Logger::kTrace: + return "TRACE"; + case trantor::Logger::kDebug: + return "DEBUG"; + case trantor::Logger::kInfo: + return "INFO"; + case trantor::Logger::kWarn: + return "WARN"; + case trantor::Logger::kError: + return "ERROR"; + default: + return "UNKNOWN"; + } +} } // namespace logging_utils_helper \ No newline at end of file From 39a166a085536097ffcf27e85a874707443f8f51 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 19 Nov 2024 11:22:37 +0700 Subject: [PATCH 6/6] fix: logging issue --- engine/services/engine_service.cc | 2 +- engine/services/hardware_service.cc | 14 ++++++-------- engine/utils/logging_utils.h | 8 ++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index d793989af..8d8a4a65c 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -899,7 +899,7 @@ cpp::result EngineService::LoadEngine( CTL_WRN("Method SetFileLogger is not supported yet"); } if (en->IsSupported("SetLogLevel")) { - en->SetLogLevel(trantor::Logger::logLevel()); + en->SetLogLevel(logging_utils_helper::global_log_level); } else { CTL_WRN("Method SetLogLevel is not supported yet"); } diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc index 8510bdd53..905b17107 100644 --- a/engine/services/hardware_service.cc +++ b/engine/services/hardware_service.cc @@ -57,6 +57,7 @@ HardwareInfo HardwareService::GetHardwareInfo() { } bool HardwareService::Restart(const std::string& host, int port) { + namespace luh = logging_utils_helper; if (!ahc_) return true; auto exe = commands::GetCortexServerBinary(); @@ -117,8 +118,7 @@ bool HardwareService::Restart(const std::string& host, int port) { std::string params = "--ignore_cout"; params += " --config_file_path " + get_config_file_path(); params += " --data_folder_path " + get_data_folder_path(); - params += " --loglevel " + - logging_utils_helper::LogLevelStr(trantor::Logger::logLevel()); + params += " --loglevel " + luh::LogLevelStr(luh::global_log_level); std::string cmds = cortex_utils::GetCurrentPath() + "/" + exe + " " + params; // Create child process if (!CreateProcess( @@ -168,12 +168,10 @@ bool HardwareService::Restart(const std::string& host, int port) { CTL_INF("LD_LIBRARY_PATH: " << getenv(name)); #endif std::string p = cortex_utils::GetCurrentPath() + "/" + exe; - execl( - p.c_str(), exe.c_str(), "--ignore_cout", "--config_file_path", - get_config_file_path().c_str(), "--data_folder_path", - get_data_folder_path().c_str(), "--loglevel", - logging_utils_helper::LogLevelStr(trantor::Logger::logLevel()).c_str(), - (char*)0); + execl(p.c_str(), exe.c_str(), "--ignore_cout", "--config_file_path", + get_config_file_path().c_str(), "--data_folder_path", + get_data_folder_path().c_str(), "--loglevel", + luh::LogLevelStr(luh::global_log_level).c_str(), (char*)0); } else { // Parent process if (!TryConnectToServer(host, port)) { diff --git a/engine/utils/logging_utils.h b/engine/utils/logging_utils.h index 523b2cd9c..d2c04a7e8 100644 --- a/engine/utils/logging_utils.h +++ b/engine/utils/logging_utils.h @@ -32,25 +32,33 @@ inline bool is_server = false; } namespace logging_utils_helper { +// In macOS, the default log level is reset to INFO when we load engine +// Use a global log level to save the value +inline trantor::Logger::LogLevel global_log_level = trantor::Logger::kInfo; inline void SetLogLevel(const std::string& log_level, bool ignore_cout) { if (log_level == "TRACE") { trantor::Logger::setLogLevel(trantor::Logger::kTrace); + global_log_level = trantor::Logger::kTrace; if (!ignore_cout) std::cout << "Set log level to TRACE" << std::endl; } else if (log_level == "DEBUG") { trantor::Logger::setLogLevel(trantor::Logger::kDebug); + global_log_level = trantor::Logger::kDebug; if (!ignore_cout) std::cout << "Set log level to DEBUG" << std::endl; } else if (log_level == "INFO") { trantor::Logger::setLogLevel(trantor::Logger::kInfo); + global_log_level = trantor::Logger::kInfo; if (!ignore_cout) std::cout << "Set log level to INFO" << std::endl; } else if (log_level == "WARN") { trantor::Logger::setLogLevel(trantor::Logger::kWarn); + global_log_level = trantor::Logger::kWarn; if (!ignore_cout) std::cout << "Set log level to WARN" << std::endl; } else if (log_level == "ERROR") { trantor::Logger::setLogLevel(trantor::Logger::kError); + global_log_level = trantor::Logger::kError; if (!ignore_cout) std::cout << "Set log level to ERROR" << std::endl; } else {