diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 6f0493ec2..99bdd0009 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -7,10 +7,10 @@ echo "enableCors: true" >> /root/.cortexrc # Install the engine cortex engines install llama-cpp -s /opt/cortex.llamacpp -cortex engines list # Start the cortex server cortex start +cortex engines list # Keep the container running by tailing the log files tail -f /root/cortexcpp/logs/cortex.log & diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 5ffabf23c..06e778b7e 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -176,10 +176,11 @@ aux_source_directory(cortex-common CORTEX_COMMON) aux_source_directory(config CONFIG_SRC) aux_source_directory(database DB_SRC) aux_source_directory(migrations MIGR_SRC) +aux_source_directory(utils UTILS_SRC) target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) -target_sources(${TARGET_NAME} PRIVATE ${CONFIG_SRC} ${CTL_SRC} ${COMMON_SRC} ${SERVICES_SRC} ${DB_SRC} ${MIGR_SRC}) +target_sources(${TARGET_NAME} PRIVATE ${UTILS_SRC} ${CONFIG_SRC} ${CTL_SRC} ${COMMON_SRC} ${SERVICES_SRC} ${DB_SRC} ${MIGR_SRC}) set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR} diff --git a/engine/cli/CMakeLists.txt b/engine/cli/CMakeLists.txt index db2bed828..42d00ebd5 100644 --- a/engine/cli/CMakeLists.txt +++ b/engine/cli/CMakeLists.txt @@ -84,6 +84,10 @@ add_executable(${TARGET_NAME} main.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/hardware_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/utils/easywsclient.cc ${CMAKE_CURRENT_SOURCE_DIR}/utils/download_progress.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/config_yaml_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_manager_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/curl_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/system_info_utils.cc ) target_link_libraries(${TARGET_NAME} PRIVATE CLI11::CLI11) diff --git a/engine/cli/commands/engine_list_cmd.cc b/engine/cli/commands/engine_list_cmd.cc index b010e8687..35584dcd2 100644 --- a/engine/cli/commands/engine_list_cmd.cc +++ b/engine/cli/commands/engine_list_cmd.cc @@ -4,6 +4,7 @@ #include "common/engine_servicei.h" #include "server_start_cmd.h" #include "utils/curl_utils.h" +#include "utils/engine_constants.h" #include "utils/logging_utils.h" #include "utils/url_parser.h" // clang-format off diff --git a/engine/cli/commands/ps_cmd.cc b/engine/cli/commands/ps_cmd.cc index c692ffc00..24ef497c6 100644 --- a/engine/cli/commands/ps_cmd.cc +++ b/engine/cli/commands/ps_cmd.cc @@ -2,6 +2,7 @@ #include #include #include "utils/curl_utils.h" +#include "utils/engine_constants.h" #include "utils/format_utils.h" #include "utils/logging_utils.h" #include "utils/string_utils.h" diff --git a/engine/cli/commands/server_start_cmd.cc b/engine/cli/commands/server_start_cmd.cc index cfed72c24..ba4f7bd82 100644 --- a/engine/cli/commands/server_start_cmd.cc +++ b/engine/cli/commands/server_start_cmd.cc @@ -1,6 +1,7 @@ #include "server_start_cmd.h" #include "commands/cortex_upd_cmd.h" #include "utils/cortex_utils.h" +#include "utils/engine_constants.h" #include "utils/file_manager_utils.h" #include "utils/widechar_conv.h" @@ -27,6 +28,10 @@ bool TryConnectToServer(const std::string& host, int port) { bool ServerStartCmd::Exec(const std::string& host, int port, const std::optional& log_level) { + if (IsServerAlive(host, port)) { + CLI_LOG("The server has already started"); + return true; + } std::string log_level_; if (!log_level.has_value()) { log_level_ = "INFO"; diff --git a/engine/cli/main.cc b/engine/cli/main.cc index 52fc5591f..8ed4beb61 100644 --- a/engine/cli/main.cc +++ b/engine/cli/main.cc @@ -148,7 +148,6 @@ int main(int argc, char* argv[]) { if (should_check_for_latest_llamacpp_version) { std::thread t1([]() { - auto config = file_manager_utils::GetCortexConfig(); // TODO: namh current we only check for llamacpp. Need to add support for other engine auto get_latest_version = []() -> cpp::result { try { @@ -176,6 +175,7 @@ int main(int argc, char* argv[]) { auto now = std::chrono::system_clock::now(); CTL_DBG("latest llama.cpp version: " << res.value()); + auto config = file_manager_utils::GetCortexConfig(); config.checkedForLlamacppUpdateAt = std::chrono::duration_cast( now.time_since_epoch()) diff --git a/engine/e2e-test/main.py b/engine/e2e-test/main.py index 9ef2970f9..e874ab3a0 100644 --- a/engine/e2e-test/main.py +++ b/engine/e2e-test/main.py @@ -3,26 +3,16 @@ ### e2e tests are expensive, have to keep engines tests in order from test_api_engine_list import TestApiEngineList -from test_api_engine_install import TestApiEngineInstall -from test_api_engine_get import TestApiEngineGet - -### models, keeps in order, note that we only uninstall engine after finishing all models test -from test_api_model_pull_direct_url import TestApiModelPullDirectUrl -from test_api_model_start_stop import TestApiModelStartStop -from test_api_model_get import TestApiModelGet -from test_api_model_list import TestApiModelList -from test_api_model_update import TestApiModelUpdate -from test_api_model_delete import TestApiModelDelete +from test_api_engine import TestApiEngine +from test_api_model import TestApiModel from test_api_model_import import TestApiModelImport -from test_api_engine_uninstall import TestApiEngineUninstall ### from test_cli_engine_get import TestCliEngineGet from test_cli_engine_install import TestCliEngineInstall from test_cli_engine_list import TestCliEngineList from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model_delete import TestCliModelDelete -from test_cli_model_pull_direct_url import TestCliModelPullDirectUrl +from test_cli_model import TestCliModel from test_cli_server_start import TestCliServerStart from test_cortex_update import TestCortexUpdate from test_create_log_folder import TestCreateLogFolder diff --git a/engine/e2e-test/test_api_engine_uninstall.py b/engine/e2e-test/test_api_engine.py similarity index 60% rename from engine/e2e-test/test_api_engine_uninstall.py rename to engine/e2e-test/test_api_engine.py index 1951e5c3a..57b47b879 100644 --- a/engine/e2e-test/test_api_engine_uninstall.py +++ b/engine/e2e-test/test_api_engine.py @@ -1,29 +1,49 @@ -import time - import pytest import requests +import time from test_runner import ( - run, - start_server_if_needed, + start_server, stop_server, wait_for_websocket_download_success_event, ) - -class TestApiEngineUninstall: +class TestApiEngine: @pytest.fixture(autouse=True) def setup_and_teardown(self): # Setup - start_server_if_needed() + success = start_server() + if not success: + raise Exception("Failed to start server") yield # Teardown stop_server() + + # engines get + def test_engines_get_llamacpp_should_be_successful(self): + response = requests.get("http://localhost:3928/engines/llama-cpp") + assert response.status_code == 200 + + # engines install + def test_engines_install_llamacpp_specific_version_and_variant(self): + data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} + response = requests.post( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert response.status_code == 200 + def test_engines_install_llamacpp_specific_version_and_null_variant(self): + data = {"version": "v0.1.35-27.10.24"} + response = requests.post( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert response.status_code == 200 + + # engines uninstall @pytest.mark.asyncio - async def test_engines_uninstall_llamacpp_should_be_successful(self): + async def test_engines_install_uninstall_llamacpp_should_be_successful(self): response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") assert response.status_code == 200 await wait_for_websocket_download_success_event(timeout=None) @@ -33,7 +53,7 @@ async def test_engines_uninstall_llamacpp_should_be_successful(self): assert response.status_code == 200 @pytest.mark.asyncio - async def test_engines_uninstall_llamacpp_with_only_version_should_be_failed(self): + async def test_engines_install_uninstall_llamacpp_with_only_version_should_be_failed(self): # install first data = {"variant": "mac-arm64"} install_response = requests.post( @@ -50,7 +70,7 @@ async def test_engines_uninstall_llamacpp_with_only_version_should_be_failed(sel assert response.json()["message"] == "No variant provided" @pytest.mark.asyncio - async def test_engines_uninstall_llamacpp_with_variant_should_be_successful(self): + async def test_engines_install_uninstall_llamacpp_with_variant_should_be_successful(self): # install first data = {"variant": "mac-arm64"} install_response = requests.post( @@ -62,7 +82,7 @@ async def test_engines_uninstall_llamacpp_with_variant_should_be_successful(self response = requests.delete("http://127.0.0.1:3928/v1/engines/llama-cpp/install") assert response.status_code == 200 - def test_engines_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful( + def test_engines_install_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful( self, ): data = {"variant": "mac-arm64", "version": "v0.1.35"} @@ -76,3 +96,5 @@ def test_engines_uninstall_llamacpp_with_specific_variant_and_version_should_be_ "http://localhost:3928/v1/engines/llama-cpp/install", json=data ) assert response.status_code == 200 + + \ No newline at end of file diff --git a/engine/e2e-test/test_api_engine_get.py b/engine/e2e-test/test_api_engine_get.py deleted file mode 100644 index baa9c8037..000000000 --- a/engine/e2e-test/test_api_engine_get.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import requests -from test_runner import start_server, stop_server - - -class TestApiEngineGet: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_engines_get_llamacpp_should_be_successful(self): - response = requests.get("http://localhost:3928/engines/llama-cpp") - assert response.status_code == 200 diff --git a/engine/e2e-test/test_api_engine_install.py b/engine/e2e-test/test_api_engine_install.py deleted file mode 100644 index aabe0138d..000000000 --- a/engine/e2e-test/test_api_engine_install.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -import requests -from test_runner import start_server, stop_server - - -class TestApiEngineInstall: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_engines_install_llamacpp_should_be_successful(self): - response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") - assert response.status_code == 200 - - def test_engines_install_llamacpp_specific_version_and_variant(self): - data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} - response = requests.post( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data - ) - assert response.status_code == 200 - - def test_engines_install_llamacpp_specific_version_and_null_variant(self): - data = {"version": "v0.1.35-27.10.24"} - response = requests.post( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data - ) - assert response.status_code == 200 diff --git a/engine/e2e-test/test_api_engine_list.py b/engine/e2e-test/test_api_engine_list.py index 71b9ea8b4..f149f1450 100644 --- a/engine/e2e-test/test_api_engine_list.py +++ b/engine/e2e-test/test_api_engine_list.py @@ -22,4 +22,4 @@ def setup_and_teardown(self): def test_engines_list_api_run_successfully(self): response = requests.get("http://localhost:3928/engines") - assert response.status_code == 200 + assert response.status_code == 200 \ No newline at end of file diff --git a/engine/e2e-test/test_api_model_pull_direct_url.py b/engine/e2e-test/test_api_model.py similarity index 53% rename from engine/e2e-test/test_api_model_pull_direct_url.py rename to engine/e2e-test/test_api_model.py index 604f216f8..c2723d2ca 100644 --- a/engine/e2e-test/test_api_model_pull_direct_url.py +++ b/engine/e2e-test/test_api_model.py @@ -1,5 +1,6 @@ import pytest import requests +import time from test_runner import ( run, start_server, @@ -7,27 +8,22 @@ wait_for_websocket_download_success_event, ) - -class TestApiModelPullDirectUrl: +class TestApiModel: @pytest.fixture(autouse=True) def setup_and_teardown(self): - # Setup - stop_server() + # Setup success = start_server() if not success: raise Exception("Failed to start server") # Delete model if exists - run( - "Delete model", - [ - "models", - "delete", - "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf", - ], - ) yield # Teardown + stop_server() + + # Pull with direct url + @pytest.mark.asyncio + async def test_model_pull_with_direct_url_should_be_success(self): run( "Delete model", [ @@ -36,10 +32,7 @@ def setup_and_teardown(self): "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf", ], ) - stop_server() - - @pytest.mark.asyncio - async def test_model_pull_with_direct_url_should_be_success(self): + myobj = { "model": "https://huggingface.co/afrideva/zephyr-smol_llama-100m-sft-full-GGUF/blob/main/zephyr-smol_llama-100m-sft-full.q2_k.gguf" } @@ -54,6 +47,15 @@ async def test_model_pull_with_direct_url_should_be_success(self): get_model_response.json()["model"] == "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf" ) + + run( + "Delete model", + [ + "models", + "delete", + "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf", + ], + ) @pytest.mark.asyncio async def test_model_pull_with_direct_url_should_have_desired_name(self): @@ -73,3 +75,58 @@ async def test_model_pull_with_direct_url_should_have_desired_name(self): get_model_response.json()["name"] == "smol_llama_100m" ) + + run( + "Delete model", + [ + "models", + "delete", + "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf", + ], + ) + + async def test_models_start_stop_should_be_successful(self): + print("Install engine") + response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") + assert response.status_code == 200 + await wait_for_websocket_download_success_event(timeout=None) + # TODO(sang) need to fix for cuda download + time.sleep(30) + + print("Pull model") + json_body = {"model": "tinyllama:gguf"} + response = requests.post("http://localhost:3928/v1/models/pull", json=json_body) + assert response.status_code == 200, f"Failed to pull model: tinyllama:gguf" + await wait_for_websocket_download_success_event(timeout=None) + + # get API + print("Get model") + response = requests.get("http://localhost:3928/v1/models/tinyllama:gguf") + assert response.status_code == 200 + + # list API + print("List model") + response = requests.get("http://localhost:3928/v1/models") + assert response.status_code == 200 + + print("Start model") + json_body = {"model": "tinyllama:gguf"} + response = requests.post( + "http://localhost:3928/v1/models/start", json=json_body + ) + assert response.status_code == 200, f"status_code: {response.status_code}" + + print("Stop model") + response = requests.post("http://localhost:3928/v1/models/stop", json=json_body) + assert response.status_code == 200, f"status_code: {response.status_code}" + + # update API + print("Update model") + body_json = {'model': 'tinyllama:gguf'} + response = requests.patch("http://localhost:3928/v1/models/tinyllama:gguf", json = body_json) + assert response.status_code == 200 + + # delete API + print("Delete model") + response = requests.delete("http://localhost:3928/v1/models/tinyllama:gguf") + assert response.status_code == 200 \ No newline at end of file diff --git a/engine/e2e-test/test_api_model_delete.py b/engine/e2e-test/test_api_model_delete.py deleted file mode 100644 index 455032a9b..000000000 --- a/engine/e2e-test/test_api_model_delete.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import requests -from test_runner import start_server, stop_server - - -class TestApiModelDelete: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_models_delete_should_be_successful(self): - response = requests.delete("http://localhost:3928/v1/models/tinyllama:gguf") - assert response.status_code == 200 diff --git a/engine/e2e-test/test_api_model_get.py b/engine/e2e-test/test_api_model_get.py deleted file mode 100644 index dd58ca2a4..000000000 --- a/engine/e2e-test/test_api_model_get.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import requests -from test_runner import popen, run -from test_runner import start_server, stop_server - - -class TestApiModelGet: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - stop_server() - - def test_models_get_should_be_successful(self): - response = requests.get("http://localhost:3928/v1/models/tinyllama:gguf") - assert response.status_code == 200 diff --git a/engine/e2e-test/test_api_model_list.py b/engine/e2e-test/test_api_model_list.py deleted file mode 100644 index 5e2a4b901..000000000 --- a/engine/e2e-test/test_api_model_list.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import requests -from test_runner import start_server, stop_server - - -class TestApiModelList: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_models_list_should_be_successful(self): - response = requests.get("http://localhost:3928/v1/models") - assert response.status_code == 200 diff --git a/engine/e2e-test/test_api_model_start_stop.py b/engine/e2e-test/test_api_model_start_stop.py deleted file mode 100644 index 78c20e8da..000000000 --- a/engine/e2e-test/test_api_model_start_stop.py +++ /dev/null @@ -1,46 +0,0 @@ -import time - -import pytest -import requests -from test_runner import ( - run, - start_server_if_needed, - stop_server, - wait_for_websocket_download_success_event, -) - - -class TestApiModelStartStop: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - start_server_if_needed() - run("Delete model", ["models", "delete", "tinyllama:gguf"]) - - yield - - # Teardown - stop_server() - - @pytest.mark.asyncio - async def test_models_start_should_be_successful(self): - response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") - assert response.status_code == 200 - await wait_for_websocket_download_success_event(timeout=None) - # TODO(sang) need to fix for cuda download - time.sleep(30) - - json_body = {"model": "tinyllama:gguf"} - response = requests.post("http://localhost:3928/v1/models/pull", json=json_body) - assert response.status_code == 200, f"Failed to pull model: tinyllama:gguf" - await wait_for_websocket_download_success_event(timeout=None) - - json_body = {"model": "tinyllama:gguf"} - response = requests.post( - "http://localhost:3928/v1/models/start", json=json_body - ) - assert response.status_code == 200, f"status_code: {response.status_code}" - - response = requests.post("http://localhost:3928/v1/models/stop", json=json_body) - assert response.status_code == 200, f"status_code: {response.status_code}" diff --git a/engine/e2e-test/test_api_model_update.py b/engine/e2e-test/test_api_model_update.py deleted file mode 100644 index f862c8907..000000000 --- a/engine/e2e-test/test_api_model_update.py +++ /dev/null @@ -1,23 +0,0 @@ -import pytest -import requests -from test_runner import popen, run -from test_runner import start_server, stop_server - - -class TestApiModelUpdate: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - stop_server() - - def test_models_update_should_be_successful(self): - body_json = {'model': 'tinyllama:gguf'} - response = requests.patch("http://localhost:3928/v1/models/tinyllama:gguf", json = body_json) - assert response.status_code == 200 diff --git a/engine/e2e-test/test_cli_model_delete.py b/engine/e2e-test/test_cli_model.py similarity index 58% rename from engine/e2e-test/test_cli_model_delete.py rename to engine/e2e-test/test_cli_model.py index 06cc3a4c3..f6aad4ae9 100644 --- a/engine/e2e-test/test_cli_model_delete.py +++ b/engine/e2e-test/test_cli_model.py @@ -1,5 +1,7 @@ import pytest import requests +import os +from pathlib import Path from test_runner import ( run, start_server, @@ -7,8 +9,7 @@ wait_for_websocket_download_success_event, ) - -class TestCliModelDelete: +class TestCliModel: @pytest.fixture(autouse=True) def setup_and_teardown(self): @@ -23,7 +24,20 @@ def setup_and_teardown(self): # Clean up run("Delete model", ["models", "delete", "tinyllama:gguf"]) stop_server() - + + def test_model_pull_with_direct_url_should_be_success(self): + exit_code, output, error = run( + "Pull model", + [ + "pull", + "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/blob/main/tinyllama-1.1b-chat-v0.3.Q2_K.gguf", + ], + timeout=None, capture=False + ) + root = Path.home() + assert os.path.exists(root / "cortexcpp" / "models" / "huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf") + assert exit_code == 0, f"Model pull failed with error: {error}" + @pytest.mark.asyncio async def test_models_delete_should_be_successful(self): json_body = {"model": "tinyllama:gguf"} @@ -35,4 +49,4 @@ async def test_models_delete_should_be_successful(self): "Delete model", ["models", "delete", "tinyllama:gguf"] ) assert "Model tinyllama:gguf deleted successfully" in output - assert exit_code == 0, f"Model does not exist: {error}" + assert exit_code == 0, f"Model does not exist: {error}" \ No newline at end of file diff --git a/engine/e2e-test/test_cli_model_pull_direct_url.py b/engine/e2e-test/test_cli_model_pull_direct_url.py deleted file mode 100644 index b10d1593d..000000000 --- a/engine/e2e-test/test_cli_model_pull_direct_url.py +++ /dev/null @@ -1,32 +0,0 @@ -from test_runner import run -import os -from pathlib import Path - -class TestCliModelPullDirectUrl: - - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_model_pull_with_direct_url_should_be_success(self): - exit_code, output, error = run( - "Pull model", - [ - "pull", - "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/blob/main/tinyllama-1.1b-chat-v0.3.Q2_K.gguf", - ], - timeout=None, capture=False - ) - root = Path.home() - assert os.path.exists(root / "cortexcpp" / "models" / "huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf") - assert exit_code == 0, f"Model pull failed with error: {error}" - # TODO: verify that the model has been pull successfully - # TODO: skip this test. since download model is taking too long - diff --git a/engine/test/components/CMakeLists.txt b/engine/test/components/CMakeLists.txt index 4a15b7c8b..58c5d83d6 100644 --- a/engine/test/components/CMakeLists.txt +++ b/engine/test/components/CMakeLists.txt @@ -12,6 +12,10 @@ add_executable(${PROJECT_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/../../services/config_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../services/download_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../database/models.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/config_yaml_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/file_manager_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/curl_utils.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/system_info_utils.cc ) find_package(Drogon CONFIG REQUIRED) diff --git a/engine/utils/config_yaml_utils.cc b/engine/utils/config_yaml_utils.cc new file mode 100644 index 000000000..4d6f47ebe --- /dev/null +++ b/engine/utils/config_yaml_utils.cc @@ -0,0 +1,177 @@ +#include "config_yaml_utils.h" + +namespace config_yaml_utils { +cpp::result CortexConfigMgr::DumpYamlConfig( + const CortexConfig& config, const std::string& path) { + std::lock_guard l(mtx_); + std::filesystem::path config_file_path{path}; + + try { + std::ofstream out_file(config_file_path); + if (!out_file) { + throw std::runtime_error("Failed to open output file."); + } + // Workaround to save file as utf8 BOM + const unsigned char utf8_bom[] = {0xEF, 0xBB, 0xBF}; + out_file.write(reinterpret_cast(utf8_bom), sizeof(utf8_bom)); + YAML::Node node; + node["logFolderPath"] = config.logFolderPath; + node["logLlamaCppPath"] = config.logLlamaCppPath; + node["logTensorrtLLMPath"] = config.logTensorrtLLMPath; + node["logOnnxPath"] = config.logOnnxPath; + node["dataFolderPath"] = config.dataFolderPath; + node["maxLogLines"] = config.maxLogLines; + node["apiServerHost"] = config.apiServerHost; + node["apiServerPort"] = config.apiServerPort; + node["checkedForUpdateAt"] = config.checkedForUpdateAt; + node["checkedForLlamacppUpdateAt"] = config.checkedForLlamacppUpdateAt; + node["latestRelease"] = config.latestRelease; + node["latestLlamacppRelease"] = config.latestLlamacppRelease; + node["huggingFaceToken"] = config.huggingFaceToken; + node["gitHubUserAgent"] = config.gitHubUserAgent; + node["gitHubToken"] = config.gitHubToken; + node["llamacppVariant"] = config.llamacppVariant; + node["llamacppVersion"] = config.llamacppVersion; + node["enableCors"] = config.enableCors; + node["allowedOrigins"] = config.allowedOrigins; + node["proxyUrl"] = config.proxyUrl; + node["verifyProxySsl"] = config.verifyProxySsl; + node["verifyProxyHostSsl"] = config.verifyProxyHostSsl; + node["proxyUsername"] = config.proxyUsername; + node["proxyPassword"] = config.proxyPassword; + node["noProxy"] = config.noProxy; + node["verifyPeerSsl"] = config.verifyPeerSsl; + node["verifyHostSsl"] = config.verifyHostSsl; + + out_file << node; + out_file.close(); + return {}; + } catch (const std::exception& e) { + CTL_ERR("Error writing to file: " << e.what()); + return cpp::fail("Error writing to file: " + std::string(e.what())); + } +} + +CortexConfig CortexConfigMgr::FromYaml(const std::string& path, + const CortexConfig& default_cfg) { + std::unique_lock l(mtx_); + std::filesystem::path config_file_path{path}; + if (!std::filesystem::exists(config_file_path)) { + throw std::runtime_error("File not found: " + path); + } + + try { + auto node = YAML::LoadFile(config_file_path.string()); + bool should_update_config = + (!node["logFolderPath"] || !node["dataFolderPath"] || + !node["maxLogLines"] || !node["apiServerHost"] || + !node["apiServerPort"] || !node["checkedForUpdateAt"] || + !node["checkedForLlamacppUpdateAt"] || !node["latestRelease"] || + !node["latestLlamacppRelease"] || !node["logLlamaCppPath"] || + !node["logOnnxPath"] || !node["logTensorrtLLMPath"] || + !node["huggingFaceToken"] || !node["gitHubUserAgent"] || + !node["gitHubToken"] || !node["llamacppVariant"] || + !node["llamacppVersion"] || !node["enableCors"] || + !node["allowedOrigins"] || !node["proxyUrl"] || + !node["proxyUsername"] || !node["proxyPassword"] || + !node["verifyPeerSsl"] || !node["verifyHostSsl"] || + !node["verifyProxySsl"] || !node["verifyProxyHostSsl"] || + !node["noProxy"]); + + CortexConfig config = { + .logFolderPath = node["logFolderPath"] + ? node["logFolderPath"].as() + : default_cfg.logFolderPath, + .logLlamaCppPath = node["logLlamaCppPath"] + ? node["logLlamaCppPath"].as() + : default_cfg.logLlamaCppPath, + .logTensorrtLLMPath = node["logTensorrtLLMPath"] + ? node["logTensorrtLLMPath"].as() + : default_cfg.logTensorrtLLMPath, + .logOnnxPath = node["logOnnxPath"] + ? node["logOnnxPath"].as() + : default_cfg.logOnnxPath, + .dataFolderPath = node["dataFolderPath"] + ? node["dataFolderPath"].as() + : default_cfg.dataFolderPath, + .maxLogLines = node["maxLogLines"] ? node["maxLogLines"].as() + : default_cfg.maxLogLines, + .apiServerHost = node["apiServerHost"] + ? node["apiServerHost"].as() + : default_cfg.apiServerHost, + .apiServerPort = node["apiServerPort"] + ? node["apiServerPort"].as() + : default_cfg.apiServerPort, + .checkedForUpdateAt = node["checkedForUpdateAt"] + ? node["checkedForUpdateAt"].as() + : default_cfg.checkedForUpdateAt, + .checkedForLlamacppUpdateAt = + node["checkedForLlamacppUpdateAt"] + ? node["checkedForLlamacppUpdateAt"].as() + : default_cfg.checkedForLlamacppUpdateAt, + .latestRelease = node["latestRelease"] + ? node["latestRelease"].as() + : default_cfg.latestRelease, + .latestLlamacppRelease = + node["latestLlamacppRelease"] + ? node["latestLlamacppRelease"].as() + : default_cfg.latestLlamacppRelease, + .huggingFaceToken = node["huggingFaceToken"] + ? node["huggingFaceToken"].as() + : default_cfg.huggingFaceToken, + .gitHubUserAgent = node["gitHubUserAgent"] + ? node["gitHubUserAgent"].as() + : default_cfg.gitHubUserAgent, + .gitHubToken = node["gitHubToken"] + ? node["gitHubToken"].as() + : default_cfg.gitHubToken, + .llamacppVariant = node["llamacppVariant"] + ? node["llamacppVariant"].as() + : default_cfg.llamacppVariant, + .llamacppVersion = node["llamacppVersion"] + ? node["llamacppVersion"].as() + : default_cfg.llamacppVersion, + .enableCors = node["enableCors"] ? node["enableCors"].as() + : default_cfg.enableCors, + .allowedOrigins = + node["allowedOrigins"] + ? node["allowedOrigins"].as>() + : default_cfg.allowedOrigins, + .proxyUrl = node["proxyUrl"] ? node["proxyUrl"].as() + : default_cfg.proxyUrl, + .verifyProxySsl = node["verifyProxySsl"] + ? node["verifyProxySsl"].as() + : default_cfg.verifyProxySsl, + .verifyProxyHostSsl = node["verifyProxyHostSsl"] + ? node["verifyProxyHostSsl"].as() + : default_cfg.verifyProxyHostSsl, + .proxyUsername = node["proxyUsername"] + ? node["proxyUsername"].as() + : default_cfg.proxyUsername, + .proxyPassword = node["proxyPassword"] + ? node["proxyPassword"].as() + : default_cfg.proxyPassword, + .noProxy = node["noProxy"] ? node["noProxy"].as() + : default_cfg.noProxy, + .verifyPeerSsl = node["verifyPeerSsl"] + ? node["verifyPeerSsl"].as() + : default_cfg.verifyPeerSsl, + .verifyHostSsl = node["verifyHostSsl"] + ? node["verifyHostSsl"].as() + : default_cfg.verifyHostSsl, + }; + if (should_update_config) { + l.unlock(); + auto result = DumpYamlConfig(config, path); + if (result.has_error()) { + CTL_ERR("Failed to update config file: " << result.error()); + } + } + return config; + } catch (const YAML::BadFile& e) { + CTL_ERR("Failed to read file: " << e.what()); + throw; + } +} + +} // namespace config_yaml_utils \ No newline at end of file diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index 73c990996..aa1b4027e 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -77,178 +77,10 @@ class CortexConfigMgr { } cpp::result DumpYamlConfig(const CortexConfig& config, - const std::string& path) { - std::lock_guard l(mtx_); - std::filesystem::path config_file_path{path}; - - try { - std::ofstream out_file(config_file_path); - if (!out_file) { - throw std::runtime_error("Failed to open output file."); - } - // Workaround to save file as utf8 BOM - const unsigned char utf8_bom[] = {0xEF, 0xBB, 0xBF}; - out_file.write(reinterpret_cast(utf8_bom), sizeof(utf8_bom)); - YAML::Node node; - node["logFolderPath"] = config.logFolderPath; - node["logLlamaCppPath"] = config.logLlamaCppPath; - node["logTensorrtLLMPath"] = config.logTensorrtLLMPath; - node["logOnnxPath"] = config.logOnnxPath; - node["dataFolderPath"] = config.dataFolderPath; - node["maxLogLines"] = config.maxLogLines; - node["apiServerHost"] = config.apiServerHost; - node["apiServerPort"] = config.apiServerPort; - node["checkedForUpdateAt"] = config.checkedForUpdateAt; - node["checkedForLlamacppUpdateAt"] = config.checkedForLlamacppUpdateAt; - node["latestRelease"] = config.latestRelease; - node["latestLlamacppRelease"] = config.latestLlamacppRelease; - node["huggingFaceToken"] = config.huggingFaceToken; - node["gitHubUserAgent"] = config.gitHubUserAgent; - node["gitHubToken"] = config.gitHubToken; - node["llamacppVariant"] = config.llamacppVariant; - node["llamacppVersion"] = config.llamacppVersion; - node["enableCors"] = config.enableCors; - node["allowedOrigins"] = config.allowedOrigins; - node["proxyUrl"] = config.proxyUrl; - node["verifyProxySsl"] = config.verifyProxySsl; - node["verifyProxyHostSsl"] = config.verifyProxyHostSsl; - node["proxyUsername"] = config.proxyUsername; - node["proxyPassword"] = config.proxyPassword; - node["noProxy"] = config.noProxy; - node["verifyPeerSsl"] = config.verifyPeerSsl; - node["verifyHostSsl"] = config.verifyHostSsl; - - out_file << node; - out_file.close(); - return {}; - } catch (const std::exception& e) { - CTL_ERR("Error writing to file: " << e.what()); - return cpp::fail("Error writing to file: " + std::string(e.what())); - } - } + const std::string& path); CortexConfig FromYaml(const std::string& path, - const CortexConfig& default_cfg) { - std::unique_lock l(mtx_); - std::filesystem::path config_file_path{path}; - if (!std::filesystem::exists(config_file_path)) { - throw std::runtime_error("File not found: " + path); - } - - try { - auto node = YAML::LoadFile(config_file_path.string()); - bool should_update_config = - (!node["logFolderPath"] || !node["dataFolderPath"] || - !node["maxLogLines"] || !node["apiServerHost"] || - !node["apiServerPort"] || !node["checkedForUpdateAt"] || - !node["checkedForLlamacppUpdateAt"] || !node["latestRelease"] || - !node["latestLlamacppRelease"] || !node["logLlamaCppPath"] || - !node["logOnnxPath"] || !node["logTensorrtLLMPath"] || - !node["huggingFaceToken"] || !node["gitHubUserAgent"] || - !node["gitHubToken"] || !node["llamacppVariant"] || - !node["llamacppVersion"] || !node["enableCors"] || - !node["allowedOrigins"] || !node["proxyUrl"] || - !node["proxyUsername"] || !node["proxyPassword"] || - !node["verifyPeerSsl"] || !node["verifyHostSsl"] || - !node["verifyProxySsl"] || !node["verifyProxyHostSsl"] || - !node["noProxy"]); - - CortexConfig config = { - .logFolderPath = node["logFolderPath"] - ? node["logFolderPath"].as() - : default_cfg.logFolderPath, - .logLlamaCppPath = node["logLlamaCppPath"] - ? node["logLlamaCppPath"].as() - : default_cfg.logLlamaCppPath, - .logTensorrtLLMPath = - node["logTensorrtLLMPath"] - ? node["logTensorrtLLMPath"].as() - : default_cfg.logTensorrtLLMPath, - .logOnnxPath = node["logOnnxPath"] - ? node["logOnnxPath"].as() - : default_cfg.logOnnxPath, - .dataFolderPath = node["dataFolderPath"] - ? node["dataFolderPath"].as() - : default_cfg.dataFolderPath, - .maxLogLines = node["maxLogLines"] ? node["maxLogLines"].as() - : default_cfg.maxLogLines, - .apiServerHost = node["apiServerHost"] - ? node["apiServerHost"].as() - : default_cfg.apiServerHost, - .apiServerPort = node["apiServerPort"] - ? node["apiServerPort"].as() - : default_cfg.apiServerPort, - .checkedForUpdateAt = node["checkedForUpdateAt"] - ? node["checkedForUpdateAt"].as() - : default_cfg.checkedForUpdateAt, - .checkedForLlamacppUpdateAt = - node["checkedForLlamacppUpdateAt"] - ? node["checkedForLlamacppUpdateAt"].as() - : default_cfg.checkedForLlamacppUpdateAt, - .latestRelease = node["latestRelease"] - ? node["latestRelease"].as() - : default_cfg.latestRelease, - .latestLlamacppRelease = - node["latestLlamacppRelease"] - ? node["latestLlamacppRelease"].as() - : default_cfg.latestLlamacppRelease, - .huggingFaceToken = node["huggingFaceToken"] - ? node["huggingFaceToken"].as() - : default_cfg.huggingFaceToken, - .gitHubUserAgent = node["gitHubUserAgent"] - ? node["gitHubUserAgent"].as() - : default_cfg.gitHubUserAgent, - .gitHubToken = node["gitHubToken"] - ? node["gitHubToken"].as() - : default_cfg.gitHubToken, - .llamacppVariant = node["llamacppVariant"] - ? node["llamacppVariant"].as() - : default_cfg.llamacppVariant, - .llamacppVersion = node["llamacppVersion"] - ? node["llamacppVersion"].as() - : default_cfg.llamacppVersion, - .enableCors = node["enableCors"] ? node["enableCors"].as() - : default_cfg.enableCors, - .allowedOrigins = - node["allowedOrigins"] - ? node["allowedOrigins"].as>() - : default_cfg.allowedOrigins, - .proxyUrl = node["proxyUrl"] ? node["proxyUrl"].as() - : default_cfg.proxyUrl, - .verifyProxySsl = node["verifyProxySsl"] - ? node["verifyProxySsl"].as() - : default_cfg.verifyProxySsl, - .verifyProxyHostSsl = node["verifyProxyHostSsl"] - ? node["verifyProxyHostSsl"].as() - : default_cfg.verifyProxyHostSsl, - .proxyUsername = node["proxyUsername"] - ? node["proxyUsername"].as() - : default_cfg.proxyUsername, - .proxyPassword = node["proxyPassword"] - ? node["proxyPassword"].as() - : default_cfg.proxyPassword, - .noProxy = node["noProxy"] ? node["noProxy"].as() - : default_cfg.noProxy, - .verifyPeerSsl = node["verifyPeerSsl"] - ? node["verifyPeerSsl"].as() - : default_cfg.verifyPeerSsl, - .verifyHostSsl = node["verifyHostSsl"] - ? node["verifyHostSsl"].as() - : default_cfg.verifyHostSsl, - }; - if (should_update_config) { - l.unlock(); - auto result = DumpYamlConfig(config, path); - if (result.has_error()) { - CTL_ERR("Failed to update config file: " << result.error()); - } - } - return config; - } catch (const YAML::BadFile& e) { - CTL_ERR("Failed to read file: " << e.what()); - throw; - } - } + const CortexConfig& default_cfg); }; } // namespace config_yaml_utils diff --git a/engine/utils/curl_utils.cc b/engine/utils/curl_utils.cc new file mode 100644 index 000000000..71f263a6a --- /dev/null +++ b/engine/utils/curl_utils.cc @@ -0,0 +1,321 @@ +#include "curl_utils.h" + +#include "utils/engine_constants.h" +#include "utils/file_manager_utils.h" +#include "utils/logging_utils.h" + +#include "utils/string_utils.h" +#include "utils/url_parser.h" + +namespace curl_utils { +namespace { +size_t WriteCallback(void* contents, size_t size, size_t nmemb, + std::string* output) { + size_t totalSize = size * nmemb; + output->append((char*)contents, totalSize); + return totalSize; +} + +void SetUpProxy(CURL* handle, const std::string& url) { + auto config = file_manager_utils::GetCortexConfig(); + if (!config.proxyUrl.empty()) { + auto proxy_url = config.proxyUrl; + auto verify_proxy_ssl = config.verifyProxySsl; + auto verify_proxy_host_ssl = config.verifyProxyHostSsl; + + auto verify_ssl = config.verifyPeerSsl; + auto verify_host_ssl = config.verifyHostSsl; + + auto proxy_username = config.proxyUsername; + auto proxy_password = config.proxyPassword; + auto no_proxy = config.noProxy; + + CTL_INF("=== Proxy configuration ==="); + CTL_INF("Request url: " << url); + CTL_INF("Proxy url: " << proxy_url); + CTL_INF("Verify proxy ssl: " << verify_proxy_ssl); + CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl); + CTL_INF("Verify ssl: " << verify_ssl); + CTL_INF("Verify host ssl: " << verify_host_ssl); + CTL_INF("No proxy: " << no_proxy); + + curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str()); + if (string_utils::StartsWith(proxy_url, "https")) { + curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTPS); + } + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, verify_host_ssl ? 2L : 0L); + + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, + verify_proxy_ssl ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, + verify_proxy_host_ssl ? 2L : 0L); + + auto proxy_auth = proxy_username + ":" + proxy_password; + curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str()); + + curl_easy_setopt(handle, CURLOPT_NOPROXY, no_proxy.c_str()); + } +} +} // namespace + +std::optional> GetHeaders( + const std::string& url) { + auto url_obj = url_parser::FromUrlString(url); + if (url_obj.has_error()) { + return std::nullopt; + } + + if (url_obj->host == kHuggingFaceHost) { + std::unordered_map headers{}; + headers["Content-Type"] = "application/json"; + auto const& token = file_manager_utils::GetCortexConfig().huggingFaceToken; + if (!token.empty()) { + headers["Authorization"] = "Bearer " + token; + + // for debug purpose + auto min_token_size = 6; + if (token.size() < min_token_size) { + CTL_WRN("Hugging Face token is too short"); + } else { + CTL_INF("Using authentication with Hugging Face token: " + + token.substr(token.size() - min_token_size)); + } + } + + return headers; + } + + if (url_obj->host == kGitHubHost) { + std::unordered_map headers{}; + headers["Accept"] = "application/vnd.github.v3+json"; + // github API requires user-agent https://docs.github.com/en/rest/using-the-rest-api/getting-started-with-the-rest-api?apiVersion=2022-11-28#user-agent + auto user_agent = file_manager_utils::GetCortexConfig().gitHubUserAgent; + auto gh_token = file_manager_utils::GetCortexConfig().gitHubToken; + headers["User-Agent"] = + user_agent.empty() ? kDefaultGHUserAgent : user_agent; + if (!gh_token.empty()) { + headers["Authorization"] = "Bearer " + gh_token; + + // for debug purpose + auto min_token_size = 6; + if (gh_token.size() < min_token_size) { + CTL_WRN("Github token is too short"); + } else { + CTL_INF("Using authentication with Github token: " + + gh_token.substr(gh_token.size() - min_token_size)); + } + } + return headers; + } + + return std::nullopt; +} + +cpp::result SimpleGet(const std::string& url, + const int timeout) { + auto curl = curl_easy_init(); + + if (!curl) { + return cpp::fail("Failed to init CURL"); + } + + auto headers = GetHeaders(url); + curl_slist* curl_headers = nullptr; + if (headers.has_value()) { + for (const auto& [key, value] : headers.value()) { + auto header = key + ": " + value; + curl_headers = curl_slist_append(curl_headers, header.c_str()); + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); + } + + std::string readBuffer; + + SetUpProxy(curl, url); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + if (timeout > 0) { + curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); + } + + // Perform the request + auto res = curl_easy_perform(curl); + + curl_slist_free_all(curl_headers); + curl_easy_cleanup(curl); + if (res != CURLE_OK) { + return cpp::fail("CURL request failed: " + + static_cast(curl_easy_strerror(res))); + } + auto http_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code >= 400) { + CTL_ERR("HTTP request failed with status code: " + + std::to_string(http_code)); + return cpp::fail(readBuffer); + } + + return readBuffer; +} + +cpp::result SimpleRequest( + const std::string& url, const RequestType& request_type, + const std::string& body) { + auto curl = curl_easy_init(); + + if (!curl) { + return cpp::fail("Failed to init CURL"); + } + + auto headers = GetHeaders(url); + curl_slist* curl_headers = nullptr; + curl_headers = + curl_slist_append(curl_headers, "Content-Type: application/json"); + curl_headers = curl_slist_append(curl_headers, "Expect:"); + + if (headers.has_value()) { + for (const auto& [key, value] : headers.value()) { + auto header = key + ": " + value; + curl_headers = curl_slist_append(curl_headers, header.c_str()); + } + } + std::string readBuffer; + + SetUpProxy(curl, url); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + if (request_type == RequestType::PATCH) { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PATCH"); + } else if (request_type == RequestType::POST) { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + } else if (request_type == RequestType::DEL) { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + } + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.length()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + + // Perform the request + auto res = curl_easy_perform(curl); + + auto http_code = 0L; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + + // Clean up + curl_slist_free_all(curl_headers); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) { + CTL_ERR("CURL request failed: " + std::string(curl_easy_strerror(res))); + return cpp::fail("CURL request failed: " + + static_cast(curl_easy_strerror(res))); + } + + if (http_code >= 400) { + CTL_ERR("HTTP request failed with status code: " + + std::to_string(http_code)); + return cpp::fail(readBuffer); + } + + return readBuffer; +} + +cpp::result ReadRemoteYaml(const std::string& url) { + auto result = SimpleGet(url); + if (result.has_error()) { + CTL_ERR("Failed to get Yaml from " + url + ": " + result.error()); + return cpp::fail(result.error()); + } + + try { + return YAML::Load(result.value()); + } catch (const std::exception& e) { + return cpp::fail("YAML from " + url + + " parsing error: " + std::string(e.what())); + } +} + +cpp::result SimpleGetJson(const std::string& url, + const int timeout) { + auto result = SimpleGet(url, timeout); + if (result.has_error()) { + CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); + return cpp::fail(result.error()); + } + + Json::Value root; + Json::Reader reader; + if (!reader.parse(result.value(), root)) { + return cpp::fail("JSON from " + url + + " parsing error: " + reader.getFormattedErrorMessages()); + } + + return root; +} + +cpp::result SimplePostJson(const std::string& url, + const std::string& body) { + auto result = SimpleRequest(url, RequestType::POST, body); + if (result.has_error()) { + CTL_INF("url: " + url); + CTL_INF("body: " + body); + CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); + return cpp::fail(result.error()); + } + + CTL_INF("Response: " + result.value()); + Json::Value root; + Json::Reader reader; + if (!reader.parse(result.value(), root)) { + return cpp::fail("JSON from " + url + + " parsing error: " + reader.getFormattedErrorMessages()); + } + + return root; +} + +cpp::result SimpleDeleteJson( + const std::string& url, const std::string& body) { + auto result = SimpleRequest(url, RequestType::DEL, body); + if (result.has_error()) { + CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); + return cpp::fail(result.error()); + } + + CTL_INF("Response: " + result.value()); + Json::Value root; + Json::Reader reader; + if (!reader.parse(result.value(), root)) { + return cpp::fail("JSON from " + url + + " parsing error: " + reader.getFormattedErrorMessages()); + } + + return root; +} + +cpp::result SimplePatchJson(const std::string& url, + const std::string& body) { + auto result = SimpleRequest(url, RequestType::PATCH, body); + if (result.has_error()) { + CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); + return cpp::fail(result.error()); + } + + CTL_INF("Response: " + result.value()); + Json::Value root; + Json::Reader reader; + if (!reader.parse(result.value(), root)) { + return cpp::fail("JSON from " + url + + " parsing error: " + reader.getFormattedErrorMessages()); + } + + return root; +} +} // namespace curl_utils \ No newline at end of file diff --git a/engine/utils/curl_utils.h b/engine/utils/curl_utils.h index c56808b56..64b5fc339 100644 --- a/engine/utils/curl_utils.h +++ b/engine/utils/curl_utils.h @@ -5,335 +5,43 @@ #include #include #include +#include #include -#include "utils/engine_constants.h" -#include "utils/file_manager_utils.h" -#include "utils/logging_utils.h" +#include + #include "utils/result.hpp" -#include "utils/string_utils.h" -#include "utils/url_parser.h" enum class RequestType { GET, PATCH, POST, DEL }; namespace curl_utils { -namespace { -size_t WriteCallback(void* contents, size_t size, size_t nmemb, - std::string* output) { - size_t totalSize = size * nmemb; - output->append((char*)contents, totalSize); - return totalSize; -} - -void SetUpProxy(CURL* handle, const std::string& url) { - auto config = file_manager_utils::GetCortexConfig(); - if (!config.proxyUrl.empty()) { - auto proxy_url = config.proxyUrl; - auto verify_proxy_ssl = config.verifyProxySsl; - auto verify_proxy_host_ssl = config.verifyProxyHostSsl; - - auto verify_ssl = config.verifyPeerSsl; - auto verify_host_ssl = config.verifyHostSsl; - - auto proxy_username = config.proxyUsername; - auto proxy_password = config.proxyPassword; - auto no_proxy = config.noProxy; - - CTL_INF("=== Proxy configuration ==="); - CTL_INF("Request url: " << url); - CTL_INF("Proxy url: " << proxy_url); - CTL_INF("Verify proxy ssl: " << verify_proxy_ssl); - CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl); - CTL_INF("Verify ssl: " << verify_ssl); - CTL_INF("Verify host ssl: " << verify_host_ssl); - CTL_INF("No proxy: " << no_proxy); - - curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str()); - if (string_utils::StartsWith(proxy_url, "https")) { - curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTPS); - } - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, verify_host_ssl ? 2L : 0L); - - curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, - verify_proxy_ssl ? 1L : 0L); - curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, - verify_proxy_host_ssl ? 2L : 0L); - - auto proxy_auth = proxy_username + ":" + proxy_password; - curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str()); - - curl_easy_setopt(handle, CURLOPT_NOPROXY, no_proxy.c_str()); - } -} -} // namespace - -inline std::optional> GetHeaders( +std::optional> GetHeaders( const std::string& url); -inline cpp::result SimpleGet(const std::string& url, - const int timeout = -1) { - auto curl = curl_easy_init(); - - if (!curl) { - return cpp::fail("Failed to init CURL"); - } +cpp::result SimpleGet(const std::string& url, + const int timeout = -1); - auto headers = GetHeaders(url); - curl_slist* curl_headers = nullptr; - if (headers.has_value()) { - for (const auto& [key, value] : headers.value()) { - auto header = key + ": " + value; - curl_headers = curl_slist_append(curl_headers, header.c_str()); - } - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); - } - - std::string readBuffer; - - SetUpProxy(curl, url); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); - if (timeout > 0) { - curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); - } - - // Perform the request - auto res = curl_easy_perform(curl); - - curl_slist_free_all(curl_headers); - curl_easy_cleanup(curl); - if (res != CURLE_OK) { - return cpp::fail("CURL request failed: " + - static_cast(curl_easy_strerror(res))); - } - auto http_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - if (http_code >= 400) { - CTL_ERR("HTTP request failed with status code: " + - std::to_string(http_code)); - return cpp::fail(readBuffer); - } - - return readBuffer; -} - -inline cpp::result SimpleRequest( +cpp::result SimpleRequest( const std::string& url, const RequestType& request_type, - const std::string& body = "") { - auto curl = curl_easy_init(); - - if (!curl) { - return cpp::fail("Failed to init CURL"); - } - - auto headers = GetHeaders(url); - curl_slist* curl_headers = nullptr; - curl_headers = - curl_slist_append(curl_headers, "Content-Type: application/json"); - curl_headers = curl_slist_append(curl_headers, "Expect:"); - - if (headers.has_value()) { - for (const auto& [key, value] : headers.value()) { - auto header = key + ": " + value; - curl_headers = curl_slist_append(curl_headers, header.c_str()); - } - } - std::string readBuffer; - - SetUpProxy(curl, url); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - if (request_type == RequestType::PATCH) { - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PATCH"); - } else if (request_type == RequestType::POST) { - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - } else if (request_type == RequestType::DEL) { - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - } - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + const std::string& body = ""); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.length()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - - // Perform the request - auto res = curl_easy_perform(curl); - - auto http_code = 0L; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - - // Clean up - curl_slist_free_all(curl_headers); - curl_easy_cleanup(curl); - - if (res != CURLE_OK) { - CTL_ERR("CURL request failed: " + std::string(curl_easy_strerror(res))); - return cpp::fail("CURL request failed: " + - static_cast(curl_easy_strerror(res))); - } - - if (http_code >= 400) { - CTL_ERR("HTTP request failed with status code: " + - std::to_string(http_code)); - return cpp::fail(readBuffer); - } - - return readBuffer; -} - -inline cpp::result ReadRemoteYaml( - const std::string& url) { - auto result = SimpleGet(url); - if (result.has_error()) { - CTL_ERR("Failed to get Yaml from " + url + ": " + result.error()); - return cpp::fail(result.error()); - } - - try { - return YAML::Load(result.value()); - } catch (const std::exception& e) { - return cpp::fail("YAML from " + url + - " parsing error: " + std::string(e.what())); - } -} +cpp::result ReadRemoteYaml(const std::string& url); /** * SimpleGetJson is a helper function that sends a GET request to the given URL * * [timeout] is an optional parameter that specifies the timeout for the request. In second. */ -inline cpp::result SimpleGetJson( - const std::string& url, const int timeout = -1) { - auto result = SimpleGet(url, timeout); - if (result.has_error()) { - CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); - return cpp::fail(result.error()); - } - - Json::Value root; - Json::Reader reader; - if (!reader.parse(result.value(), root)) { - return cpp::fail("JSON from " + url + - " parsing error: " + reader.getFormattedErrorMessages()); - } - - return root; -} - -inline cpp::result SimplePostJson( - const std::string& url, const std::string& body = "") { - auto result = SimpleRequest(url, RequestType::POST, body); - if (result.has_error()) { - CTL_INF("url: " + url); - CTL_INF("body: " + body); - CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); - return cpp::fail(result.error()); - } - - CTL_INF("Response: " + result.value()); - Json::Value root; - Json::Reader reader; - if (!reader.parse(result.value(), root)) { - return cpp::fail("JSON from " + url + - " parsing error: " + reader.getFormattedErrorMessages()); - } - - return root; -} - -inline cpp::result SimpleDeleteJson( - const std::string& url, const std::string& body = "") { - auto result = SimpleRequest(url, RequestType::DEL, body); - if (result.has_error()) { - CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); - return cpp::fail(result.error()); - } - - CTL_INF("Response: " + result.value()); - Json::Value root; - Json::Reader reader; - if (!reader.parse(result.value(), root)) { - return cpp::fail("JSON from " + url + - " parsing error: " + reader.getFormattedErrorMessages()); - } - - return root; -} - -inline cpp::result SimplePatchJson( - const std::string& url, const std::string& body = "") { - auto result = SimpleRequest(url, RequestType::PATCH, body); - if (result.has_error()) { - CTL_ERR("Failed to get JSON from " + url + ": " + result.error()); - return cpp::fail(result.error()); - } - - CTL_INF("Response: " + result.value()); - Json::Value root; - Json::Reader reader; - if (!reader.parse(result.value(), root)) { - return cpp::fail("JSON from " + url + - " parsing error: " + reader.getFormattedErrorMessages()); - } - - return root; -} - -inline std::optional> GetHeaders( - const std::string& url) { - auto url_obj = url_parser::FromUrlString(url); - if (url_obj.has_error()) { - return std::nullopt; - } - - if (url_obj->host == kHuggingFaceHost) { - std::unordered_map headers{}; - headers["Content-Type"] = "application/json"; - auto const& token = file_manager_utils::GetCortexConfig().huggingFaceToken; - if (!token.empty()) { - headers["Authorization"] = "Bearer " + token; - - // for debug purpose - auto min_token_size = 6; - if (token.size() < min_token_size) { - CTL_WRN("Hugging Face token is too short"); - } else { - CTL_INF("Using authentication with Hugging Face token: " + - token.substr(token.size() - min_token_size)); - } - } +cpp::result SimpleGetJson(const std::string& url, + const int timeout = -1); - return headers; - } +cpp::result SimplePostJson( + const std::string& url, const std::string& body = ""); - if (url_obj->host == kGitHubHost) { - std::unordered_map headers{}; - headers["Accept"] = "application/vnd.github.v3+json"; - // github API requires user-agent https://docs.github.com/en/rest/using-the-rest-api/getting-started-with-the-rest-api?apiVersion=2022-11-28#user-agent - auto user_agent = file_manager_utils::GetCortexConfig().gitHubUserAgent; - auto gh_token = file_manager_utils::GetCortexConfig().gitHubToken; - headers["User-Agent"] = - user_agent.empty() ? kDefaultGHUserAgent : user_agent; - if (!gh_token.empty()) { - headers["Authorization"] = "Bearer " + gh_token; +cpp::result SimpleDeleteJson( + const std::string& url, const std::string& body = ""); - // for debug purpose - auto min_token_size = 6; - if (gh_token.size() < min_token_size) { - CTL_WRN("Github token is too short"); - } else { - CTL_INF("Using authentication with Github token: " + - gh_token.substr(gh_token.size() - min_token_size)); - } - } - return headers; - } +cpp::result SimplePatchJson( + const std::string& url, const std::string& body = ""); - return std::nullopt; -} } // namespace curl_utils diff --git a/engine/utils/file_manager_utils.cc b/engine/utils/file_manager_utils.cc new file mode 100644 index 000000000..9650dd973 --- /dev/null +++ b/engine/utils/file_manager_utils.cc @@ -0,0 +1,367 @@ +#include "file_manager_utils.h" + +#include "logging_utils.h" + +#include "utils/engine_constants.h" +#include "utils/result.hpp" +#include "utils/widechar_conv.h" + +#if defined(__APPLE__) && defined(__MACH__) +#include +#elif defined(__linux__) +#include +#elif defined(_WIN32) +#include +#include +#include +#endif + +namespace file_manager_utils { +std::filesystem::path GetExecutableFolderContainerPath() { +#if defined(__APPLE__) && defined(__MACH__) + char buffer[1024]; + uint32_t size = sizeof(buffer); + + if (_NSGetExecutablePath(buffer, &size) == 0) { + // CTL_DBG("Executable path: " << buffer); + return std::filesystem::path{buffer}.parent_path(); + } else { + CTL_ERR("Failed to get executable path"); + return std::filesystem::current_path(); + } +#elif defined(__linux__) + char buffer[1024]; + ssize_t len = readlink("/proc/self/exe", buffer, sizeof(buffer) - 1); + if (len != -1) { + buffer[len] = '\0'; + // CTL_DBG("Executable path: " << buffer); + return std::filesystem::path{buffer}.parent_path(); + } else { + CTL_ERR("Failed to get executable path"); + return std::filesystem::current_path(); + } +#elif defined(_WIN32) + wchar_t buffer[MAX_PATH]; + GetModuleFileNameW(NULL, buffer, MAX_PATH); + // CTL_DBG("Executable path: " << buffer); + return std::filesystem::path{buffer}.parent_path(); +#else + LOG_ERROR << "Unsupported platform!"; + return std::filesystem::current_path(); +#endif +} + +std::filesystem::path GetHomeDirectoryPath() { +#ifdef _WIN32 + const wchar_t* homeDir = _wgetenv(L"USERPROFILE"); + if (!homeDir) { + // Fallback if USERPROFILE is not set + const wchar_t* homeDrive = _wgetenv(L"HOMEDRIVE"); + const wchar_t* homePath = _wgetenv(L"HOMEPATH"); + if (homeDrive && homePath) { + return std::filesystem::path(homeDrive) / std::filesystem::path(homePath); + } else { + throw std::runtime_error("Cannot determine the home directory"); + } + } +#else + const char* homeDir = std::getenv("HOME"); + if (!homeDir) { + throw std::runtime_error("Cannot determine the home directory"); + } +#endif + return std::filesystem::path(homeDir); +} + +std::filesystem::path GetConfigurationPath() { +#ifndef CORTEX_CONFIG_FILE_PATH +#define CORTEX_CONFIG_FILE_PATH kDefaultConfigurationPath +#endif + +#ifndef CORTEX_VARIANT +#define CORTEX_VARIANT kProdVariant +#endif + std::string config_file_path; + if (cortex_config_file_path.empty()) { + config_file_path = CORTEX_CONFIG_FILE_PATH; + } else { + config_file_path = cortex_config_file_path; + } + + if (config_file_path != kDefaultConfigurationPath) { +// CTL_INF("Config file path: " + config_file_path); +#if defined(_WIN32) + return std::filesystem::u8path(config_file_path); +#else + return std::filesystem::path(config_file_path); +#endif + } + + std::string variant{CORTEX_VARIANT}; + std::string env_postfix{""}; + if (variant == kBetaVariant) { + env_postfix.append("-").append(kBetaVariant); + } else if (variant == kNightlyVariant) { + env_postfix.append("-").append(kNightlyVariant); + } + + std::string config_file_name{kCortexConfigurationFileName}; + config_file_name.append(env_postfix); + // CTL_INF("Config file name: " + config_file_name); + + auto home_path = GetHomeDirectoryPath(); + auto configuration_path = home_path / config_file_name; + return configuration_path; +} + +std::string GetDefaultDataFolderName() { +#ifndef CORTEX_VARIANT +#define CORTEX_VARIANT "prod" +#endif + std::string default_data_folder_name{kCortexFolderName}; + std::string variant{CORTEX_VARIANT}; + std::string env_postfix{""}; + if (variant == kBetaVariant) { + env_postfix.append("-").append(kBetaVariant); + } else if (variant == kNightlyVariant) { + env_postfix.append("-").append(kNightlyVariant); + } + default_data_folder_name.append(env_postfix); + return default_data_folder_name; +} + +cpp::result UpdateCortexConfig( + const config_yaml_utils::CortexConfig& config) { + auto config_path = GetConfigurationPath(); + if (!std::filesystem::exists(config_path)) { + CTL_ERR("Config file not found: " << config_path.string()); + return cpp::fail("Config file not found: " + config_path.string()); + } + + return cyu::CortexConfigMgr::GetInstance().DumpYamlConfig( + config, config_path.string()); +} + +config_yaml_utils::CortexConfig GetDefaultConfig() { + auto config_path = GetConfigurationPath(); + auto default_data_folder_name = GetDefaultDataFolderName(); + auto default_data_folder_path = + cortex_data_folder_path.empty() + ? file_manager_utils::GetHomeDirectoryPath() / + default_data_folder_name + : std::filesystem::path(cortex_data_folder_path); + + return config_yaml_utils::CortexConfig{ +#if defined(_WIN32) + .logFolderPath = + cortex::wc::WstringToUtf8(default_data_folder_path.wstring()), +#else + .logFolderPath = default_data_folder_path.string(), +#endif + .logLlamaCppPath = kLogsLlamacppBaseName, + .logTensorrtLLMPath = kLogsTensorrtllmBaseName, + .logOnnxPath = kLogsOnnxBaseName, +#if defined(_WIN32) + .dataFolderPath = + cortex::wc::WstringToUtf8(default_data_folder_path.wstring()), +#else + .dataFolderPath = default_data_folder_path.string(), +#endif + .maxLogLines = config_yaml_utils::kDefaultMaxLines, + .apiServerHost = config_yaml_utils::kDefaultHost, + .apiServerPort = config_yaml_utils::kDefaultPort, + .checkedForUpdateAt = config_yaml_utils::kDefaultCheckedForUpdateAt, + .checkedForLlamacppUpdateAt = + config_yaml_utils::kDefaultCheckedForLlamacppUpdateAt, + .latestRelease = config_yaml_utils::kDefaultLatestRelease, + .latestLlamacppRelease = config_yaml_utils::kDefaultLatestLlamacppRelease, + .enableCors = config_yaml_utils::kDefaultCorsEnabled, + .allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins, + .proxyUrl = "", + .verifyProxySsl = true, + .verifyProxyHostSsl = true, + .proxyUsername = "", + .proxyPassword = "", + .noProxy = config_yaml_utils::kDefaultNoProxy, + .verifyPeerSsl = true, + .verifyHostSsl = true, + }; +} + +cpp::result CreateConfigFileIfNotExist() { + auto config_path = GetConfigurationPath(); + if (std::filesystem::exists(config_path)) { + // already exists, no need to create + return {}; + } + + CLI_LOG("Config file not found. Creating one at " + config_path.string()); + auto config = GetDefaultConfig(); + CLI_LOG("Default data folder path: " + config.dataFolderPath); + return cyu::CortexConfigMgr::GetInstance().DumpYamlConfig( + config, config_path.string()); +} + +config_yaml_utils::CortexConfig GetCortexConfig() { + auto config_path = GetConfigurationPath(); + + auto default_cfg = GetDefaultConfig(); + return config_yaml_utils::CortexConfigMgr::GetInstance().FromYaml( + config_path.string(), default_cfg); +} + +std::filesystem::path GetCortexDataPath() { + auto result = CreateConfigFileIfNotExist(); + if (result.has_error()) { + CTL_ERR("Error creating config file: " << result.error()); + return std::filesystem::path{}; + } + + auto config = GetCortexConfig(); + std::filesystem::path data_folder_path; + if (!config.dataFolderPath.empty()) { +#if defined(_WIN32) + data_folder_path = std::filesystem::u8path(config.dataFolderPath); +#else + data_folder_path = std::filesystem::path(config.dataFolderPath); +#endif + } else { + auto home_path = GetHomeDirectoryPath(); + data_folder_path = home_path / kCortexFolderName; + } + + if (!std::filesystem::exists(data_folder_path)) { + CLI_LOG("Cortex home folder not found. Create one: " + + data_folder_path.string()); + std::filesystem::create_directory(data_folder_path); + } + return data_folder_path; +} + +std::filesystem::path GetCortexLogPath() { + // TODO: We will need to support user to move the data folder to other place. + // TODO: get the variant of cortex. As discussed, we will have: prod, beta, nightly + + // currently we will store cortex data at ~/cortexcpp + auto config = GetCortexConfig(); + std::filesystem::path log_folder_path; + if (!config.logFolderPath.empty()) { + log_folder_path = std::filesystem::path(config.logFolderPath); + } else { + auto home_path = GetHomeDirectoryPath(); + log_folder_path = home_path / kCortexFolderName; + } + + if (!std::filesystem::exists(log_folder_path)) { + CTL_INF("Cortex log folder not found. Create one: " + + log_folder_path.string()); + std::filesystem::create_directory(log_folder_path); + } + return log_folder_path; +} + +void CreateDirectoryRecursively(const std::string& path) { + // Create the directories if they don't exist + if (std::filesystem::create_directories(path)) { + CTL_INF(path + " successfully created!"); + } else { + CTL_INF(path + " already exist!"); + } +} + +std::filesystem::path GetModelsContainerPath() { + auto result = CreateConfigFileIfNotExist(); + if (result.has_error()) { + CTL_ERR("Error creating config file: " << result.error()); + } + auto cortex_path = GetCortexDataPath(); + auto models_container_path = cortex_path / "models"; + + if (!std::filesystem::exists(models_container_path)) { + CTL_INF("Model container folder not found. Create one: " + << models_container_path.string()); + std::filesystem::create_directories(models_container_path); + } + + return models_container_path; +} + +std::filesystem::path GetCudaToolkitPath(const std::string& engine) { + auto engine_path = getenv("ENGINE_PATH") + ? std::filesystem::path(getenv("ENGINE_PATH")) + : GetCortexDataPath(); + + auto cuda_path = engine_path / "engines" / engine / "deps"; + if (!std::filesystem::exists(cuda_path)) { + std::filesystem::create_directories(cuda_path); + } + + return cuda_path; +} + +std::filesystem::path GetEnginesContainerPath() { + auto cortex_path = getenv("ENGINE_PATH") + ? std::filesystem::path(getenv("ENGINE_PATH")) + : GetCortexDataPath(); + auto engines_container_path = cortex_path / "engines"; + + if (!std::filesystem::exists(engines_container_path)) { + CTL_INF("Engine container folder not found. Create one: " + << engines_container_path.string()); + std::filesystem::create_directory(engines_container_path); + } + + return engines_container_path; +} + +std::filesystem::path GetContainerFolderPath(const std::string_view type) { + std::filesystem::path container_folder_path; + + if (type == "Model") { + container_folder_path = GetModelsContainerPath(); + } else if (type == "Engine") { + container_folder_path = GetEnginesContainerPath(); + } else if (type == "CudaToolkit") { + container_folder_path = + std::filesystem::temp_directory_path() / "cuda-dependencies"; + } else if (type == "Cortex") { + container_folder_path = std::filesystem::temp_directory_path() / "cortex"; + } else { + container_folder_path = std::filesystem::temp_directory_path() / "misc"; + } + + if (!std::filesystem::exists(container_folder_path)) { + CTL_INF("Creating folder: " << container_folder_path.string() << "\n"); + std::filesystem::create_directories(container_folder_path); + } + + return container_folder_path; +} + +std::string DownloadTypeToString(DownloadType type) { + switch (type) { + case DownloadType::Model: + return "Model"; + case DownloadType::Engine: + return "Engine"; + case DownloadType::Miscellaneous: + return "Misc"; + case DownloadType::CudaToolkit: + return "CudaToolkit"; + case DownloadType::Cortex: + return "Cortex"; + default: + return "UNKNOWN"; + } +} + +std::filesystem::path ToRelativeCortexDataPath( + const std::filesystem::path& path) { + return Subtract(path, GetCortexDataPath()); +} + +std::filesystem::path ToAbsoluteCortexDataPath( + const std::filesystem::path& path) { + return GetAbsolutePath(GetCortexDataPath(), path); +} +} // namespace file_manager_utils \ No newline at end of file diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index 72310385c..a7a1b09c2 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -3,21 +3,7 @@ #include #include #include "common/download_task.h" -#include "logging_utils.h" #include "utils/config_yaml_utils.h" -#include "utils/engine_constants.h" -#include "utils/result.hpp" -#include "utils/widechar_conv.h" - -#if defined(__APPLE__) && defined(__MACH__) -#include -#elif defined(__linux__) -#include -#elif defined(_WIN32) -#include -#include -#include -#endif namespace file_manager_utils { namespace cyu = config_yaml_utils; @@ -34,344 +20,38 @@ inline std::string cortex_config_file_path; inline std::string cortex_data_folder_path; -inline std::filesystem::path GetExecutableFolderContainerPath() { -#if defined(__APPLE__) && defined(__MACH__) - char buffer[1024]; - uint32_t size = sizeof(buffer); - - if (_NSGetExecutablePath(buffer, &size) == 0) { - // CTL_DBG("Executable path: " << buffer); - return std::filesystem::path{buffer}.parent_path(); - } else { - CTL_ERR("Failed to get executable path"); - return std::filesystem::current_path(); - } -#elif defined(__linux__) - char buffer[1024]; - ssize_t len = readlink("/proc/self/exe", buffer, sizeof(buffer) - 1); - if (len != -1) { - buffer[len] = '\0'; - // CTL_DBG("Executable path: " << buffer); - return std::filesystem::path{buffer}.parent_path(); - } else { - CTL_ERR("Failed to get executable path"); - return std::filesystem::current_path(); - } -#elif defined(_WIN32) - wchar_t buffer[MAX_PATH]; - GetModuleFileNameW(NULL, buffer, MAX_PATH); - // CTL_DBG("Executable path: " << buffer); - return std::filesystem::path{buffer}.parent_path(); -#else - LOG_ERROR << "Unsupported platform!"; - return std::filesystem::current_path(); -#endif -} - -inline std::filesystem::path GetHomeDirectoryPath() { -#ifdef _WIN32 - const wchar_t* homeDir = _wgetenv(L"USERPROFILE"); - if (!homeDir) { - // Fallback if USERPROFILE is not set - const wchar_t* homeDrive = _wgetenv(L"HOMEDRIVE"); - const wchar_t* homePath = _wgetenv(L"HOMEPATH"); - if (homeDrive && homePath) { - return std::filesystem::path(homeDrive) / std::filesystem::path(homePath); - } else { - throw std::runtime_error("Cannot determine the home directory"); - } - } -#else - const char* homeDir = std::getenv("HOME"); - if (!homeDir) { - throw std::runtime_error("Cannot determine the home directory"); - } -#endif - return std::filesystem::path(homeDir); -} - -inline std::filesystem::path GetConfigurationPath() { -#ifndef CORTEX_CONFIG_FILE_PATH -#define CORTEX_CONFIG_FILE_PATH kDefaultConfigurationPath -#endif - -#ifndef CORTEX_VARIANT -#define CORTEX_VARIANT kProdVariant -#endif - std::string config_file_path; - if (cortex_config_file_path.empty()) { - config_file_path = CORTEX_CONFIG_FILE_PATH; - } else { - config_file_path = cortex_config_file_path; - } - - if (config_file_path != kDefaultConfigurationPath) { -// CTL_INF("Config file path: " + config_file_path); -#if defined(_WIN32) - return std::filesystem::u8path(config_file_path); -#else - return std::filesystem::path(config_file_path); -#endif - } - - std::string variant{CORTEX_VARIANT}; - std::string env_postfix{""}; - if (variant == kBetaVariant) { - env_postfix.append("-").append(kBetaVariant); - } else if (variant == kNightlyVariant) { - env_postfix.append("-").append(kNightlyVariant); - } - - std::string config_file_name{kCortexConfigurationFileName}; - config_file_name.append(env_postfix); - // CTL_INF("Config file name: " + config_file_name); - - auto home_path = GetHomeDirectoryPath(); - auto configuration_path = home_path / config_file_name; - return configuration_path; -} - -inline std::string GetDefaultDataFolderName() { -#ifndef CORTEX_VARIANT -#define CORTEX_VARIANT "prod" -#endif - std::string default_data_folder_name{kCortexFolderName}; - std::string variant{CORTEX_VARIANT}; - std::string env_postfix{""}; - if (variant == kBetaVariant) { - env_postfix.append("-").append(kBetaVariant); - } else if (variant == kNightlyVariant) { - env_postfix.append("-").append(kNightlyVariant); - } - default_data_folder_name.append(env_postfix); - return default_data_folder_name; -} - -inline cpp::result UpdateCortexConfig( - const config_yaml_utils::CortexConfig& config) { - auto config_path = GetConfigurationPath(); - if (!std::filesystem::exists(config_path)) { - CTL_ERR("Config file not found: " << config_path.string()); - return cpp::fail("Config file not found: " + config_path.string()); - } - - return cyu::CortexConfigMgr::GetInstance().DumpYamlConfig( - config, config_path.string()); -} - -inline config_yaml_utils::CortexConfig GetDefaultConfig() { - auto config_path = GetConfigurationPath(); - auto default_data_folder_name = GetDefaultDataFolderName(); - auto default_data_folder_path = - cortex_data_folder_path.empty() - ? file_manager_utils::GetHomeDirectoryPath() / - default_data_folder_name - : std::filesystem::path(cortex_data_folder_path); - - return config_yaml_utils::CortexConfig{ -#if defined(_WIN32) - .logFolderPath = - cortex::wc::WstringToUtf8(default_data_folder_path.wstring()), -#else - .logFolderPath = default_data_folder_path.string(), -#endif - .logLlamaCppPath = kLogsLlamacppBaseName, - .logTensorrtLLMPath = kLogsTensorrtllmBaseName, - .logOnnxPath = kLogsOnnxBaseName, -#if defined(_WIN32) - .dataFolderPath = - cortex::wc::WstringToUtf8(default_data_folder_path.wstring()), -#else - .dataFolderPath = default_data_folder_path.string(), -#endif - .maxLogLines = config_yaml_utils::kDefaultMaxLines, - .apiServerHost = config_yaml_utils::kDefaultHost, - .apiServerPort = config_yaml_utils::kDefaultPort, - .checkedForUpdateAt = config_yaml_utils::kDefaultCheckedForUpdateAt, - .checkedForLlamacppUpdateAt = - config_yaml_utils::kDefaultCheckedForLlamacppUpdateAt, - .latestRelease = config_yaml_utils::kDefaultLatestRelease, - .latestLlamacppRelease = config_yaml_utils::kDefaultLatestLlamacppRelease, - .enableCors = config_yaml_utils::kDefaultCorsEnabled, - .allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins, - .proxyUrl = "", - .verifyProxySsl = true, - .verifyProxyHostSsl = true, - .proxyUsername = "", - .proxyPassword = "", - .noProxy = config_yaml_utils::kDefaultNoProxy, - .verifyPeerSsl = true, - .verifyHostSsl = true, - }; -} - -inline cpp::result CreateConfigFileIfNotExist() { - auto config_path = GetConfigurationPath(); - if (std::filesystem::exists(config_path)) { - // already exists, no need to create - return {}; - } - - CLI_LOG("Config file not found. Creating one at " + config_path.string()); - auto config = GetDefaultConfig(); - CLI_LOG("Default data folder path: " + config.dataFolderPath); - return cyu::CortexConfigMgr::GetInstance().DumpYamlConfig( - config, config_path.string()); -} - -inline config_yaml_utils::CortexConfig GetCortexConfig() { - auto config_path = GetConfigurationPath(); - - auto default_cfg = GetDefaultConfig(); - return config_yaml_utils::CortexConfigMgr::GetInstance().FromYaml( - config_path.string(), default_cfg); -} - -inline std::filesystem::path GetCortexDataPath() { - auto result = CreateConfigFileIfNotExist(); - if (result.has_error()) { - CTL_ERR("Error creating config file: " << result.error()); - return std::filesystem::path{}; - } - - auto config = GetCortexConfig(); - std::filesystem::path data_folder_path; - if (!config.dataFolderPath.empty()) { -#if defined(_WIN32) - data_folder_path = std::filesystem::u8path(config.dataFolderPath); -#else - data_folder_path = std::filesystem::path(config.dataFolderPath); -#endif - } else { - auto home_path = GetHomeDirectoryPath(); - data_folder_path = home_path / kCortexFolderName; - } - - if (!std::filesystem::exists(data_folder_path)) { - CLI_LOG("Cortex home folder not found. Create one: " + - data_folder_path.string()); - std::filesystem::create_directory(data_folder_path); - } - return data_folder_path; -} - -inline std::filesystem::path GetCortexLogPath() { - // TODO: We will need to support user to move the data folder to other place. - // TODO: get the variant of cortex. As discussed, we will have: prod, beta, nightly - - // currently we will store cortex data at ~/cortexcpp - auto config = GetCortexConfig(); - std::filesystem::path log_folder_path; - if (!config.logFolderPath.empty()) { - log_folder_path = std::filesystem::path(config.logFolderPath); - } else { - auto home_path = GetHomeDirectoryPath(); - log_folder_path = home_path / kCortexFolderName; - } - - if (!std::filesystem::exists(log_folder_path)) { - CTL_INF("Cortex log folder not found. Create one: " + - log_folder_path.string()); - std::filesystem::create_directory(log_folder_path); - } - return log_folder_path; -} +std::filesystem::path GetExecutableFolderContainerPath(); -inline void CreateDirectoryRecursively(const std::string& path) { - // Create the directories if they don't exist - if (std::filesystem::create_directories(path)) { - CTL_INF(path + " successfully created!"); - } else { - CTL_INF(path + " already exist!"); - } -} +std::filesystem::path GetHomeDirectoryPath(); -inline std::filesystem::path GetModelsContainerPath() { - auto result = CreateConfigFileIfNotExist(); - if (result.has_error()) { - CTL_ERR("Error creating config file: " << result.error()); - } - auto cortex_path = GetCortexDataPath(); - auto models_container_path = cortex_path / "models"; +std::filesystem::path GetConfigurationPath(); - if (!std::filesystem::exists(models_container_path)) { - CTL_INF("Model container folder not found. Create one: " - << models_container_path.string()); - std::filesystem::create_directories(models_container_path); - } +std::string GetDefaultDataFolderName(); - return models_container_path; -} +cpp::result UpdateCortexConfig( + const config_yaml_utils::CortexConfig& config); -inline std::filesystem::path GetCudaToolkitPath(const std::string& engine) { - auto engine_path = getenv("ENGINE_PATH") - ? std::filesystem::path(getenv("ENGINE_PATH")) - : GetCortexDataPath(); +config_yaml_utils::CortexConfig GetDefaultConfig(); - auto cuda_path = engine_path / "engines" / engine / "deps"; - if (!std::filesystem::exists(cuda_path)) { - std::filesystem::create_directories(cuda_path); - } +cpp::result CreateConfigFileIfNotExist(); - return cuda_path; -} +config_yaml_utils::CortexConfig GetCortexConfig(); -inline std::filesystem::path GetEnginesContainerPath() { - auto cortex_path = getenv("ENGINE_PATH") - ? std::filesystem::path(getenv("ENGINE_PATH")) - : GetCortexDataPath(); - auto engines_container_path = cortex_path / "engines"; +std::filesystem::path GetCortexDataPath(); - if (!std::filesystem::exists(engines_container_path)) { - CTL_INF("Engine container folder not found. Create one: " - << engines_container_path.string()); - std::filesystem::create_directory(engines_container_path); - } +std::filesystem::path GetCortexLogPath(); - return engines_container_path; -} +void CreateDirectoryRecursively(const std::string& path); -inline std::filesystem::path GetContainerFolderPath( - const std::string_view type) { - std::filesystem::path container_folder_path; +std::filesystem::path GetModelsContainerPath(); - if (type == "Model") { - container_folder_path = GetModelsContainerPath(); - } else if (type == "Engine") { - container_folder_path = GetEnginesContainerPath(); - } else if (type == "CudaToolkit") { - container_folder_path = - std::filesystem::temp_directory_path() / "cuda-dependencies"; - } else if (type == "Cortex") { - container_folder_path = std::filesystem::temp_directory_path() / "cortex"; - } else { - container_folder_path = std::filesystem::temp_directory_path() / "misc"; - } +std::filesystem::path GetCudaToolkitPath(const std::string& engine); - if (!std::filesystem::exists(container_folder_path)) { - CTL_INF("Creating folder: " << container_folder_path.string() << "\n"); - std::filesystem::create_directories(container_folder_path); - } +std::filesystem::path GetEnginesContainerPath(); - return container_folder_path; -} +std::filesystem::path GetContainerFolderPath(const std::string_view type); -inline std::string DownloadTypeToString(DownloadType type) { - switch (type) { - case DownloadType::Model: - return "Model"; - case DownloadType::Engine: - return "Engine"; - case DownloadType::Miscellaneous: - return "Misc"; - case DownloadType::CudaToolkit: - return "CudaToolkit"; - case DownloadType::Cortex: - return "Cortex"; - default: - return "UNKNOWN"; - } -} +std::string DownloadTypeToString(DownloadType type); inline std::filesystem::path GetAbsolutePath(const std::filesystem::path& base, const std::filesystem::path& r) { @@ -399,14 +79,10 @@ inline std::filesystem::path Subtract(const std::filesystem::path& path, } } -inline std::filesystem::path ToRelativeCortexDataPath( - const std::filesystem::path& path) { - return Subtract(path, GetCortexDataPath()); -} +std::filesystem::path ToRelativeCortexDataPath( + const std::filesystem::path& path); -inline std::filesystem::path ToAbsoluteCortexDataPath( - const std::filesystem::path& path) { - return GetAbsolutePath(GetCortexDataPath(), path); -} +std::filesystem::path ToAbsoluteCortexDataPath( + const std::filesystem::path& path); } // namespace file_manager_utils diff --git a/engine/utils/huggingface_utils.h b/engine/utils/huggingface_utils.h index 99df2aa77..f2895c363 100644 --- a/engine/utils/huggingface_utils.h +++ b/engine/utils/huggingface_utils.h @@ -5,6 +5,7 @@ #include #include #include "utils/curl_utils.h" +#include "utils/engine_constants.h" #include "utils/json_parser_utils.h" #include "utils/result.hpp" #include "utils/url_parser.h" diff --git a/engine/utils/system_info_utils.cc b/engine/utils/system_info_utils.cc new file mode 100644 index 000000000..e80bce035 --- /dev/null +++ b/engine/utils/system_info_utils.cc @@ -0,0 +1,141 @@ +#include "system_info_utils.h" +#include "utils/logging_utils.h" + +namespace system_info_utils { +std::pair GetDriverAndCudaVersion() { + if (!IsNvidiaSmiAvailable()) { + CTL_INF("nvidia-smi is not available!"); + return {}; + } + try { + std::string driver_version; + std::string cuda_version; + CommandExecutor cmd("nvidia-smi"); + auto output = cmd.execute(); + + const std::regex driver_version_reg(kDriverVersionRegex); + std::smatch driver_match; + + if (std::regex_search(output, driver_match, driver_version_reg)) { + LOG_INFO << "Gpu Driver Version: " << driver_match[1].str(); + driver_version = driver_match[1].str(); + } else { + LOG_ERROR << "Gpu Driver not found!"; + return {}; + } + + const std::regex cuda_version_reg(kCudaVersionRegex); + std::smatch cuda_match; + + if (std::regex_search(output, cuda_match, cuda_version_reg)) { + LOG_INFO << "CUDA Version: " << cuda_match[1].str(); + cuda_version = cuda_match[1].str(); + } else { + LOG_ERROR << "CUDA Version not found!"; + return {}; + } + return std::pair(driver_version, cuda_version); + } catch (const std::exception& e) { + LOG_ERROR << "Error: " << e.what(); + return {}; + } +} + +std::vector GetGpuInfoListVulkan() { + std::vector gpuInfoList; + + try { + // NOTE: current ly we don't have logic to download vulkaninfoSDK +#ifdef _WIN32 + CommandExecutor cmd("vulkaninfoSDK.exe --summary"); +#else + CommandExecutor cmd("vulkaninfoSDK --summary"); +#endif + auto output = cmd.execute(); + + // Regular expression patterns to match each field + std::regex gpu_block_reg(R"(GPU(\d+):)"); + std::regex field_pattern(R"(\s*(\w+)\s*=\s*(.*))"); + + std::sregex_iterator iter(output.begin(), output.end(), gpu_block_reg); + std::sregex_iterator end; + + while (iter != end) { + GpuInfo gpuInfo; + + // Extract GPU ID from the GPU block pattern (e.g., GPU0 -> id = "0") + gpuInfo.id = (*iter)[1].str(); + + auto gpu_start_pos = iter->position(0) + iter->length(0); + auto gpu_end_pos = std::next(iter) != end ? std::next(iter)->position(0) + : std::string::npos; + std::string gpu_block = + output.substr(gpu_start_pos, gpu_end_pos - gpu_start_pos); + + std::sregex_iterator field_iter(gpu_block.begin(), gpu_block.end(), + field_pattern); + + while (field_iter != end) { + std::string key = (*field_iter)[1].str(); + std::string value = (*field_iter)[2].str(); + + if (key == "deviceName") + gpuInfo.name = value; + else if (key == "apiVersion") + gpuInfo.compute_cap = value; + + gpuInfo.vram_total = ""; // not available + gpuInfo.arch = GetGpuArch(gpuInfo.name); + + ++field_iter; + } + + gpuInfoList.push_back(gpuInfo); + ++iter; + } + } catch (const std::exception& e) { + LOG_ERROR << "Error: " << e.what(); + } + + return gpuInfoList; +} + +std::vector GetGpuInfoList() { + std::vector gpuInfoList; + if (!IsNvidiaSmiAvailable()) + return gpuInfoList; + try { + auto [driver_version, cuda_version] = GetDriverAndCudaVersion(); + if (driver_version.empty() || cuda_version.empty()) + return gpuInfoList; + + CommandExecutor cmd(kGpuQueryCommand); + auto output = cmd.execute(); + + const std::regex gpu_info_reg(kGpuInfoRegex); + std::smatch match; + std::string::const_iterator search_start(output.cbegin()); + + while ( + std::regex_search(search_start, output.cend(), match, gpu_info_reg)) { + GpuInfo gpuInfo = { + match[1].str(), // id + match[2].str(), // vram_total + match[3].str(), // vram_free + match[4].str(), // name + GetGpuArch(match[4].str()), // arch + driver_version, // driver_version + cuda_version, // cuda_driver_version + match[5].str(), // compute_cap + match[6].str() // uuid + }; + gpuInfoList.push_back(gpuInfo); + search_start = match.suffix().first; + } + } catch (const std::exception& e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + return gpuInfoList; +} +} // namespace system_info_utils \ No newline at end of file diff --git a/engine/utils/system_info_utils.h b/engine/utils/system_info_utils.h index f2fab10cb..0907884be 100644 --- a/engine/utils/system_info_utils.h +++ b/engine/utils/system_info_utils.h @@ -8,7 +8,7 @@ #include #include "utils/command_executor.h" #include "utils/engine_constants.h" -#include "utils/logging_utils.h" + #ifdef _WIN32 #include #endif @@ -101,44 +101,7 @@ inline bool IsNvidiaSmiAvailable() { #endif } -inline std::pair GetDriverAndCudaVersion() { - if (!IsNvidiaSmiAvailable()) { - CTL_INF("nvidia-smi is not available!"); - return {}; - } - try { - std::string driver_version; - std::string cuda_version; - CommandExecutor cmd("nvidia-smi"); - auto output = cmd.execute(); - - const std::regex driver_version_reg(kDriverVersionRegex); - std::smatch driver_match; - - if (std::regex_search(output, driver_match, driver_version_reg)) { - LOG_INFO << "Gpu Driver Version: " << driver_match[1].str(); - driver_version = driver_match[1].str(); - } else { - LOG_ERROR << "Gpu Driver not found!"; - return {}; - } - - const std::regex cuda_version_reg(kCudaVersionRegex); - std::smatch cuda_match; - - if (std::regex_search(output, cuda_match, cuda_version_reg)) { - LOG_INFO << "CUDA Version: " << cuda_match[1].str(); - cuda_version = cuda_match[1].str(); - } else { - LOG_ERROR << "CUDA Version not found!"; - return {}; - } - return std::pair(driver_version, cuda_version); - } catch (const std::exception& e) { - LOG_ERROR << "Error: " << e.what(); - return {}; - } -} +std::pair GetDriverAndCudaVersion(); struct GpuInfo { std::string id; @@ -153,101 +116,7 @@ struct GpuInfo { std::string uuid; }; -inline std::vector GetGpuInfoListVulkan() { - std::vector gpuInfoList; - - try { - // NOTE: current ly we don't have logic to download vulkaninfoSDK -#ifdef _WIN32 - CommandExecutor cmd("vulkaninfoSDK.exe --summary"); -#else - CommandExecutor cmd("vulkaninfoSDK --summary"); -#endif - auto output = cmd.execute(); - - // Regular expression patterns to match each field - std::regex gpu_block_reg(R"(GPU(\d+):)"); - std::regex field_pattern(R"(\s*(\w+)\s*=\s*(.*))"); - - std::sregex_iterator iter(output.begin(), output.end(), gpu_block_reg); - std::sregex_iterator end; - - while (iter != end) { - GpuInfo gpuInfo; - - // Extract GPU ID from the GPU block pattern (e.g., GPU0 -> id = "0") - gpuInfo.id = (*iter)[1].str(); - - auto gpu_start_pos = iter->position(0) + iter->length(0); - auto gpu_end_pos = std::next(iter) != end ? std::next(iter)->position(0) - : std::string::npos; - std::string gpu_block = - output.substr(gpu_start_pos, gpu_end_pos - gpu_start_pos); +std::vector GetGpuInfoListVulkan(); - std::sregex_iterator field_iter(gpu_block.begin(), gpu_block.end(), - field_pattern); - - while (field_iter != end) { - std::string key = (*field_iter)[1].str(); - std::string value = (*field_iter)[2].str(); - - if (key == "deviceName") - gpuInfo.name = value; - else if (key == "apiVersion") - gpuInfo.compute_cap = value; - - gpuInfo.vram_total = ""; // not available - gpuInfo.arch = GetGpuArch(gpuInfo.name); - - ++field_iter; - } - - gpuInfoList.push_back(gpuInfo); - ++iter; - } - } catch (const std::exception& e) { - LOG_ERROR << "Error: " << e.what(); - } - - return gpuInfoList; -} - -inline std::vector GetGpuInfoList() { - std::vector gpuInfoList; - if (!IsNvidiaSmiAvailable()) - return gpuInfoList; - try { - auto [driver_version, cuda_version] = GetDriverAndCudaVersion(); - if (driver_version.empty() || cuda_version.empty()) - return gpuInfoList; - - CommandExecutor cmd(kGpuQueryCommand); - auto output = cmd.execute(); - - const std::regex gpu_info_reg(kGpuInfoRegex); - std::smatch match; - std::string::const_iterator search_start(output.cbegin()); - - while ( - std::regex_search(search_start, output.cend(), match, gpu_info_reg)) { - GpuInfo gpuInfo = { - match[1].str(), // id - match[2].str(), // vram_total - match[3].str(), // vram_free - match[4].str(), // name - GetGpuArch(match[4].str()), // arch - driver_version, // driver_version - cuda_version, // cuda_driver_version - match[5].str(), // compute_cap - match[6].str() // uuid - }; - gpuInfoList.push_back(gpuInfo); - search_start = match.suffix().first; - } - } catch (const std::exception& e) { - std::cerr << "Error: " << e.what() << std::endl; - } - - return gpuInfoList; -} +std::vector GetGpuInfoList(); } // namespace system_info_utils