From 832c57c6e30f617604ce06385f2756f4ad892b3b Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Mon, 13 May 2024 10:36:11 +0700 Subject: [PATCH 01/14] feat: engines endpoint and cortex.python --- .../e2e-test-python-runtime-linux-and-mac.sh | 0 .../e2e-test-python-runtime-windows.bat | 0 cortex-cpp/CMakeLists.txt | 1 + cortex-cpp/common/base.h | 11 +- cortex-cpp/controllers/server.cc | 208 ++++++++++++------ cortex-cpp/controllers/server.h | 31 ++- cortex-cpp/cortex-common/EngineI.h | 11 +- cortex-cpp/cortex-common/cortexpythoni.h | 22 ++ cortex-cpp/engines/cortex.python/engine.cmake | 38 ++++ cortex-cpp/main.cc | 23 ++ cortex-cpp/utils/cortex_utils.h | 2 + 11 files changed, 271 insertions(+), 76 deletions(-) create mode 100644 .github/scripts/e2e-test-python-runtime-linux-and-mac.sh create mode 100644 .github/scripts/e2e-test-python-runtime-windows.bat create mode 100644 cortex-cpp/cortex-common/cortexpythoni.h create mode 100644 cortex-cpp/engines/cortex.python/engine.cmake diff --git a/.github/scripts/e2e-test-python-runtime-linux-and-mac.sh b/.github/scripts/e2e-test-python-runtime-linux-and-mac.sh new file mode 100644 index 000000000..e69de29bb diff --git a/.github/scripts/e2e-test-python-runtime-windows.bat b/.github/scripts/e2e-test-python-runtime-windows.bat new file mode 100644 index 000000000..e69de29bb diff --git a/cortex-cpp/CMakeLists.txt b/cortex-cpp/CMakeLists.txt index 8c01d2256..ae9c31ad2 100644 --- a/cortex-cpp/CMakeLists.txt +++ b/cortex-cpp/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.5) project(cortex-cpp C CXX) include(engines/cortex.llamacpp/engine.cmake) +include(engines/cortex.python/engine.cmake) include(CheckIncludeFileCXX) check_include_file_cxx(any HAS_ANY) diff --git a/cortex-cpp/common/base.h b/cortex-cpp/common/base.h index 43d612c1b..3156c54ae 100644 --- a/cortex-cpp/common/base.h +++ b/cortex-cpp/common/base.h @@ -8,14 +8,21 @@ class BaseModel { virtual ~BaseModel() {} // Model management - virtual void LoadModel(const HttpRequestPtr& req, - std::function&& callback) = 0; + virtual void LoadModel( + const HttpRequestPtr& req, + std::function&& callback) = 0; virtual void UnloadModel( const HttpRequestPtr& req, std::function&& callback) = 0; virtual void ModelStatus( const HttpRequestPtr& req, std::function&& callback) = 0; + virtual void GetEngines( + const HttpRequestPtr& req, + std::function&& callback) = 0; + virtual void FineTuning( + const HttpRequestPtr& req, + std::function&& callback) = 0; }; class BaseChatCompletion { diff --git a/cortex-cpp/controllers/server.cc b/cortex-cpp/controllers/server.cc index 143632a3c..bd9c6ee74 100644 --- a/cortex-cpp/controllers/server.cc +++ b/cortex-cpp/controllers/server.cc @@ -13,24 +13,25 @@ using json = nlohmann::json; namespace inferences { namespace { constexpr static auto kLlamaEngine = "cortex.llamacpp"; -constexpr static auto kLlamaLibPath = "/engines/cortex.llamacpp"; +constexpr static auto kPythonRuntimeEngine = "cortex.python"; } // namespace -server::server() - : engine_{nullptr} { +server::server(){ - // Some default values for now below - // log_disable(); // Disable the log to file feature, reduce bloat for - // target - // system () - }; + // Some default values for now below + // log_disable(); // Disable the log to file feature, reduce bloat for + // target + // system () +}; server::~server() {} void server::ChatCompletion( const HttpRequestPtr& req, std::function&& callback) { - if (!IsEngineLoaded()) { + auto engine_type = + (*(req->getJsonObject())).get("engine", kLlamaEngine).asString(); + if (!IsEngineLoaded(engine_type)) { Json::Value res; res["message"] = "Engine is not loaded yet"; auto resp = cortex_utils::nitroHttpJsonResponse(res); @@ -44,10 +45,11 @@ void server::ChatCompletion( auto json_body = req->getJsonObject(); bool is_stream = (*json_body).get("stream", false).asBool(); auto q = std::make_shared(); - engine_->HandleChatCompletion(json_body, - [q](Json::Value status, Json::Value res) { - q->push(std::make_pair(status, res)); - }); + std::get(engines_[engine_type].engine) + ->HandleChatCompletion(json_body, + [q](Json::Value status, Json::Value res) { + q->push(std::make_pair(status, res)); + }); LOG_TRACE << "Wait to chat completion responses"; if (is_stream) { ProcessStreamRes(std::move(callback), q); @@ -59,7 +61,9 @@ void server::ChatCompletion( void server::Embedding(const HttpRequestPtr& req, std::function&& callback) { - if (!IsEngineLoaded()) { + auto engine_type = + (*(req->getJsonObject())).get("engine", kLlamaEngine).asString(); + if (!IsEngineLoaded(engine_type)) { Json::Value res; res["message"] = "Engine is not loaded yet"; auto resp = cortex_utils::nitroHttpJsonResponse(res); @@ -71,10 +75,11 @@ void server::Embedding(const HttpRequestPtr& req, LOG_TRACE << "Start embedding"; SyncQueue q; - engine_->HandleEmbedding(req->getJsonObject(), - [&q](Json::Value status, Json::Value res) { - q.push(std::make_pair(status, res)); - }); + std::get(engines_[engine_type].engine) + ->HandleEmbedding(req->getJsonObject(), + [&q](Json::Value status, Json::Value res) { + q.push(std::make_pair(status, res)); + }); LOG_TRACE << "Wait to embedding"; ProcessNonStreamRes(std::move(callback), q); LOG_TRACE << "Done embedding"; @@ -83,7 +88,9 @@ void server::Embedding(const HttpRequestPtr& req, void server::UnloadModel( const HttpRequestPtr& req, std::function&& callback) { - if (!IsEngineLoaded()) { + auto engine_type = + (*(req->getJsonObject())).get("engine", kLlamaEngine).asString(); + if (!IsEngineLoaded(engine_type)) { Json::Value res; res["message"] = "Engine is not loaded yet"; auto resp = cortex_utils::nitroHttpJsonResponse(res); @@ -93,21 +100,24 @@ void server::UnloadModel( return; } LOG_TRACE << "Start unload model"; - engine_->UnloadModel( - req->getJsonObject(), - [cb = std::move(callback)](Json::Value status, Json::Value res) { - auto resp = cortex_utils::nitroHttpJsonResponse(res); - resp->setStatusCode( - static_cast(status["status_code"].asInt())); - cb(resp); - }); + std::get(engines_[engine_type].engine) + ->UnloadModel( + req->getJsonObject(), + [cb = std::move(callback)](Json::Value status, Json::Value res) { + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(static_cast( + status["status_code"].asInt())); + cb(resp); + }); LOG_TRACE << "Done unload model"; } void server::ModelStatus( const HttpRequestPtr& req, std::function&& callback) { - if (!IsEngineLoaded()) { + auto engine_type = + (*(req->getJsonObject())).get("engine", kLlamaEngine).asString(); + if (!IsEngineLoaded(engine_type)) { Json::Value res; res["message"] = "Engine is not loaded yet"; auto resp = cortex_utils::nitroHttpJsonResponse(res); @@ -118,64 +128,138 @@ void server::ModelStatus( } LOG_TRACE << "Start to get model status"; - engine_->GetModelStatus( - req->getJsonObject(), - [cb = std::move(callback)](Json::Value status, Json::Value res) { - auto resp = cortex_utils::nitroHttpJsonResponse(res); - resp->setStatusCode( - static_cast(status["status_code"].asInt())); - cb(resp); - }); + std::get(engines_[engine_type].engine) + ->GetModelStatus( + req->getJsonObject(), + [cb = std::move(callback)](Json::Value status, Json::Value res) { + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(static_cast( + status["status_code"].asInt())); + cb(resp); + }); LOG_TRACE << "Done get model status"; } +void server::GetEngines( + const HttpRequestPtr& req, + std::function&& callback) { + Json::Value res; + Json::Value engine_array(Json::arrayValue); + for (const auto& [s, _] : engines_) { + Json::Value val; + val["id"] = s; + val["object"] = "engine"; + engine_array.append(val); + } + + res["object"] = "list"; + res["data"] = engine_array; + + auto resp = cortex_utils::nitroHttpJsonResponse(res); + callback(resp); +} + +void server::FineTuning( + const HttpRequestPtr& req, + std::function&& callback) { + auto engine_type = + (*(req->getJsonObject())).get("engine", kPythonRuntimeEngine).asString(); + + if (engines_.find(engine_type) == engines_.end()) { + try { + std::string abs_path = + cortex_utils::GetCurrentPath() + cortex_utils::kPythonRuntimeLibPath; + engines_[engine_type].dl = + std::make_unique(abs_path, "engine"); + } catch (const cortex_cpp::dylib::load_error& e) { + + LOG_ERROR << "Could not load engine: " << e.what(); + engines_.erase(engine_type); + + Json::Value res; + res["message"] = "Could not load engine " + engine_type; + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(k500InternalServerError); + callback(resp); + return; + } + + auto func = engines_[engine_type].dl->get_function( + "get_engine"); + engines_[engine_type].engine = func(); + LOG_INFO << "Loaded engine: " << engine_type; + } + + LOG_TRACE << "Start to fine-tuning"; + auto& en = std::get(engines_[engine_type].engine); + if (en->IsSupported("HandlePythonFileExecutionRequest")) { + en->HandlePythonFileExecutionRequest( + req->getJsonObject(), + [cb = std::move(callback)](Json::Value status, Json::Value res) { + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(static_cast( + status["status_code"].asInt())); + cb(resp); + }); + } else { + Json::Value res; + res["message"] = "Method is not supported yet"; + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(k500InternalServerError); + callback(resp); + LOG_WARN << "Method is not supported yet"; + } + LOG_TRACE << "Done fine-tuning"; +} + void server::LoadModel(const HttpRequestPtr& req, std::function&& callback) { auto engine_type = (*(req->getJsonObject())).get("engine", kLlamaEngine).asString(); - if (!dylib_ || engine_type != cur_engine_name_) { - cur_engine_name_ = engine_type; - // TODO: change this when we get more engines + + // We have not loaded engine yet, should load it before using it + if (engines_.find(engine_type) == engines_.end()) { + // TODO(sang) we cannot run cortex.llamacpp and cortex.tensorrt-llm at the same time. + // So need an unload engine machanism to handle. auto get_engine_path = [](std::string_view e) { if (e == kLlamaEngine) { - return kLlamaLibPath; + return cortex_utils::kLlamaLibPath; } - return kLlamaLibPath; + return cortex_utils::kLlamaLibPath; }; try { - std::string abs_path = cortex_utils::GetCurrentPath() + - get_engine_path(cur_engine_name_); - dylib_ = + std::string abs_path = + cortex_utils::GetCurrentPath() + get_engine_path(engine_type); + engines_[engine_type].dl = std::make_unique(abs_path, "engine"); } catch (const cortex_cpp::dylib::load_error& e) { LOG_ERROR << "Could not load engine: " << e.what(); - dylib_.reset(); - engine_ = nullptr; - } + engines_.erase(engine_type); - if (!dylib_) { Json::Value res; - res["message"] = "Could not load engine " + cur_engine_name_; + res["message"] = "Could not load engine " + engine_type; auto resp = cortex_utils::nitroHttpJsonResponse(res); resp->setStatusCode(k500InternalServerError); callback(resp); return; } - auto func = dylib_->get_function("get_engine"); - engine_ = func(); - LOG_INFO << "Loaded engine: " << cur_engine_name_; + + auto func = + engines_[engine_type].dl->get_function("get_engine"); + engines_[engine_type].engine = func(); + LOG_INFO << "Loaded engine: " << engine_type; } LOG_TRACE << "Load model"; - engine_->LoadModel( - req->getJsonObject(), - [cb = std::move(callback)](Json::Value status, Json::Value res) { - auto resp = cortex_utils::nitroHttpJsonResponse(res); - resp->setStatusCode( - static_cast(status["status_code"].asInt())); - cb(resp); - }); + auto& en = std::get(engines_[engine_type].engine); + en->LoadModel(req->getJsonObject(), [cb = std::move(callback)]( + Json::Value status, Json::Value res) { + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode( + static_cast(status["status_code"].asInt())); + cb(resp); + }); LOG_TRACE << "Done load model"; } @@ -222,8 +306,8 @@ void server::ProcessNonStreamRes(std::function cb, cb(resp); } -bool server::IsEngineLoaded() { - return !!engine_; +bool server::IsEngineLoaded(const std::string& e) { + return engines_.find(e) != engines_.end(); } } // namespace inferences \ No newline at end of file diff --git a/cortex-cpp/controllers/server.h b/cortex-cpp/controllers/server.h index ba7bbb97b..2fe8ecf92 100644 --- a/cortex-cpp/controllers/server.h +++ b/cortex-cpp/controllers/server.h @@ -14,9 +14,11 @@ #include #include #include +#include #include "common/base.h" #include "cortex-common/EngineI.h" +#include "cortex-common/cortexpythoni.h" #include "trantor/utils/SerialTaskQueue.h" #include "utils/dylib.h" #include "utils/json.hpp" @@ -31,9 +33,9 @@ using namespace drogon; namespace inferences { class server : public drogon::HttpController, - public BaseModel, - public BaseChatCompletion, - public BaseEmbedding { + public BaseModel, + public BaseChatCompletion, + public BaseEmbedding { struct SyncQueue; public: @@ -46,9 +48,14 @@ class server : public drogon::HttpController, METHOD_ADD(server::LoadModel, "loadmodel", Post); METHOD_ADD(server::UnloadModel, "unloadmodel", Post); METHOD_ADD(server::ModelStatus, "modelstatus", Post); + METHOD_ADD(server::GetEngines, "engines", Get); + + // cortex.python API + METHOD_ADD(server::FineTuning, "finetuning", Post); // Openai compatible path ADD_METHOD_TO(server::ChatCompletion, "/v1/chat/completions", Post); + ADD_METHOD_TO(server::FineTuning, "/v1/fine_tuning/job", Post); // ADD_METHOD_TO(server::handlePrelight, "/v1/chat/completions", Options); // NOTE: prelight will be added back when browser support is properly planned @@ -72,13 +79,20 @@ class server : public drogon::HttpController, void ModelStatus( const HttpRequestPtr& req, std::function&& callback) override; + void GetEngines( + const HttpRequestPtr& req, + std::function&& callback) override; + + void FineTuning( + const HttpRequestPtr& req, + std::function&& callback) override; private: void ProcessStreamRes(std::function cb, std::shared_ptr q); void ProcessNonStreamRes(std::function cb, SyncQueue& q); - bool IsEngineLoaded(); + bool IsEngineLoaded(const std::string& e); private: struct SyncQueue { @@ -120,8 +134,11 @@ class server : public drogon::HttpController, }; private: - std::unique_ptr dylib_; - EngineI* engine_; - std::string cur_engine_name_; + using EngineV = std::variant; + struct EngineInfo { + std::unique_ptr dl; + EngineV engine; + }; + std::unordered_map engines_; }; }; // namespace inferences \ No newline at end of file diff --git a/cortex-cpp/cortex-common/EngineI.h b/cortex-cpp/cortex-common/EngineI.h index b8770b230..53956624b 100644 --- a/cortex-cpp/cortex-common/EngineI.h +++ b/cortex-cpp/cortex-common/EngineI.h @@ -9,19 +9,20 @@ class EngineI { public: virtual ~EngineI() {} + // cortex.llamacpp interface virtual void HandleChatCompletion( - std::shared_ptr jsonBody, + std::shared_ptr json_body, std::function&& callback) = 0; virtual void HandleEmbedding( - std::shared_ptr jsonBody, + std::shared_ptr json_body, std::function&& callback) = 0; virtual void LoadModel( - std::shared_ptr jsonBody, + std::shared_ptr json_body, std::function&& callback) = 0; virtual void UnloadModel( - std::shared_ptr jsonBody, + std::shared_ptr json_body, std::function&& callback) = 0; virtual void GetModelStatus( - std::shared_ptr jsonBody, + std::shared_ptr json_body, std::function&& callback) = 0; }; diff --git a/cortex-cpp/cortex-common/cortexpythoni.h b/cortex-cpp/cortex-common/cortexpythoni.h new file mode 100644 index 000000000..06a79838f --- /dev/null +++ b/cortex-cpp/cortex-common/cortexpythoni.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "json/value.h" + +class CortexPythonEngineI { + public: + virtual ~CortexPythonEngineI() {} + + virtual bool IsSupported(const std::string& f) = 0; + + virtual void ExecutePythonFile(std::string binary_execute_path, + std::string file_execution_path, + std::string python_library_path) = 0; + + virtual void HandlePythonFileExecutionRequest( + std::shared_ptr json_body, + std::function&& callback) = 0; +}; + diff --git a/cortex-cpp/engines/cortex.python/engine.cmake b/cortex-cpp/engines/cortex.python/engine.cmake new file mode 100644 index 000000000..2a6a6ec0d --- /dev/null +++ b/cortex-cpp/engines/cortex.python/engine.cmake @@ -0,0 +1,38 @@ +# cortex.python release version +set(VERSION 0.1.1) +set(ENGINE_VERSION v${VERSION}) +set(ENGINE_NAME cortex.python) + +# MESSAGE("ENGINE_VERSION=" ${ENGINE_VERSION}) + +# Download library based on instructions +if(UNIX AND NOT APPLE) + set(LIBRARY_NAME ${ENGINE_NAME}-${VERSION}-linux-amd64.tar.gz) +elseif(UNIX) + if(MAC_ARM64) + set(LIBRARY_NAME ${ENGINE_NAME}-${VERSION}-mac-arm64.tar.gz) + else() + set(LIBRARY_NAME ${ENGINE_NAME}-${VERSION}-mac-amd64.tar.gz) + endif() +else() + set(LIBRARY_NAME ${ENGINE_NAME}-${VERSION}-windows-amd64.tar.gz) +endif() + + +set(LIBPYTHONRUNTIME_ENGINE_URL https://github.com/janhq/cortex.python/releases/download/${ENGINE_VERSION}/${LIBRARY_NAME}) +MESSAGE("LIBPYTHONRUNTIME_ENGINE_URL=" ${LIBPYTHONRUNTIME_ENGINE_URL}) +MESSAGE("LIBARRY_NAME=" ${LIBRARY_NAME}) +set(LIBPYTHONRUNTIME_ENGINE_PATH ${CMAKE_BINARY_DIR}/engines/${LIBRARY_NAME}) + +# MESSAGE("CMAKE_BINARY_DIR = " ${CMAKE_BINARY_DIR}) + +file(DOWNLOAD ${LIBPYTHONRUNTIME_ENGINE_URL} ${LIBPYTHONRUNTIME_ENGINE_PATH} STATUS LIBPYTHONRUNTIME_ENGINE_DOWNLOAD_STATUS) +list(GET LIBPYTHONRUNTIME_ENGINE_DOWNLOAD_STATUS 0 LIBPYTHONRUNTIME_ENGINE_DOWNLOAD_STATUS_NO) +# MESSAGE("file = " ${CMAKE_BINARY_DIR}/engines/${LIBRARY_NAME}) + +if(LIBPYTHONRUNTIME_ENGINE_DOWNLOAD_STATUS_NO) + message(STATUS "Pre-built library not downloaded. (${LIBPYTHONRUNTIME_ENGINE_DOWNLOAD_STATUS})") +else() + message(STATUS "Linking downloaded pre-built library.") + file(ARCHIVE_EXTRACT INPUT ${CMAKE_BINARY_DIR}/engines/${LIBRARY_NAME} DESTINATION ${CMAKE_BINARY_DIR}/engines/) +endif() \ No newline at end of file diff --git a/cortex-cpp/main.cc b/cortex-cpp/main.cc index 12cabeb0c..04190d92d 100644 --- a/cortex-cpp/main.cc +++ b/cortex-cpp/main.cc @@ -2,7 +2,9 @@ #include #include // for PATH_MAX #include +#include "cortex-common/cortexpythoni.h" #include "utils/cortex_utils.h" +#include "utils/dylib.h" #if defined(__APPLE__) && defined(__MACH__) #include // for dirname() @@ -18,6 +20,27 @@ #endif int main(int argc, char* argv[]) { + // Check if this process is for python execution + if (argc > 1) { + if (strcmp(argv[1], "--run_python_file") == 0) { + std::string py_home_path = (argc > 3) ? argv[3] : ""; + std::unique_ptr dl; + try { + std::string abs_path = cortex_utils::GetCurrentPath() + + cortex_utils::kPythonRuntimeLibPath; + dl = std::make_unique(abs_path, "engine"); + } catch (const cortex_cpp::dylib::load_error& e) { + LOG_ERROR << "Could not load engine: " << e.what(); + return 1; + } + + auto func = dl->get_function("get_engine"); + auto e = func(); + e->ExecutePythonFile(argv[0], argv[2], py_home_path); + return 0; + } + } + int thread_num = 1; std::string host = "127.0.0.1"; int port = 3928; diff --git a/cortex-cpp/utils/cortex_utils.h b/cortex-cpp/utils/cortex_utils.h index 3c4fdd2dd..c0670a431 100644 --- a/cortex-cpp/utils/cortex_utils.h +++ b/cortex-cpp/utils/cortex_utils.h @@ -25,6 +25,8 @@ #endif namespace cortex_utils { +constexpr static auto kLlamaLibPath = "/engines/cortex.llamacpp"; +constexpr static auto kPythonRuntimeLibPath = "/engines/cortex.python"; inline std::string models_folder = "./models"; From 83e34d0f8adaf8868b2c6b6d42e140e605fe8f9d Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 09:11:50 +0700 Subject: [PATCH 02/14] chore: cortex.python v0.1.2 --- cortex-cpp/engines/cortex.python/engine.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-cpp/engines/cortex.python/engine.cmake b/cortex-cpp/engines/cortex.python/engine.cmake index 2a6a6ec0d..f5ac71d15 100644 --- a/cortex-cpp/engines/cortex.python/engine.cmake +++ b/cortex-cpp/engines/cortex.python/engine.cmake @@ -1,5 +1,5 @@ # cortex.python release version -set(VERSION 0.1.1) +set(VERSION 0.1.2) set(ENGINE_VERSION v${VERSION}) set(ENGINE_NAME cortex.python) From 59dec6858e2cd28772d74ae4f8810643677d9ec5 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 10:33:58 +0700 Subject: [PATCH 03/14] fix: e2e testing for cortex.python --- .../scripts/e2e-test-python-linux-and-mac.sh | 97 ++++++++++++++ .../e2e-test-python-runtime-linux-and-mac.sh | 0 .../e2e-test-python-runtime-windows.bat | 0 .github/scripts/e2e-test-python-windows.bat | 119 ++++++++++++++++++ .github/scripts/python-file-to-test.py | 9 ++ .github/workflows/cortex-cpp-quality-gate.yml | 26 ++++ cortex-cpp/Makefile | 24 ++++ 7 files changed, 275 insertions(+) create mode 100755 .github/scripts/e2e-test-python-linux-and-mac.sh delete mode 100644 .github/scripts/e2e-test-python-runtime-linux-and-mac.sh delete mode 100644 .github/scripts/e2e-test-python-runtime-windows.bat create mode 100755 .github/scripts/e2e-test-python-windows.bat create mode 100644 .github/scripts/python-file-to-test.py diff --git a/.github/scripts/e2e-test-python-linux-and-mac.sh b/.github/scripts/e2e-test-python-linux-and-mac.sh new file mode 100755 index 000000000..607110fb2 --- /dev/null +++ b/.github/scripts/e2e-test-python-linux-and-mac.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +## Example run command +# ./e2e-test-python-linux-and-mac.sh '../../examples/build/server' './e2e-test.py' + +# Check for required arguments +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + +BINARY_PATH=$1 +PYTHON_FILE_EXECUTION_PATH=$2 + +rm /tmp/python-file-execution-res.log /tmp/server.log + +# Random port to ensure it's not used +min=10000 +max=11000 +range=$((max - min + 1)) +PORT=$((RANDOM % range + min)) + +# Install numpy for Python +export PYTHONHOME=$(pwd)/engines/cortex.python/python/ +export LD_LIBRARY_PATH="$PYTHONHOME:$LD_LIBRARY_PATH" +echo "Set Python HOME to $PYTHONHOME" +./engines/cortex.python/python/bin/python3 -m ensurepip +./engines/cortex.python/python/bin/python3 -m pip install --upgrade pip +./engines/cortex.python/python/bin/python3 -m pip install numpy --target=$PYTHONHOME/lib/python/site-packages/ + +# Start the binary file +"$BINARY_PATH" 1 127.0.0.1 $PORT >/tmp/server.log & + +pid=$! + +if ! ps -p $pid >/dev/null; then + echo "server failed to start. Logs:" + cat /tmp/server.log + exit 1 +fi + +# Wait for a few seconds to let the server start +sleep 3 + +# Run the curl commands +response1=$(curl --connect-timeout 60 -o /tmp/python-file-execution-res.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/v1/fine_tuning/job" \ + --header 'Content-Type: application/json' \ + --data '{ + "file_execution_path": "'$PYTHON_FILE_EXECUTION_PATH'" + }') + +error_occurred=0 + +# Verify the response +if [[ "$response1" -ne 200 ]]; then + echo "The python file execution curl command failed with status code: $response1" + cat /tmp/python-file-execution-res.log + error_occurred=1 +fi + +# Verify the output of the Python file in output.txt +OUTPUT_FILE="./output.txt" +EXPECTED_OUTPUT="1 2 3" # Replace with the expected content + +if [[ -f "$OUTPUT_FILE" ]]; then + actual_output=$(cat "$OUTPUT_FILE") + if [[ "$actual_output" != "$EXPECTED_OUTPUT" ]]; then + echo "The output of the Python file does not match the expected output." + echo "Expected: $EXPECTED_OUTPUT" + echo "Actual: $actual_output" + error_occurred=1 + else + echo "The output of the Python file matches the expected output." + fi +else + echo "Output file $OUTPUT_FILE does not exist." + error_occurred=1 +fi + + +if [[ "$error_occurred" -eq 1 ]]; then + echo "Server test run failed!!!!!!!!!!!!!!!!!!!!!!" + echo "Server Error Logs:" + cat /tmp/server.log + kill $pid + echo "An error occurred while running the server." + exit 1 +fi + +echo "----------------------" +echo "Log server:" +cat /tmp/server.log + +echo "Server test run successfully!" + +# Kill the server process +kill $pid \ No newline at end of file diff --git a/.github/scripts/e2e-test-python-runtime-linux-and-mac.sh b/.github/scripts/e2e-test-python-runtime-linux-and-mac.sh deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/scripts/e2e-test-python-runtime-windows.bat b/.github/scripts/e2e-test-python-runtime-windows.bat deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/scripts/e2e-test-python-windows.bat b/.github/scripts/e2e-test-python-windows.bat new file mode 100755 index 000000000..2e673f58f --- /dev/null +++ b/.github/scripts/e2e-test-python-windows.bat @@ -0,0 +1,119 @@ +@echo off + +setlocal enabledelayedexpansion + +set "TEMP=C:\Users\%UserName%\AppData\Local\Temp" + +rem Check for required arguments +if "%~2"=="" ( + echo Usage: %~0 ^ ^ + exit /b 1 +) + +set "BINARY_PATH=%~1" +set "PYTHON_FILE_EXECUTION_PATH=%~2" + +for %%i in ("%BINARY_PATH%") do set "BINARY_NAME=%%~nxi" + +echo BINARY_NAME=%BINARY_NAME% + +del %TEMP%\response1.log 2>nul +del %TEMP%\server.log 2>nul + +set /a min=9999 +set /a max=11000 +set /a range=max-min+1 +set /a PORT=%min% + %RANDOM% %% %range% + +rem Install numpy for Python +set "PYTHONHOME=%cd%\engines\cortex.python\python" +echo Set Python HOME to %PYTHONHOME% +%PYTHONHOME%\python.exe -m ensurepip +%PYTHONHOME%\python.exe -m pip install --upgrade pip +%PYTHONHOME%\python.exe -m pip install numpy --target=%PYTHONHOME%\Lib\site-packages\ + +rem Start the binary file +start "" /B "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > "%TEMP%\server.log" 2>&1 + +ping -n 3 127.0.0.1 > nul + +rem Capture the PID of the started process with "server" in its name +for /f "tokens=2" %%a in ('tasklist /fi "imagename eq %BINARY_NAME%" /fo list ^| findstr /B "PID:"') do ( + set "pid=%%a" +) + +echo pid=%pid% + +if not defined pid ( + echo server failed to start. Logs: + type %TEMP%\server.log + echo. + exit /b 1 +) + +rem Wait for a few seconds to let the server start + +rem Define JSON strings for curl data +call set "PYTHON_FILE_EXECUTION_PATH_STRING=%%PYTHON_FILE_EXECUTION_PATH:\=\\%%" +set "curl_data1={\"file_execution_path\":\"%PYTHON_FILE_EXECUTION_PATH_STRING%\"}" + +rem Print the values of curl_data for debugging +echo curl_data1=%curl_data1% + +rem Run the curl commands and capture the status code +curl.exe --connect-timeout 60 -o "%TEMP%\response1.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/fine_tuning/job" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1.log 2>&1 + +set "error_occurred=0" + +rem Read the status code directly from the response file +set "response1=" +for /f %%a in (%TEMP%\response1.log) do set "response1=%%a" + +if "%response1%" neq "200" ( + echo The first curl command failed with status code: %response1% + type %TEMP%\response1.log + echo. + set "error_occurred=1" +) + +echo ---------------------- +echo Log python file execution: +type %TEMP%\response1.log +echo. + +rem Verification step: Check the contents of output.txt +set "expected_output=1 2 3" +set "actual_output=" +if exist "output.txt" ( + for /f "delims=" %%x in (output.txt) do set "actual_output=%%x" + if "!actual_output!"=="!expected_output!" ( + echo Verification succeeded: output.txt contains the expected data. + ) else ( + echo Verification failed: output.txt does not contain the expected data. + echo Expected: !expected_output! + echo Actual: !actual_output! + set "error_occurred=1" + ) +) else ( + echo Verification failed: output.txt does not exist. + set "error_occurred=1" +) + +echo ---------------------- +echo Server logs: +type %TEMP%\server.log +echo. + +if "%error_occurred%"=="1" ( + echo Server test run failed!!!!!!!!!!!!!!!!!!!!!! + taskkill /f /pid %pid% + echo An error occurred while running the server. + exit /b 1 +) + +echo Server test run successfully! + +rem Kill the server process +taskkill /f /im server.exe 2>nul || exit /B 0 + +endlocal \ No newline at end of file diff --git a/.github/scripts/python-file-to-test.py b/.github/scripts/python-file-to-test.py new file mode 100644 index 000000000..e897baf42 --- /dev/null +++ b/.github/scripts/python-file-to-test.py @@ -0,0 +1,9 @@ +import sys; +for path in sys.path: + print(path) + +import numpy as np +print("Numpy version: " + np.__version__) + +with open('output.txt', 'w') as file: + file.write(' '.join(map(str, np.array([1, 2, 3])))) \ No newline at end of file diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 33c8a4533..e84efa4e7 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -12,6 +12,7 @@ on: env: LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf EMBEDDING_MODEL_URL: https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf + PYTHON_FILE_EXECUTION_PATH: "python-file-to-test.py" jobs: build-and-test: @@ -26,107 +27,126 @@ jobs: runs-on: "ubuntu-18-04" cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF" run-e2e: true + run-python-e2e: true - os: "linux" name: "amd64-avx" runs-on: "ubuntu-18-04" cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF" run-e2e: false + run-python-e2e: false - os: "linux" name: "amd64-avx512" runs-on: "ubuntu-18-04" cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF" run-e2e: false + run-python-e2e: false - os: "linux" name: "amd64-vulkan" runs-on: "ubuntu-18-04-cuda-11-7" cmake-flags: "-DLLAMA_VULKAN=ON -DLLAMA_NATIVE=OFF" run-e2e: false + run-python-e2e: false - os: "linux" name: "amd64-cuda-11-7" runs-on: "ubuntu-18-04-cuda-11-7" cmake-flags: "-DCUDA_11_7=ON -DLLAMA_NATIVE=OFF -DLLAMA_CUDA=ON" run-e2e: false + run-python-e2e: false - os: "linux" name: "amd64-cuda-12-0" runs-on: "ubuntu-18-04-cuda-12-0" cmake-flags: "-DCUDA_12_0=ON -DLLAMA_NATIVE=OFF -DLLAMA_CUDA=ON" run-e2e: false + run-python-e2e: false - os: "mac" name: "amd64" runs-on: "macos-13" cmake-flags: "" run-e2e: true + run-python-e2e: true - os: "mac" name: "arm64" runs-on: "mac-silicon" cmake-flags: "-DMAC_ARM64=ON" run-e2e: true + run-python-e2e: true - os: "windows" name: "amd64-avx2" runs-on: "windows-latest" cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: true + run-python-e2e: true - os: "windows" name: "amd64-avx" runs-on: "windows-latest" cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-avx512" runs-on: "windows-latest" cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-vulkan" runs-on: "windows-latest" cmake-flags: "-DLLAMA_VULKAN=ON -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-avx2-cuda-12-0" runs-on: "windows-cuda-12-0" cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF -DCUDA_12_0=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-avx-cuda-12-0" runs-on: "windows-cuda-12-0" cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF -DCUDA_12_0=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-avx512-cuda-12-0" runs-on: "windows-cuda-12-0" cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF -DCUDA_12_0=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-avx2-cuda-11-7" runs-on: "windows-cuda-11-7" cmake-flags: "-DLLAMA_AVX2=ON -DLLAMA_NATIVE=OFF -DCUDA_11_7=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false - os: "windows" name: "amd64-avx-cuda-11-7" runs-on: "windows-cuda-11-7" cmake-flags: "-DLLAMA_AVX2=OFF -DLLAMA_NATIVE=OFF -DCUDA_11_7=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false + - os: "windows" name: "amd64-avx512-cuda-11-7" runs-on: "windows-cuda-11-7" cmake-flags: "-DLLAMA_AVX512=ON -DLLAMA_NATIVE=OFF -DCUDA_11_7=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUDA=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE" run-e2e: false + run-python-e2e: false steps: - name: Clone @@ -161,6 +181,12 @@ jobs: cd cortex-cpp make run-e2e-test RUN_TESTS=true LLM_MODEL_URL=${{ env.LLM_MODEL_URL }} EMBEDDING_MODEL_URL=${{ env.EMBEDDING_MODEL_URL }} + - name: Run python e2e testing + if: ${{ matrix.run-python-e2e }} + run: | + cd cortex-cpp + make run-python-e2e-test RUN_TESTS=true PYTHON_FILE_EXECUTION_PATH=${{ env.PYTHON_FILE_EXECUTION_PATH }} + - name: Upload Artifact uses: actions/upload-artifact@v2 with: diff --git a/cortex-cpp/Makefile b/cortex-cpp/Makefile index 98486f023..926ce1bd6 100644 --- a/cortex-cpp/Makefile +++ b/cortex-cpp/Makefile @@ -6,6 +6,7 @@ CMAKE_EXTRA_FLAGS ?= "" RUN_TESTS ?= false LLM_MODEL_URL ?= "https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" EMBEDDING_MODEL_URL ?= "https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf" +PYTHON_FILE_EXECUTION_PATH ?= "python-file-to-test.py" CODE_SIGN ?= false AZURE_KEY_VAULT_URI ?= xxxx AZURE_CLIENT_ID ?= xxxx @@ -97,6 +98,29 @@ else rm -rf uploads/; endif +run-python-e2e-test: +ifeq ($(RUN_TESTS),false) + @echo "Skipping tests" + @exit 0 +endif +ifeq ($(OS),Windows_NT) + @powershell -Command "cp -r build\engines\cortex.python cortex-cpp\engines\;" + @powershell -Command "cd cortex-cpp;..\..\.github\scripts\e2e-test-python-windows.bat cortex-cpp.exe ..\..\.github\scripts\$(PYTHON_FILE_EXECUTION_PATH);" + @powershell -Command "rm -r .\engines\cortex.python;" +else ifeq ($(shell uname -s),Linux) + @cp -rf build/engines/cortex.python cortex-cpp/engines/; \ + cd cortex-cpp; \ + chmod +x ../../.github/scripts/e2e-test-python-linux-and-mac.sh && ../../.github/scripts/e2e-test-python-linux-and-mac.sh ./cortex-cpp ../../.github/scripts/$(PYTHON_FILE_EXECUTION_PATH); \ + rm -rf uploads/; \ + rm -rf ./engines/cortex.python; +else + @cp -rf build/engines/cortex.python cortex-cpp/engines/; \ + cd cortex-cpp; \ + chmod +x ../../.github/scripts/e2e-test-python-linux-and-mac.sh && ../../.github/scripts/e2e-test-python-linux-and-mac.sh ./cortex-cpp ../../.github/scripts/$(PYTHON_FILE_EXECUTION_PATH); \ + rm -rf uploads/; \ + rm -rf ./engines/cortex.python; +endif + clean: ifeq ($(OS),Windows_NT) @powershell -Command "rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz;" From 9a3fb416c9451b72ecdb80421686856384166a1a Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 14:30:40 +0700 Subject: [PATCH 04/14] test: print LD_LIBRARY_PATH --- .github/scripts/e2e-test-python-linux-and-mac.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/e2e-test-python-linux-and-mac.sh b/.github/scripts/e2e-test-python-linux-and-mac.sh index 607110fb2..23c42adb1 100755 --- a/.github/scripts/e2e-test-python-linux-and-mac.sh +++ b/.github/scripts/e2e-test-python-linux-and-mac.sh @@ -24,6 +24,7 @@ PORT=$((RANDOM % range + min)) export PYTHONHOME=$(pwd)/engines/cortex.python/python/ export LD_LIBRARY_PATH="$PYTHONHOME:$LD_LIBRARY_PATH" echo "Set Python HOME to $PYTHONHOME" +echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" ./engines/cortex.python/python/bin/python3 -m ensurepip ./engines/cortex.python/python/bin/python3 -m pip install --upgrade pip ./engines/cortex.python/python/bin/python3 -m pip install numpy --target=$PYTHONHOME/lib/python/site-packages/ From 1cd35d1ca2898938eb35c7df63873c78f2ad8453 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 16:57:41 +0700 Subject: [PATCH 05/14] test: linux --- .github/scripts/e2e-test-python-linux-and-mac.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/e2e-test-python-linux-and-mac.sh b/.github/scripts/e2e-test-python-linux-and-mac.sh index 23c42adb1..586e1024c 100755 --- a/.github/scripts/e2e-test-python-linux-and-mac.sh +++ b/.github/scripts/e2e-test-python-linux-and-mac.sh @@ -23,6 +23,7 @@ PORT=$((RANDOM % range + min)) # Install numpy for Python export PYTHONHOME=$(pwd)/engines/cortex.python/python/ export LD_LIBRARY_PATH="$PYTHONHOME:$LD_LIBRARY_PATH" +export DYLD_FALLBACK_LIBRARY_PATH="$PYTHONHOME:$DYLD_FALLBACK_LIBRARY_PATH" echo "Set Python HOME to $PYTHONHOME" echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" ./engines/cortex.python/python/bin/python3 -m ensurepip From 2d0102000ffe14216502d115fab60fdf41b8538e Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 18:11:24 +0700 Subject: [PATCH 06/14] chore: cortex.python to 0.1.3 --- cortex-cpp/engines/cortex.python/engine.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-cpp/engines/cortex.python/engine.cmake b/cortex-cpp/engines/cortex.python/engine.cmake index f5ac71d15..bdc4897f1 100644 --- a/cortex-cpp/engines/cortex.python/engine.cmake +++ b/cortex-cpp/engines/cortex.python/engine.cmake @@ -1,5 +1,5 @@ # cortex.python release version -set(VERSION 0.1.2) +set(VERSION 0.1.3) set(ENGINE_VERSION v${VERSION}) set(ENGINE_NAME cortex.python) From 0a5002e13b4dfb3fbbbd6f11f0bd517cb3e7a3fc Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Thu, 23 May 2024 08:44:44 +0700 Subject: [PATCH 07/14] test: DYLD_LIBRARY_PATH --- .github/scripts/e2e-test-python-linux-and-mac.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/e2e-test-python-linux-and-mac.sh b/.github/scripts/e2e-test-python-linux-and-mac.sh index 586e1024c..9cb1a7729 100755 --- a/.github/scripts/e2e-test-python-linux-and-mac.sh +++ b/.github/scripts/e2e-test-python-linux-and-mac.sh @@ -23,6 +23,7 @@ PORT=$((RANDOM % range + min)) # Install numpy for Python export PYTHONHOME=$(pwd)/engines/cortex.python/python/ export LD_LIBRARY_PATH="$PYTHONHOME:$LD_LIBRARY_PATH" +export DYLD_LIBRARY_PATH="$PYTHONHOME:$DYLD_LIBRARY_PATH" export DYLD_FALLBACK_LIBRARY_PATH="$PYTHONHOME:$DYLD_FALLBACK_LIBRARY_PATH" echo "Set Python HOME to $PYTHONHOME" echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" From d2fd21690411505798ad3ecc44f27e9bae1ea0bf Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Thu, 23 May 2024 15:29:25 +0700 Subject: [PATCH 08/14] chore: cortex.python to 0.1.4 --- .github/scripts/e2e-test-python-linux-and-mac.sh | 1 - cortex-cpp/engines/cortex.python/engine.cmake | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/e2e-test-python-linux-and-mac.sh b/.github/scripts/e2e-test-python-linux-and-mac.sh index 9cb1a7729..586e1024c 100755 --- a/.github/scripts/e2e-test-python-linux-and-mac.sh +++ b/.github/scripts/e2e-test-python-linux-and-mac.sh @@ -23,7 +23,6 @@ PORT=$((RANDOM % range + min)) # Install numpy for Python export PYTHONHOME=$(pwd)/engines/cortex.python/python/ export LD_LIBRARY_PATH="$PYTHONHOME:$LD_LIBRARY_PATH" -export DYLD_LIBRARY_PATH="$PYTHONHOME:$DYLD_LIBRARY_PATH" export DYLD_FALLBACK_LIBRARY_PATH="$PYTHONHOME:$DYLD_FALLBACK_LIBRARY_PATH" echo "Set Python HOME to $PYTHONHOME" echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" diff --git a/cortex-cpp/engines/cortex.python/engine.cmake b/cortex-cpp/engines/cortex.python/engine.cmake index bdc4897f1..95c25e13a 100644 --- a/cortex-cpp/engines/cortex.python/engine.cmake +++ b/cortex-cpp/engines/cortex.python/engine.cmake @@ -1,5 +1,5 @@ # cortex.python release version -set(VERSION 0.1.3) +set(VERSION 0.1.4) set(ENGINE_VERSION v${VERSION}) set(ENGINE_NAME cortex.python) From 8111fb05dc15d6cb918a0a0ef0b5f5789413df00 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Thu, 23 May 2024 20:37:15 +0700 Subject: [PATCH 09/14] chore: cortex.python to 0.1.5 --- cortex-cpp/Makefile | 3 ++- cortex-cpp/engines/cortex.python/engine.cmake | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cortex-cpp/Makefile b/cortex-cpp/Makefile index 926ce1bd6..a40b92808 100644 --- a/cortex-cpp/Makefile +++ b/cortex-cpp/Makefile @@ -106,7 +106,8 @@ endif ifeq ($(OS),Windows_NT) @powershell -Command "cp -r build\engines\cortex.python cortex-cpp\engines\;" @powershell -Command "cd cortex-cpp;..\..\.github\scripts\e2e-test-python-windows.bat cortex-cpp.exe ..\..\.github\scripts\$(PYTHON_FILE_EXECUTION_PATH);" - @powershell -Command "rm -r .\engines\cortex.python;" + @powershell -Command "rm *.txt;" + @powershell -Command "cd engines; rm -r cortex.python;" else ifeq ($(shell uname -s),Linux) @cp -rf build/engines/cortex.python cortex-cpp/engines/; \ cd cortex-cpp; \ diff --git a/cortex-cpp/engines/cortex.python/engine.cmake b/cortex-cpp/engines/cortex.python/engine.cmake index 95c25e13a..fa6705fde 100644 --- a/cortex-cpp/engines/cortex.python/engine.cmake +++ b/cortex-cpp/engines/cortex.python/engine.cmake @@ -1,5 +1,5 @@ # cortex.python release version -set(VERSION 0.1.4) +set(VERSION 0.1.5) set(ENGINE_VERSION v${VERSION}) set(ENGINE_NAME cortex.python) From 1021d967ce5eb80842424b16116d50b2c7b6bf4d Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Fri, 24 May 2024 08:39:46 +0700 Subject: [PATCH 10/14] chore: cortex.llamacpp to 0.1.9 --- cortex-cpp/engines/cortex.llamacpp/engine.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-cpp/engines/cortex.llamacpp/engine.cmake b/cortex-cpp/engines/cortex.llamacpp/engine.cmake index 3d543ae11..0ecc3e8f2 100644 --- a/cortex-cpp/engines/cortex.llamacpp/engine.cmake +++ b/cortex-cpp/engines/cortex.llamacpp/engine.cmake @@ -1,5 +1,5 @@ # cortex.llamacpp release version -set(VERSION 0.1.6) +set(VERSION 0.1.9) set(ENGINE_VERSION v${VERSION}) add_compile_definitions(CORTEX_LLAMACPP_VERSION="${VERSION}") From f9e797189838706983e9c499a749adfb83815225 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Fri, 24 May 2024 14:17:03 +0700 Subject: [PATCH 11/14] fix: rm cortex.python after running python e2e tests --- cortex-cpp/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cortex-cpp/Makefile b/cortex-cpp/Makefile index a40b92808..84461326e 100644 --- a/cortex-cpp/Makefile +++ b/cortex-cpp/Makefile @@ -107,7 +107,9 @@ ifeq ($(OS),Windows_NT) @powershell -Command "cp -r build\engines\cortex.python cortex-cpp\engines\;" @powershell -Command "cd cortex-cpp;..\..\.github\scripts\e2e-test-python-windows.bat cortex-cpp.exe ..\..\.github\scripts\$(PYTHON_FILE_EXECUTION_PATH);" @powershell -Command "rm *.txt;" + @powershell -Command "ls .\engines;" @powershell -Command "cd engines; rm -r cortex.python;" + @powershell -Command "ls;" else ifeq ($(shell uname -s),Linux) @cp -rf build/engines/cortex.python cortex-cpp/engines/; \ cd cortex-cpp; \ From 2c2b4bf0ad3a2b7015347285bbb8e8a98c53df6e Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Sun, 26 May 2024 15:40:59 +0700 Subject: [PATCH 12/14] fix: python windows --- cortex-cpp/CMakeLists.txt | 4 +++- cortex-cpp/Makefile | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cortex-cpp/CMakeLists.txt b/cortex-cpp/CMakeLists.txt index ae9c31ad2..824ba67b7 100644 --- a/cortex-cpp/CMakeLists.txt +++ b/cortex-cpp/CMakeLists.txt @@ -2,7 +2,9 @@ cmake_minimum_required(VERSION 3.5) project(cortex-cpp C CXX) include(engines/cortex.llamacpp/engine.cmake) -include(engines/cortex.python/engine.cmake) +if(NOT LLAMA_CUDA) + include(engines/cortex.python/engine.cmake) +endif() include(CheckIncludeFileCXX) check_include_file_cxx(any HAS_ANY) diff --git a/cortex-cpp/Makefile b/cortex-cpp/Makefile index 84461326e..20cd4da7f 100644 --- a/cortex-cpp/Makefile +++ b/cortex-cpp/Makefile @@ -106,10 +106,8 @@ endif ifeq ($(OS),Windows_NT) @powershell -Command "cp -r build\engines\cortex.python cortex-cpp\engines\;" @powershell -Command "cd cortex-cpp;..\..\.github\scripts\e2e-test-python-windows.bat cortex-cpp.exe ..\..\.github\scripts\$(PYTHON_FILE_EXECUTION_PATH);" - @powershell -Command "rm *.txt;" - @powershell -Command "ls .\engines;" - @powershell -Command "cd engines; rm -r cortex.python;" - @powershell -Command "ls;" + @powershell -Command "cd .\cortex-cpp\; rm *.txt;" + @powershell -Command "mkdir python-temp; cd .\cortex-cpp\engines; Move-Item -Path .\cortex.python -Destination ..\..\python-temp;" else ifeq ($(shell uname -s),Linux) @cp -rf build/engines/cortex.python cortex-cpp/engines/; \ cd cortex-cpp; \ From 861e754df8f011baa66bae3b50e9d3e59e0b9592 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Mon, 27 May 2024 10:42:57 +0700 Subject: [PATCH 13/14] fix: pull python --- cortex-cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-cpp/CMakeLists.txt b/cortex-cpp/CMakeLists.txt index 824ba67b7..4e2092afe 100644 --- a/cortex-cpp/CMakeLists.txt +++ b/cortex-cpp/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5) project(cortex-cpp C CXX) include(engines/cortex.llamacpp/engine.cmake) -if(NOT LLAMA_CUDA) +if(NOT LLAMA_CUDA AND (LLAMA_AVX2 OR APPLE)) include(engines/cortex.python/engine.cmake) endif() include(CheckIncludeFileCXX) From 38ce4daa35cc3ea39c79e9a073518adfe65c0b38 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 28 May 2024 17:49:22 +0700 Subject: [PATCH 14/14] chore: drogon to v1.9.4 --- cortex-cpp/cortex-cpp-deps/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-cpp/cortex-cpp-deps/CMakeLists.txt b/cortex-cpp/cortex-cpp-deps/CMakeLists.txt index 4e080a026..d6feb5991 100644 --- a/cortex-cpp/cortex-cpp-deps/CMakeLists.txt +++ b/cortex-cpp/cortex-cpp-deps/CMakeLists.txt @@ -66,7 +66,7 @@ ExternalProject_Add( ExternalProject_Add( drogon GIT_REPOSITORY https://github.com/drogonframework/drogon - GIT_TAG v1.9.2 + GIT_TAG v1.9.4 CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DOPENSSL_USE_STATIC_LIBS=TRUE