From 4d4c7d1f41d41d0bc65f7784f98b7d1d8e73f48a Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Wed, 2 Oct 2024 08:29:02 +0700 Subject: [PATCH 1/3] Update new version python --- src/python_engine.cc | 2 +- third-party/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python_engine.cc b/src/python_engine.cc index fb1cbdd..033af63 100644 --- a/src/python_engine.cc +++ b/src/python_engine.cc @@ -86,7 +86,7 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( child_process_args.push_back(const_cast(file_execution_path.c_str())); if (python_library_path != "") child_process_args.push_back(const_cast(python_library_path.c_str())); - + child_process_args.push_back(nullptr); pid_t pid; int status = posix_spawn(&pid, child_process_exe_path.c_str(), nullptr, diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 1bed5e7..8ada8d4 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -153,7 +153,7 @@ if (UNIX) # APPLE and LINUX # Download and install Python3 from source ExternalProject_Add( Python3 - URL https://www.python.org/ftp/python/3.10.4/Python-3.10.4.tgz + URL https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz PREFIX ${THIRD_PARTY_INSTALL_PATH} CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${THIRD_PARTY_INSTALL_PATH}/lib:${LD_LIBRARY_PATH} ./configure --prefix= --enable-optimizations --with-ensurepip=install --enable-shared ${PYTHON_INSTALL_CONFIG_HOST} ${PYTHON_INSTALL_CONFIG_BUILD} ${PYTHON_INSTALL_CONFIG_SSL} BUILD_COMMAND make -j12 @@ -172,7 +172,7 @@ else() # WINDOWS ExternalProject_Add( Python3 PREFIX "${CMAKE_BINARY_DIR}/python" - URL "https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe" + URL "https://www.python.org/ftp/python/3.12.7/python-3.12.7-amd64.exe" DOWNLOAD_NO_EXTRACT 1 CONFIGURE_COMMAND "" # No configure command BUILD_COMMAND "" # No build command From fcbffaebe2d62c296afed1dc00682a0e30f32097 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Wed, 2 Oct 2024 08:30:09 +0700 Subject: [PATCH 2/3] Update to run script --- src/python_engine.cc | 107 ++++++++++++------ src/python_utils.h | 261 +++++++++++++++++++++++++------------------ 2 files changed, 223 insertions(+), 145 deletions(-) diff --git a/src/python_engine.cc b/src/python_engine.cc index 033af63..19dc7fe 100644 --- a/src/python_engine.cc +++ b/src/python_engine.cc @@ -3,11 +3,11 @@ #include "trantor/utils/Logger.h" #if defined(_WIN32) - #include +#include #else - #include - #include - extern char **environ; +#include +#include +extern char** environ; #endif constexpr const int k200OK = 200; @@ -16,13 +16,14 @@ constexpr const int k500InternalServerError = 500; PythonEngine::~PythonEngine() {} -void PythonEngine::ExecutePythonFile( - std::string binary_execute_path, - std::string file_execution_path, - std::string python_library_path) { +void PythonEngine::ExecutePythonFile(std::string binary_execute_path, + std::string file_execution_path, + std::string python_library_path) { - std::string current_dir_path = python_utils::GetDirectoryPathFromFilePath(binary_execute_path); - python_utils::ExecutePythonFile(current_dir_path, file_execution_path, python_library_path); + std::string current_dir_path = + python_utils::GetDirectoryPathFromFilePath(binary_execute_path); + python_utils::ExecutePythonFile(current_dir_path, file_execution_path, + python_library_path); } void PythonEngine::HandlePythonFileExecutionRequest( @@ -34,10 +35,9 @@ void PythonEngine::HandlePythonFileExecutionRequest( std::move(callback)); } - void PythonEngine::HandlePythonFileExecutionRequestImpl( PythonRuntime::PythonFileExecution::PythonFileExecutionRequest&& request, - std::function && callback) { + std::function&& callback) { std::string file_execution_path = request.file_execution_path; std::string python_library_path = request.python_library_path; @@ -45,12 +45,12 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( Json::Value json_resp; Json::Value status_resp; - if (file_execution_path == "") { - LOG_ERROR << "No specified Python file path"; - json_resp["message"] = "No specified Python file path"; - status_resp["status_code"] = k400BadRequest; - callback(std::move(status_resp), std::move(json_resp)); - return; + if (file_execution_path.empty()) { + LOG_ERROR << "No specified Python file path"; + json_resp["message"] = "No specified Python file path"; + status_resp["status_code"] = k400BadRequest; + callback(std::move(status_resp), std::move(json_resp)); + return; } json_resp["message"] = "Executing the Python file"; @@ -59,9 +59,10 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( #if defined(_WIN32) std::wstring exe_path = python_utils::getCurrentExecutablePath(); std::string exe_args_string = " --run_python_file " + file_execution_path; - if (python_library_path != "") - exe_args_string += " " + python_library_path; - std::wstring pyArgs = exe_path + python_utils::stringToWString(exe_args_string); + if (!python_library_path.empty()) + exe_args_string += " --python_library_path " + python_library_path; + std::wstring pyArgs = + exe_path + python_utils::stringToWString(exe_args_string); STARTUPINFOW si; PROCESS_INFORMATION pi; @@ -69,14 +70,18 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( si.cb = sizeof(si); ZeroMemory(&pi, sizeof(pi)); - if (!CreateProcessW(const_cast(exe_path.data()), // the path to the executable file - const_cast(pyArgs.data()), // command line arguments passed to the child - NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { - LOG_ERROR << "Failed to create child process: " << GetLastError(); - json_resp["message"] = "Failed to execute the Python file"; + if (!CreateProcessW( + const_cast(exe_path.c_str()), + const_cast(pyArgs.c_str()), + NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { + LOG_ERROR << "Failed to create child process: " << GetLastError(); + json_resp["message"] = "Failed to execute the Python file"; + status_resp["status_code"] = k500InternalServerError; } else { LOG_INFO << "Created child process for Python embedding"; WaitForSingleObject(pi.hProcess, INFINITE); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); } #else std::string child_process_exe_path = python_utils::getCurrentExecutablePath(); @@ -84,15 +89,37 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( child_process_args.push_back(const_cast(child_process_exe_path.c_str())); child_process_args.push_back(const_cast("--run_python_file")); child_process_args.push_back(const_cast(file_execution_path.c_str())); - if (python_library_path != "") - child_process_args.push_back(const_cast(python_library_path.c_str())); - child_process_args.push_back(nullptr); + + if (!python_library_path.empty()) { + child_process_args.push_back(const_cast(python_library_path.c_str())); + } + child_process_args.push_back(NULL); + pid_t pid; + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + + // Prepare environment variables + std::vector env_strings; + std::vector env_ptrs; + for (char** env = environ; *env != nullptr; env++) { + env_strings.push_back(*env); + } + + // Add or modify PYTHONPATH + std::string pythonpath = "PYTHONPATH=" + python_library_path; + env_strings.push_back(pythonpath); + + // Convert environment strings to char* pointers + for (const auto& s : env_strings) { + env_ptrs.push_back(const_cast(s.c_str())); + } + env_ptrs.push_back(nullptr); - int status = posix_spawn(&pid, child_process_exe_path.c_str(), nullptr, - nullptr, child_process_args.data(), environ); - - if (status) { + int status = posix_spawn(&pid, child_process_exe_path.c_str(), &actions, + nullptr, child_process_args.data(), env_ptrs.data()); + + if (status != 0) { LOG_ERROR << "Failed to spawn process: " << strerror(status); json_resp["message"] = "Failed to execute the Python file"; status_resp["status_code"] = k500InternalServerError; @@ -103,16 +130,24 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( LOG_ERROR << "Error waiting for child process"; json_resp["message"] = "Failed to execute the Python file"; status_resp["status_code"] = k500InternalServerError; + } else if (WIFEXITED(stat_loc)) { + int exit_status = WEXITSTATUS(stat_loc); + if (exit_status != 0) { + LOG_ERROR << "Child process exited with status: " << exit_status; + json_resp["message"] = "Python script execution failed"; + status_resp["status_code"] = k500InternalServerError; + } } } + + posix_spawn_file_actions_destroy(&actions); #endif callback(std::move(status_resp), std::move(json_resp)); - return; -}; +} extern "C" { CortexPythonEngineI* get_engine() { return new PythonEngine(); } -} // extern C +} // extern C \ No newline at end of file diff --git a/src/python_utils.h b/src/python_utils.h index 70c8317..0f6e8b6 100644 --- a/src/python_utils.h +++ b/src/python_utils.h @@ -1,39 +1,35 @@ #pragma once #include +#include +#include #include +#include #include #include -#include -#include - -#include "trantor/utils/Logger.h" +#include #if defined(_WIN32) - #define PY_DL HMODULE - #define PY_LOAD_LIB(path) LoadLibraryW(python_utils::stringToWString(path).c_str()); - #define GET_PY_FUNC GetProcAddress - #define PY_FREE_LIB FreeLibrary +#include +#define PY_DL HMODULE +#define PY_LOAD_LIB(path) \ + LoadLibraryW(python_utils::stringToWString(path).c_str()) +#define GET_PY_FUNC GetProcAddress +#define PY_FREE_LIB FreeLibrary #else - #define PY_DL void* - #define PY_LOAD_LIB(path) dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL); - #define GET_PY_FUNC dlsym - #define PY_FREE_LIB dlclose -#endif - -#ifdef _WIN32 - #include - #include -#else - #include - #include +#include +#define PY_DL void* +#define PY_LOAD_LIB(path) dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL) +#define GET_PY_FUNC dlsym +#define PY_FREE_LIB dlclose #endif #if __APPLE__ -#include #include #endif +#include "trantor/utils/Logger.h" + namespace python_utils { typedef long Py_ssize_t; @@ -44,85 +40,76 @@ typedef void (*PyErr_PrintFunc)(); typedef int (*PyRun_SimpleStringFunc)(const char*); typedef int (*PyRun_SimpleFileFunc)(FILE*, const char*); typedef int (*PyList_InsertFunc)(PyObject*, Py_ssize_t, PyObject*); -typedef int (*PyList_SetSliceFunc)(PyObject*, Py_ssize_t, Py_ssize_t, PyObject*); +typedef int (*PyList_SetSliceFunc)(PyObject*, Py_ssize_t, Py_ssize_t, + PyObject*); typedef PyObject* (*PySys_GetObjectFunc)(const char*); typedef PyObject* (*PyUnicode_FromStringFunc)(const char*); typedef Py_ssize_t (*PyList_SizeFunc)(PyObject*); inline void SignalHandler(int signum) { LOG_WARN << "Interrupt signal (" << signum << ") received."; - abort(); + std::abort(); } #if defined(_WIN32) inline std::wstring stringToWString(const std::string& str) { - if (str.empty()) return std::wstring(); - int size_needed = MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0); + if (str.empty()) + return std::wstring(); + int size_needed = + MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0); std::wstring wstrTo(size_needed, 0); - MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &wstrTo[0], size_needed); + MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &wstrTo[0], + size_needed); return wstrTo; } + inline std::wstring getCurrentExecutablePath() { wchar_t path[MAX_PATH]; DWORD result = GetModuleFileNameW(NULL, path, MAX_PATH); if (result == 0) { - std::wcerr << L"Error getting module file name." << std::endl; - return L""; + LOG_ERROR << "Error getting module file name: " << GetLastError(); + return L""; } return std::wstring(path); } -#else +#else inline std::string getCurrentExecutablePath() { #ifdef __APPLE__ char buf[PATH_MAX]; uint32_t bufsize = PATH_MAX; - if (_NSGetExecutablePath(buf, &bufsize) == 0) { return std::string(buf); } + LOG_ERROR << "Error getting executable path on macOS"; return ""; #else - std::vector buf(PATH_MAX); - ssize_t len = readlink("/proc/self/exe", &buf[0], buf.size()); - if (len == -1 || len == buf.size()) { - std::cerr << "Error reading symlink /proc/self/exe." << std::endl; - return ""; + char buf[4096]; + ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf) - 1); + if (len != -1) { + buf[len] = '\0'; + return std::string(buf); } - return std::string(&buf[0], len); + LOG_ERROR << "Error getting executable path on Linux: " << strerror(errno); + return ""; #endif } #endif -inline std::string GetDirectoryPathFromFilePath(std::string file_path) { - size_t last_forw_slash_pos = file_path.find_last_of('/'); - size_t last_back_slash_pos = file_path.find_last_of('\\'); - - size_t pos = std::string::npos; - if (last_forw_slash_pos != std::string::npos && last_back_slash_pos != std::string::npos) { - pos = (std::max)(last_forw_slash_pos, last_back_slash_pos); - } else if (last_forw_slash_pos != std::string::npos) { - pos = last_forw_slash_pos; - } else if (last_back_slash_pos != std::string::npos) { - pos = last_back_slash_pos; - } - if (pos != std::string::npos) { - return file_path.substr(0, pos + 1); - } else { - return "./"; +inline std::string GetDirectoryPathFromFilePath(const std::string& file_path) { + size_t last_separator = file_path.find_last_of("/\\"); + if (last_separator != std::string::npos) { + return file_path.substr(0, last_separator + 1); } + return "./"; } inline std::string FindPythonDynamicLib(const std::string& lib_dir) { std::string pattern; #if defined(_WIN32) || defined(_WIN64) - // Windows - // python310.dll pattern = "python[0-9][0-9]+\\.dll"; #elif defined(__APPLE__) || defined(__MACH__) - // macOS pattern = "libpython[0-9]+\\.[0-9]+\\.dylib"; #else - // Linux or other Unix-like systems pattern = "libpython[0-9]+\\.[0-9]+\\.so.*"; #endif std::regex regex_pattern(pattern); @@ -132,101 +119,157 @@ inline std::string FindPythonDynamicLib(const std::string& lib_dir) { return entry.path().string(); } } - return ""; // Return an empty string if no matching library is found + LOG_ERROR << "No Python dynamic library found in " << lib_dir; + return ""; } -inline void ClearAndSetPythonSysPath(std::string default_py_lib_path, PY_DL py_dl) { - auto python_sys_get_object_func = (PySys_GetObjectFunc)GET_PY_FUNC(py_dl, "PySys_GetObject"); - auto python_list_insert_func = (PyList_InsertFunc)GET_PY_FUNC(py_dl, "PyList_Insert"); - auto python_unicode_from_string_func = (PyUnicode_FromStringFunc)GET_PY_FUNC(py_dl, "PyUnicode_FromString"); - auto python_list_set_slice_func = (PyList_SetSliceFunc)GET_PY_FUNC(py_dl, "PyList_SetSlice"); - auto python_list_size_func = (PyList_SizeFunc)GET_PY_FUNC(py_dl, "PyList_Size"); +inline void ClearAndSetPythonSysPath(const std::string& default_py_lib_path, + PY_DL py_dl) { + auto python_sys_get_object_func = + (PySys_GetObjectFunc)GET_PY_FUNC(py_dl, "PySys_GetObject"); + auto python_list_insert_func = + (PyList_InsertFunc)GET_PY_FUNC(py_dl, "PyList_Insert"); + auto python_unicode_from_string_func = + (PyUnicode_FromStringFunc)GET_PY_FUNC(py_dl, "PyUnicode_FromString"); + auto python_list_set_slice_func = + (PyList_SetSliceFunc)GET_PY_FUNC(py_dl, "PyList_SetSlice"); + auto python_list_size_func = + (PyList_SizeFunc)GET_PY_FUNC(py_dl, "PyList_Size"); + + if (!python_sys_get_object_func || !python_list_insert_func || + !python_unicode_from_string_func || !python_list_set_slice_func || + !python_list_size_func) { + LOG_ERROR + << "Failed to get required Python functions for sys.path manipulation"; + return; + } + PyObject* sys_path = python_sys_get_object_func("path"); - python_list_set_slice_func(sys_path, 0, python_list_size_func(sys_path), NULL); + if (!sys_path) { + LOG_ERROR << "Failed to get sys.path"; + return; + } + + python_list_set_slice_func(sys_path, 0, python_list_size_func(sys_path), + NULL); + + std::vector path_strings; #if defined(_WIN32) - std::vector pathStrings = { - python_unicode_from_string_func((default_py_lib_path).c_str()), - python_unicode_from_string_func((default_py_lib_path + "DLLs/").c_str()), - python_unicode_from_string_func((default_py_lib_path + "Lib/").c_str()), - python_unicode_from_string_func((default_py_lib_path + "Lib/site-packages/").c_str())}; + path_strings = {default_py_lib_path, default_py_lib_path + "DLLs\\", + default_py_lib_path + "Lib\\", + default_py_lib_path + "Lib\\site-packages\\"}; #else - std::vector pathStrings = { - python_unicode_from_string_func((default_py_lib_path + "lib/python/").c_str()), - python_unicode_from_string_func((default_py_lib_path + "lib/python/lib-dynload/").c_str()), - python_unicode_from_string_func((default_py_lib_path + "lib/python/site-packages/").c_str())}; + path_strings = {default_py_lib_path + "lib/python/", + default_py_lib_path + "lib/python/lib-dynload/", + default_py_lib_path + "lib/python/site-packages/"}; #endif - if (sys_path) { - for(PyObject* pathString : pathStrings) { - python_list_insert_func(sys_path, 0, pathString); + for (const auto& path : path_strings) { + PyObject* path_obj = python_unicode_from_string_func(path.c_str()); + if (path_obj) { + python_list_insert_func(sys_path, 0, path_obj); + LOG_INFO << "Added to sys.path: " << path; + } else { + LOG_ERROR << "Failed to create Unicode object for path: " << path; } } } -inline void ExecutePythonFile(std::string binary_exec_path, std::string py_file_path ,std::string py_lib_path) { - +inline void ExecutePythonFile(const std::string& binary_exec_path, + const std::string& py_file_path, + const std::string& py_lib_path) { signal(SIGINT, SignalHandler); - std::string binary_dir_path = python_utils::GetDirectoryPathFromFilePath(binary_exec_path); + std::string binary_dir_path = GetDirectoryPathFromFilePath(binary_exec_path); - bool is_default_python_lib = false; - if (py_lib_path == "") { - is_default_python_lib = true; - py_lib_path = binary_dir_path + "engines/cortex.python/python/"; - LOG_WARN << "No specified Python library path, using default Python library in " << py_lib_path; + std::string effective_py_lib_path = py_lib_path; + if (effective_py_lib_path.empty()) { + effective_py_lib_path = binary_dir_path + "engines/cortex.python/python/"; + LOG_WARN << "No specified Python library path, using default: " + << effective_py_lib_path; } - std::string py_dl_path = FindPythonDynamicLib(py_lib_path); - if (py_dl_path == "") { - LOG_ERROR << "Could not find Python dynamic library file in path: " << py_lib_path; +// Set PYTHONHOME and PYTHONPATH environment variables +#if defined(_WIN32) + _putenv_s("PYTHONHOME", effective_py_lib_path.c_str()); + _putenv_s("PYTHONPATH", effective_py_lib_path.c_str()); +#else + setenv("PYTHONHOME", effective_py_lib_path.c_str(), 1); + setenv("PYTHONPATH", effective_py_lib_path.c_str(), 1); +#endif + + LOG_INFO << "Set PYTHONHOME to: " << effective_py_lib_path; + LOG_INFO << "Set PYTHONPATH to: " << effective_py_lib_path; + + std::string py_dl_path = FindPythonDynamicLib(effective_py_lib_path); + if (py_dl_path.empty()) { + LOG_ERROR << "Could not find Python dynamic library file in path: " + << effective_py_lib_path; return; - } else { - LOG_DEBUG << "Found dynamic library file " << py_dl_path;; } + LOG_INFO << "Found dynamic library file: " << py_dl_path; PY_DL py_dl = PY_LOAD_LIB(py_dl_path); if (!py_dl) { - LOG_ERROR << "Failed to load Python dynamic library from file: " << py_dl_path; + LOG_ERROR << "Failed to load Python dynamic library from file: " + << py_dl_path; return; - } else { - LOG_INFO << "Successully loaded Python dynamic library from path: " << py_dl_path; } + LOG_INFO << "Successfully loaded Python dynamic library from path: " + << py_dl_path; - auto python_initialize_func = (Py_InitializeFunc)GET_PY_FUNC(py_dl, "Py_Initialize"); - auto python_finalize_func = (Py_FinalizeFunc)GET_PY_FUNC(py_dl, "Py_Finalize"); + auto python_initialize_func = + (Py_InitializeFunc)GET_PY_FUNC(py_dl, "Py_Initialize"); + auto python_finalize_func = + (Py_FinalizeFunc)GET_PY_FUNC(py_dl, "Py_Finalize"); auto python_err_print = (PyErr_PrintFunc)GET_PY_FUNC(py_dl, "PyErr_Print"); - auto python_run_simple_string_func = (PyRun_SimpleStringFunc)GET_PY_FUNC(py_dl, "PyRun_SimpleString"); - auto python_run_simple_pile_func = (PyRun_SimpleFileFunc)GET_PY_FUNC(py_dl, "PyRun_SimpleFile"); + auto python_run_simple_string_func = + (PyRun_SimpleStringFunc)GET_PY_FUNC(py_dl, "PyRun_SimpleString"); + auto python_run_simple_file_func = + (PyRun_SimpleFileFunc)GET_PY_FUNC(py_dl, "PyRun_SimpleFile"); - if (!python_initialize_func || !python_finalize_func || !python_err_print - || !python_run_simple_string_func || !python_run_simple_pile_func) { + if (!python_initialize_func || !python_finalize_func || !python_err_print || + !python_run_simple_string_func || !python_run_simple_file_func) { LOG_ERROR << "Failed to bind necessary Python functions"; PY_FREE_LIB(py_dl); return; } - // Start Python runtime python_initialize_func(); - if (is_default_python_lib) { - // Re-route the sys paths to Python default library - ClearAndSetPythonSysPath(py_lib_path, py_dl); - } + // Set sys.path + ClearAndSetPythonSysPath(effective_py_lib_path, py_dl); - LOG_INFO << "Trying to run Python file in path " << py_file_path; + // Add debug logging + python_run_simple_string_func(R"( +import sys +import os +print("Python version:", sys.version) +print("Python sys.path:", sys.path) +print("Python executable:", sys.executable) +print("Python prefix:", sys.prefix) +print("Python exec prefix:", sys.exec_prefix) +print("PYTHONHOME:", os.environ.get('PYTHONHOME', 'Not set')) +print("PYTHONPATH:", os.environ.get('PYTHONPATH', 'Not set')) + )"); + + LOG_INFO << "Trying to run Python file: " << py_file_path; FILE* file = fopen(py_file_path.c_str(), "r"); if (file == NULL) { - LOG_ERROR << "Failed to open file " << py_file_path; + LOG_ERROR << "Failed to open file: " << py_file_path + << ", error: " << strerror(errno); } else { - if (python_run_simple_pile_func(file, py_file_path.c_str() ) != 0) { + if (python_run_simple_file_func(file, py_file_path.c_str()) != 0) { python_err_print(); - LOG_ERROR << "Failed to execute file " << py_file_path; + LOG_ERROR << "Failed to execute file: " << py_file_path; + } else { + LOG_INFO << "Successfully executed Python file: " << py_file_path; } fclose(file); } python_finalize_func(); PY_FREE_LIB(py_dl); + LOG_INFO << "Python execution completed"; } - -} // namespace python_utils +} // namespace python_utils \ No newline at end of file From 51091b6a98f15755d727d80ba1b468ecae7a1081 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Thu, 17 Oct 2024 23:59:10 +0700 Subject: [PATCH 3/3] fix: run in window --- src/python_engine.cc | 84 ++++++++++++++++++++++++++++------ src/python_utils.h | 61 ++++++++++++++++++------ third-party/install_python.bat | 2 +- 3 files changed, 120 insertions(+), 27 deletions(-) diff --git a/src/python_engine.cc b/src/python_engine.cc index 19dc7fe..c8a9a23 100644 --- a/src/python_engine.cc +++ b/src/python_engine.cc @@ -4,6 +4,9 @@ #if defined(_WIN32) #include +#include +#include +#pragma comment(lib, "userenv.lib") #else #include #include @@ -58,11 +61,14 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( #if defined(_WIN32) std::wstring exe_path = python_utils::getCurrentExecutablePath(); - std::string exe_args_string = " --run_python_file " + file_execution_path; - if (!python_library_path.empty()) - exe_args_string += " --python_library_path " + python_library_path; - std::wstring pyArgs = - exe_path + python_utils::stringToWString(exe_args_string); + std::wstring command_line = + L"\"" + exe_path + L"\" --run_python_file \"" + + python_utils::stringToWString(file_execution_path) + L"\""; + + if (!python_library_path.empty()) { + command_line += + L" \"" + python_utils::stringToWString(python_library_path) + L"\""; + } STARTUPINFOW si; PROCESS_INFORMATION pi; @@ -70,28 +76,80 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( si.cb = sizeof(si); ZeroMemory(&pi, sizeof(pi)); - if (!CreateProcessW( - const_cast(exe_path.c_str()), - const_cast(pyArgs.c_str()), - NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { - LOG_ERROR << "Failed to create child process: " << GetLastError(); + // Prepare environment block + LPWCH envBlock = GetEnvironmentStringsW(); + if (envBlock == NULL) { + LOG_ERROR << "Failed to get environment block"; json_resp["message"] = "Failed to execute the Python file"; status_resp["status_code"] = k500InternalServerError; + callback(std::move(status_resp), std::move(json_resp)); + return; + } + + std::wstring env_block; + for (LPCWSTR env = envBlock; *env != L'\0'; env += wcslen(env) + 1) { + env_block += env; + env_block += L'\0'; + } + FreeEnvironmentStringsW(envBlock); + + // Add or modify PYTHONPATH + std::wstring pythonpath = + L"PYTHONPATH=" + python_utils::stringToWString(python_library_path); + env_block += pythonpath; + env_block += L'\0'; + env_block += L'\0'; // Double null-termination + + LPVOID pEnv = NULL; + if (!CreateEnvironmentBlock(&pEnv, NULL, FALSE)) { + LOG_ERROR << "Failed to create environment block"; + json_resp["message"] = "Failed to execute the Python file"; + status_resp["status_code"] = k500InternalServerError; + callback(std::move(status_resp), std::move(json_resp)); + return; + } + + if (!CreateProcessW(NULL, const_cast(command_line.c_str()), NULL, + NULL, FALSE, CREATE_UNICODE_ENVIRONMENT, pEnv, NULL, &si, + &pi)) { + DWORD error = GetLastError(); + LOG_ERROR << "Failed to create child process: " << error; + json_resp["message"] = "Failed to execute the Python file. Error code: " + + std::to_string(error); + status_resp["status_code"] = k500InternalServerError; } else { LOG_INFO << "Created child process for Python embedding"; WaitForSingleObject(pi.hProcess, INFINITE); + + DWORD exit_code; + if (GetExitCodeProcess(pi.hProcess, &exit_code)) { + if (exit_code != 0) { + LOG_ERROR << "Child process exited with status: " << exit_code; + json_resp["message"] = "Python script execution failed"; + status_resp["status_code"] = k500InternalServerError; + } + } else { + LOG_ERROR << "Failed to get exit code: " << GetLastError(); + json_resp["message"] = "Failed to get Python script execution status"; + status_resp["status_code"] = k500InternalServerError; + } + CloseHandle(pi.hProcess); CloseHandle(pi.hThread); } + + DestroyEnvironmentBlock(pEnv); #else std::string child_process_exe_path = python_utils::getCurrentExecutablePath(); std::vector child_process_args; - child_process_args.push_back(const_cast(child_process_exe_path.c_str())); + child_process_args.push_back( + const_cast(child_process_exe_path.c_str())); child_process_args.push_back(const_cast("--run_python_file")); child_process_args.push_back(const_cast(file_execution_path.c_str())); if (!python_library_path.empty()) { - child_process_args.push_back(const_cast(python_library_path.c_str())); + child_process_args.push_back( + const_cast(python_library_path.c_str())); } child_process_args.push_back(NULL); @@ -105,7 +163,7 @@ void PythonEngine::HandlePythonFileExecutionRequestImpl( for (char** env = environ; *env != nullptr; env++) { env_strings.push_back(*env); } - + // Add or modify PYTHONPATH std::string pythonpath = "PYTHONPATH=" + python_library_path; env_strings.push_back(pythonpath); diff --git a/src/python_utils.h b/src/python_utils.h index 0f6e8b6..ccf6fb0 100644 --- a/src/python_utils.h +++ b/src/python_utils.h @@ -102,25 +102,57 @@ inline std::string GetDirectoryPathFromFilePath(const std::string& file_path) { } return "./"; } - inline std::string FindPythonDynamicLib(const std::string& lib_dir) { - std::string pattern; + try { + LOG_INFO << "Entering FindPythonDynamicLib, lib_dir: " << lib_dir; + + if (lib_dir.empty()) { + LOG_ERROR << "lib_dir is empty"; + return ""; + } + + std::string pattern; #if defined(_WIN32) || defined(_WIN64) - pattern = "python[0-9][0-9]+\\.dll"; + pattern = "python[0-9][0-9]+\\.dll"; #elif defined(__APPLE__) || defined(__MACH__) - pattern = "libpython[0-9]+\\.[0-9]+\\.dylib"; + pattern = "libpython[0-9]+\\.[0-9]+\\.dylib"; #else - pattern = "libpython[0-9]+\\.[0-9]+\\.so.*"; + pattern = "libpython[0-9]+\\.[0-9]+\\.so.*"; #endif - std::regex regex_pattern(pattern); - for (const auto& entry : std::filesystem::directory_iterator(lib_dir)) { - std::string file_name = entry.path().filename().string(); - if (std::regex_match(file_name, regex_pattern)) { - return entry.path().string(); + LOG_INFO << "Pattern: " << pattern; + + std::regex regex_pattern(pattern); + LOG_INFO << "Regex pattern compiled successfully"; + + if (!std::filesystem::exists(lib_dir)) { + LOG_ERROR << "Directory does not exist: " << lib_dir; + return ""; } + + if (!std::filesystem::is_directory(lib_dir)) { + LOG_ERROR << "Path is not a directory: " << lib_dir; + return ""; + } + + LOG_INFO << "Starting directory iteration"; + for (const auto& entry : std::filesystem::directory_iterator(lib_dir)) { + std::string file_name = entry.path().filename().string(); + LOG_INFO << "File name: " << file_name; + if (std::regex_match(file_name, regex_pattern)) { + LOG_INFO << "Match found: " << entry.path().string(); + return entry.path().string(); + } + } + + LOG_ERROR << "No Python dynamic library found in " << lib_dir; + return ""; + } catch (const std::exception& e) { + LOG_ERROR << "Exception in FindPythonDynamicLib: " << e.what(); + return ""; + } catch (...) { + LOG_ERROR << "Unknown exception in FindPythonDynamicLib"; + return ""; } - LOG_ERROR << "No Python dynamic library found in " << lib_dir; - return ""; } inline void ClearAndSetPythonSysPath(const std::string& default_py_lib_path, @@ -183,7 +215,10 @@ inline void ExecutePythonFile(const std::string& binary_exec_path, std::string effective_py_lib_path = py_lib_path; if (effective_py_lib_path.empty()) { - effective_py_lib_path = binary_dir_path + "engines/cortex.python/python/"; + effective_py_lib_path = + (std::filesystem::path(binary_dir_path) / + std::filesystem::path("engines/cortex.python/python/")) + .string(); LOG_WARN << "No specified Python library path, using default: " << effective_py_lib_path; } diff --git a/third-party/install_python.bat b/third-party/install_python.bat index 503ddcc..82e3e6b 100644 --- a/third-party/install_python.bat +++ b/third-party/install_python.bat @@ -8,7 +8,7 @@ if not exist "%INSTALL_DIR%" mkdir "%INSTALL_DIR%" cd %INSTALL_DIR% echo Downloading Python... -curl -o python-installer.exe https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe +curl -o python-installer.exe https://www.python.org/ftp/python/3.12.7/python-3.12.7-amd64.exe echo Installing Python to %INSTALL_DIR% start /wait python-installer.exe /quiet InstallAllUsers=0 PrependPath=0 TargetDir=%INSTALL_DIR% Include_pip=1 Include_test=0