From d3414a7f4bdcb4f2fd43cdbb616e6405632f429d Mon Sep 17 00:00:00 2001 From: Clarence-1103 Date: Tue, 18 Nov 2025 17:33:51 -0800 Subject: [PATCH] fix open BUILD NUMA --- CMakeLists.txt | 1 + ucm/sparse/esa/CMakeLists.txt | 78 ++++++++++--------- ucm/sparse/esa/retrieval/CMakeLists.txt | 2 +- .../esa/retrieval/cpy/retrieval_backend.cpp | 8 +- ucm/sparse/kvcomp/CMakeLists.txt | 78 ++++++++++--------- .../kvcomp/hash_retrieval/CMakeLists.txt | 2 +- .../cpy/hash_retrieval_backend.cpp | 9 ++- ucm/sparse/kvstar/retrieve/CMakeLists.txt | 78 ++++++++++--------- .../kvstar/retrieve/core/CMakeLists.txt | 2 +- .../retrieve_task/retrieve_task_queue.cpp | 6 ++ 10 files changed, 154 insertions(+), 110 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 788dc15c..32e5e115 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(BUILD_UCM_STORE "build ucm store module." ON) option(BUILD_UCM_SPARSE "build ucm sparse module." ON) option(BUILD_UNIT_TESTS "build all unit test suits." OFF) +option(BUILD_NUMA "build numactl library" OFF) set(RUNTIME_ENVIRONMENT "simu" CACHE STRING "runtime: simu, ascend or cuda.") execute_process(COMMAND git rev-parse HEAD OUTPUT_VARIABLE UCM_COMMIT_ID OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/ucm/sparse/esa/CMakeLists.txt b/ucm/sparse/esa/CMakeLists.txt index f6192b35..9d2afcdb 100644 --- a/ucm/sparse/esa/CMakeLists.txt +++ b/ucm/sparse/esa/CMakeLists.txt @@ -1,42 +1,50 @@ -set(NUMA_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/numa_install) -FetchContent_Declare( - numactl - URL https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz - TLS_VERIFY OFF -) -FetchContent_MakeAvailable(numactl) -if(NOT EXISTS "${NUMA_INSTALL_DIR}/lib/libnuma.so") - message(STATUS "Configuring numactl...") - execute_process( - COMMAND ./configure --prefix=${NUMA_INSTALL_DIR} - WORKING_DIRECTORY ${numactl_SOURCE_DIR} - RESULT_VARIABLE numa_configure_result - OUTPUT_VARIABLE numa_configure_output - ERROR_VARIABLE numa_configure_error - ) - if(NOT numa_configure_result EQUAL 0) - message(FATAL_ERROR "Failed to configure numactl. \n" - "Result: ${numa_configure_result}\n" - "STDOUT: ${numa_configure_output}\n" - "STDERR: ${numa_configure_error}\n") - endif() +if(BUILD_NUMA) + message(STATUS "Building numactl library...") - message(STATUS "Building and installing numactl...") - execute_process( - COMMAND make install -j8 - WORKING_DIRECTORY ${numactl_SOURCE_DIR} - RESULT_VARIABLE numa_install_result - OUTPUT_VARIABLE numa_install_output - ERROR_VARIABLE numa_install_error + set(NUMA_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/numa_install) + FetchContent_Declare( + numactl + URL https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz + TLS_VERIFY OFF ) - if(NOT numa_install_result EQUAL 0) - message(FATAL_ERROR "Failed to build and install numactl. \n" - "Result: ${numa_install_result}\n" - "STDOUT: ${numa_install_output}\n" - "STDERR: ${numa_install_error}\n") + FetchContent_MakeAvailable(numactl) + if(NOT EXISTS "${NUMA_INSTALL_DIR}/lib/libnuma.so") + message(STATUS "Configuring numactl...") + execute_process( + COMMAND ./configure --prefix=${NUMA_INSTALL_DIR} + WORKING_DIRECTORY ${numactl_SOURCE_DIR} + RESULT_VARIABLE numa_configure_result + OUTPUT_VARIABLE numa_configure_output + ERROR_VARIABLE numa_configure_error + ) + if(NOT numa_configure_result EQUAL 0) + message(FATAL_ERROR "Failed to configure numactl. \n" + "Result: ${numa_configure_result}\n" + "STDOUT: ${numa_configure_output}\n" + "STDERR: ${numa_configure_error}\n") + endif() + + message(STATUS "Building and installing numactl...") + execute_process( + COMMAND make install -j8 + WORKING_DIRECTORY ${numactl_SOURCE_DIR} + RESULT_VARIABLE numa_install_result + OUTPUT_VARIABLE numa_install_output + ERROR_VARIABLE numa_install_error + ) + if(NOT numa_install_result EQUAL 0) + message(FATAL_ERROR "Failed to build and install numactl. \n" + "Result: ${numa_install_result}\n" + "STDOUT: ${numa_install_output}\n" + "STDERR: ${numa_install_error}\n") + endif() + else() + message(STATUS "Found already built libnuma. Skipping build.") endif() + + add_definitions(-DNUMA_ENABLED) else() - message(STATUS "Found already built libnuma. Skipping build.") + message(STATUS "Skipping numactl build...") endif() add_subdirectory(retrieval) diff --git a/ucm/sparse/esa/retrieval/CMakeLists.txt b/ucm/sparse/esa/retrieval/CMakeLists.txt index ad05d7bc..aabb02e5 100644 --- a/ucm/sparse/esa/retrieval/CMakeLists.txt +++ b/ucm/sparse/esa/retrieval/CMakeLists.txt @@ -12,6 +12,6 @@ target_include_directories(retrieval_backend PUBLIC # 链接所需的库 target_link_libraries(retrieval_backend PUBLIC - ${NUMA_INSTALL_DIR}/lib/libnuma.so + $<$:${NUMA_INSTALL_DIR}/lib/libnuma.so> ${Torch_LIBRARIES} ) diff --git a/ucm/sparse/esa/retrieval/cpy/retrieval_backend.cpp b/ucm/sparse/esa/retrieval/cpy/retrieval_backend.cpp index b0f3ea07..dacf3868 100644 --- a/ucm/sparse/esa/retrieval/cpy/retrieval_backend.cpp +++ b/ucm/sparse/esa/retrieval/cpy/retrieval_backend.cpp @@ -11,7 +11,9 @@ #include #include #include +#ifdef NUMA_ENABLED #include +#endif #include namespace py = pybind11; @@ -29,7 +31,6 @@ class RetrievalWorkerBackend { // Start worker threads for (auto cpu_idx : cpu_idx_tbl) { - int numaId = cpu_idx.first.cast(); py::list core_ids = cpu_idx.second.cast(); for (size_t i = 0; i < core_ids.size(); ++i) { @@ -49,12 +50,17 @@ class RetrievalWorkerBackend { std::cerr << "Error binding thread " << i << " to CPU core " << core_id << std::endl; } + #ifdef NUMA_ENABLED + int numaId = cpu_idx.first.cast(); // 设置内存亲和性 unsigned long nodeMask = 1UL << numaId; rc = set_mempolicy(MPOL_BIND, &nodeMask, sizeof(nodeMask) * 8); if (rc != 0) { std::cerr << "Error binding memory to NUMA node " << numaId << std::endl; } + #else + std::cerr << "NUMA support is disabled." << std::endl; + #endif } } diff --git a/ucm/sparse/kvcomp/CMakeLists.txt b/ucm/sparse/kvcomp/CMakeLists.txt index be24b30b..fb32a3ec 100644 --- a/ucm/sparse/kvcomp/CMakeLists.txt +++ b/ucm/sparse/kvcomp/CMakeLists.txt @@ -1,42 +1,50 @@ -set(NUMA_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/numa_install) -FetchContent_Declare( - numactl - URL https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz - TLS_VERIFY OFF -) -FetchContent_MakeAvailable(numactl) -if(NOT EXISTS "${NUMA_INSTALL_DIR}/lib/libnuma.so") - message(STATUS "Configuring numactl...") - execute_process( - COMMAND ./configure --prefix=${NUMA_INSTALL_DIR} - WORKING_DIRECTORY ${numactl_SOURCE_DIR} - RESULT_VARIABLE numa_configure_result - OUTPUT_VARIABLE numa_configure_output - ERROR_VARIABLE numa_configure_error - ) - if(NOT numa_configure_result EQUAL 0) - message(FATAL_ERROR "Failed to configure numactl. \n" - "Result: ${numa_configure_result}\n" - "STDOUT: ${numa_configure_output}\n" - "STDERR: ${numa_configure_error}\n") - endif() +if(BUILD_NUMA) + message(STATUS "Building numactl library...") - message(STATUS "Building and installing numactl...") - execute_process( - COMMAND make install -j8 - WORKING_DIRECTORY ${numactl_SOURCE_DIR} - RESULT_VARIABLE numa_install_result - OUTPUT_VARIABLE numa_install_output - ERROR_VARIABLE numa_install_error + set(NUMA_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/numa_install) + FetchContent_Declare( + numactl + URL https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz + TLS_VERIFY OFF ) - if(NOT numa_install_result EQUAL 0) - message(FATAL_ERROR "Failed to build and install numactl. \n" - "Result: ${numa_install_result}\n" - "STDOUT: ${numa_install_output}\n" - "STDERR: ${numa_install_error}\n") + FetchContent_MakeAvailable(numactl) + if(NOT EXISTS "${NUMA_INSTALL_DIR}/lib/libnuma.so") + message(STATUS "Configuring numactl...") + execute_process( + COMMAND ./configure --prefix=${NUMA_INSTALL_DIR} + WORKING_DIRECTORY ${numactl_SOURCE_DIR} + RESULT_VARIABLE numa_configure_result + OUTPUT_VARIABLE numa_configure_output + ERROR_VARIABLE numa_configure_error + ) + if(NOT numa_configure_result EQUAL 0) + message(FATAL_ERROR "Failed to configure numactl. \n" + "Result: ${numa_configure_result}\n" + "STDOUT: ${numa_configure_output}\n" + "STDERR: ${numa_configure_error}\n") + endif() + + message(STATUS "Building and installing numactl...") + execute_process( + COMMAND make install -j8 + WORKING_DIRECTORY ${numactl_SOURCE_DIR} + RESULT_VARIABLE numa_install_result + OUTPUT_VARIABLE numa_install_output + ERROR_VARIABLE numa_install_error + ) + if(NOT numa_install_result EQUAL 0) + message(FATAL_ERROR "Failed to build and install numactl. \n" + "Result: ${numa_install_result}\n" + "STDOUT: ${numa_install_output}\n" + "STDERR: ${numa_install_error}\n") + endif() + else() + message(STATUS "Found already built libnuma. Skipping build.") endif() + + add_definitions(-DNUMA_ENABLED) else() - message(STATUS "Found already built libnuma. Skipping build.") + message(STATUS "Skipping numactl build...") endif() add_subdirectory(hash_retrieval) diff --git a/ucm/sparse/kvcomp/hash_retrieval/CMakeLists.txt b/ucm/sparse/kvcomp/hash_retrieval/CMakeLists.txt index 5f2e8f7f..5be9a6fe 100644 --- a/ucm/sparse/kvcomp/hash_retrieval/CMakeLists.txt +++ b/ucm/sparse/kvcomp/hash_retrieval/CMakeLists.txt @@ -12,6 +12,6 @@ target_include_directories(hash_retrieval_backend PUBLIC # 链接所需的库 target_link_libraries(hash_retrieval_backend PUBLIC - ${NUMA_INSTALL_DIR}/lib/libnuma.so + $<$:${NUMA_INSTALL_DIR}/lib/libnuma.so> ${Torch_LIBRARIES} ) \ No newline at end of file diff --git a/ucm/sparse/kvcomp/hash_retrieval/cpy/hash_retrieval_backend.cpp b/ucm/sparse/kvcomp/hash_retrieval/cpy/hash_retrieval_backend.cpp index 18bf393a..d1eedf05 100644 --- a/ucm/sparse/kvcomp/hash_retrieval/cpy/hash_retrieval_backend.cpp +++ b/ucm/sparse/kvcomp/hash_retrieval/cpy/hash_retrieval_backend.cpp @@ -16,7 +16,9 @@ #include #include // 用于UINT16_MAX #include +#ifdef NUMA_ENABLED #include +#endif #ifdef __ARM_NEON #include // ARM NEON SIMD 指令集头文件 #elif defined(__x86_64__) || defined(_M_X64) @@ -43,7 +45,6 @@ class HashRetrievalWorkerBackend { // Start worker threads for (auto cpu_idx : cpu_idx_tbl) { - int numaId = cpu_idx.first.cast(); py::list core_ids = cpu_idx.second.cast(); for (size_t i = 0; i < core_ids.size(); ++i) { @@ -63,12 +64,18 @@ class HashRetrievalWorkerBackend { std::cerr << "Error binding thread " << i << " to CPU core " << core_id << std::endl; } + #ifdef NUMA_ENABLED + int numaId = cpu_idx.first.cast(); // 设置内存亲和性 unsigned long nodeMask = 1UL << numaId; rc = set_mempolicy(MPOL_BIND, &nodeMask, sizeof(nodeMask) * 8); if (rc != 0) { std::cerr << "Error binding memory to NUMA node " << numaId << std::endl; } + #else + std::cerr << "NUMA support is disabled." << std::endl; + #endif + } } diff --git a/ucm/sparse/kvstar/retrieve/CMakeLists.txt b/ucm/sparse/kvstar/retrieve/CMakeLists.txt index 9a2655d8..3f677776 100644 --- a/ucm/sparse/kvstar/retrieve/CMakeLists.txt +++ b/ucm/sparse/kvstar/retrieve/CMakeLists.txt @@ -54,45 +54,53 @@ set(Torch_DIR ${PYTORCH_PATH}/share/cmake/Torch/) find_package(Torch REQUIRED) include_directories(${TORCH_INCLUDE_DIRS}) -set(NUMA_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/numa_install) -FetchContent_Declare( - numactl - URL https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz - TLS_VERIFY OFF -) -FetchContent_MakeAvailable(numactl) -if(NOT EXISTS "${NUMA_INSTALL_DIR}/lib/libnuma.so") - message(STATUS "Configuring numactl...") - execute_process( - COMMAND ./configure --prefix=${NUMA_INSTALL_DIR} - WORKING_DIRECTORY ${numactl_SOURCE_DIR} - RESULT_VARIABLE numa_configure_result - OUTPUT_VARIABLE numa_configure_output - ERROR_VARIABLE numa_configure_error - ) - if(NOT numa_configure_result EQUAL 0) - message(FATAL_ERROR "Failed to configure numactl. \n" - "Result: ${numa_configure_result}\n" - "STDOUT: ${numa_configure_output}\n" - "STDERR: ${numa_configure_error}\n") - endif() +if(BUILD_NUMA) + message(STATUS "Building numactl library...") - message(STATUS "Building and installing numactl...") - execute_process( - COMMAND make install -j8 - WORKING_DIRECTORY ${numactl_SOURCE_DIR} - RESULT_VARIABLE numa_install_result - OUTPUT_VARIABLE numa_install_output - ERROR_VARIABLE numa_install_error + set(NUMA_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/numa_install) + FetchContent_Declare( + numactl + URL https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz + TLS_VERIFY OFF ) - if(NOT numa_install_result EQUAL 0) - message(FATAL_ERROR "Failed to build and install numactl. \n" - "Result: ${numa_install_result}\n" - "STDOUT: ${numa_install_output}\n" - "STDERR: ${numa_install_error}\n") + FetchContent_MakeAvailable(numactl) + if(NOT EXISTS "${NUMA_INSTALL_DIR}/lib/libnuma.so") + message(STATUS "Configuring numactl...") + execute_process( + COMMAND ./configure --prefix=${NUMA_INSTALL_DIR} + WORKING_DIRECTORY ${numactl_SOURCE_DIR} + RESULT_VARIABLE numa_configure_result + OUTPUT_VARIABLE numa_configure_output + ERROR_VARIABLE numa_configure_error + ) + if(NOT numa_configure_result EQUAL 0) + message(FATAL_ERROR "Failed to configure numactl. \n" + "Result: ${numa_configure_result}\n" + "STDOUT: ${numa_configure_output}\n" + "STDERR: ${numa_configure_error}\n") + endif() + + message(STATUS "Building and installing numactl...") + execute_process( + COMMAND make install -j8 + WORKING_DIRECTORY ${numactl_SOURCE_DIR} + RESULT_VARIABLE numa_install_result + OUTPUT_VARIABLE numa_install_output + ERROR_VARIABLE numa_install_error + ) + if(NOT numa_install_result EQUAL 0) + message(FATAL_ERROR "Failed to build and install numactl. \n" + "Result: ${numa_install_result}\n" + "STDOUT: ${numa_install_output}\n" + "STDERR: ${numa_install_error}\n") + endif() + else() + message(STATUS "Found already built libnuma. Skipping build.") endif() + + add_definitions(-DNUMA_ENABLED) else() - message(STATUS "Found already built libnuma. Skipping build.") + message(STATUS "Skipping numactl build...") endif() add_subdirectory(core) diff --git a/ucm/sparse/kvstar/retrieve/core/CMakeLists.txt b/ucm/sparse/kvstar/retrieve/core/CMakeLists.txt index dc6b5412..876166e8 100644 --- a/ucm/sparse/kvstar/retrieve/core/CMakeLists.txt +++ b/ucm/sparse/kvstar/retrieve/core/CMakeLists.txt @@ -11,6 +11,6 @@ target_include_directories(kvstar_retrieve.core PUBLIC target_link_libraries(kvstar_retrieve.core PUBLIC spdlog::spdlog fmt::fmt - ${NUMA_INSTALL_DIR}/lib/libnuma.so + $<$:${NUMA_INSTALL_DIR}/lib/libnuma.so> ${Torch_LIBRARIES} ) diff --git a/ucm/sparse/kvstar/retrieve/core/domain/retrieve_task/retrieve_task_queue.cpp b/ucm/sparse/kvstar/retrieve/core/domain/retrieve_task/retrieve_task_queue.cpp index 6ded55f0..a8fc080a 100644 --- a/ucm/sparse/kvstar/retrieve/core/domain/retrieve_task/retrieve_task_queue.cpp +++ b/ucm/sparse/kvstar/retrieve/core/domain/retrieve_task/retrieve_task_queue.cpp @@ -1,4 +1,6 @@ +#ifdef NUMA_ENABLED #include +#endif #include "retrieve_task_queue.h" #include "retrieve_task_runner.h" @@ -27,6 +29,7 @@ void RetrieveTaskQueue::Worker(const int numaId, const int bindCoreId, std::prom return; } +#ifdef NUMA_ENABLED unsigned long nodemask = 1UL << numaId; rc = set_mempolicy(MPOL_BIND, &nodemask, sizeof(nodemask) * 8); if (rc != 0) { @@ -34,6 +37,9 @@ void RetrieveTaskQueue::Worker(const int numaId, const int bindCoreId, std::prom started.set_value(Status::OsApiError()); return; } +#else + KVSTAR_DEBUG("NUMA support is disabled."); +#endif KVSTAR_DEBUG("Bind current thread {} to numa {} core {} and set memory affinity success.", thread, numaId, bindCoreId); RetrieveTaskRunner runner;