From f10ee0f82e9f05ff8329dd7b5000e75b63c03954 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 25 Oct 2024 15:06:26 +0200 Subject: [PATCH 01/82] Disable temporarily failing CI job with ICX compiler Signed-off-by: Lukasz Dorau --- .github/workflows/basic.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 7c3d2ebc95..bf4f31f909 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -58,12 +58,15 @@ jobs: level_zero_provider: 'OFF' install_tbb: 'ON' # test icx compiler - - os: 'ubuntu-22.04' - build_type: Release - compiler: {c: icx, cxx: icpx} - shared_library: 'ON' - level_zero_provider: 'ON' - install_tbb: 'ON' + # - os: 'ubuntu-22.04' + # build_type: Release + # compiler: {c: icx, cxx: icpx} + # shared_library: 'ON' + # level_zero_provider: 'ON' + # cuda_provider: 'ON' + # install_tbb: 'ON' + # disable_hwloc: 'OFF' + # link_hwloc_statically: 'OFF' # test without installing TBB - os: 'ubuntu-22.04' build_type: Release From ef1e1cd56ad726c8a3d1e1b46cd044ebf98981c5 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Thu, 31 Oct 2024 15:57:52 +0100 Subject: [PATCH 02/82] fix for the apply of the HWLOC security patch --- CMakeLists.txt | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb40457764..f4f6b8c2ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,12 +131,6 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) set(HWLOC_ENABLE_TESTING OFF) set(HWLOC_SKIP_LSTOPO ON) set(HWLOC_SKIP_TOOLS ON) - set(HWLOC_PATCH - git - apply - ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch - || - (exit 0)) message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") @@ -144,8 +138,7 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) hwloc_targ GIT_REPOSITORY ${UMF_HWLOC_REPO} GIT_TAG ${UMF_HWLOC_TAG} - PATCH_COMMAND ${HWLOC_PATCH} SOURCE_SUBDIR contrib/windows-cmake/ - FIND_PACKAGE_ARGS) + SOURCE_SUBDIR contrib/windows-cmake/ FIND_PACKAGE_ARGS) FetchContent_GetProperties(hwloc_targ) if(NOT hwloc_targ_POPULATED) @@ -162,20 +155,13 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") elseif(NOT UMF_DISABLE_HWLOC) include(FetchContent) - set(HWLOC_PATCH - git - apply - ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch - || - (exit 0)) message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") FetchContent_Declare( hwloc_targ GIT_REPOSITORY ${UMF_HWLOC_REPO} - GIT_TAG ${UMF_HWLOC_TAG} - PATCH_COMMAND ${HWLOC_PATCH}) + GIT_TAG ${UMF_HWLOC_TAG}) FetchContent_GetProperties(hwloc_targ) if(NOT hwloc_targ_POPULATED) @@ -222,6 +208,22 @@ elseif(NOT UMF_DISABLE_HWLOC) message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") endif() +if(hwloc_targ_SOURCE_DIR) + # apply security patch for HWLOC + execute_process( + COMMAND git apply ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch + WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} + OUTPUT_VARIABLE UMF_HWLOC_PATCH_OUTPUT + ERROR_VARIABLE UMF_HWLOC_PATCH_ERROR) + + if(UMF_HWLOC_PATCH_OUTPUT) + message(STATUS "HWLOC patch command output:\n${UMF_HWLOC_PATCH_OUTPUT}") + endif() + if(UMF_HWLOC_PATCH_ERROR) + message(WARNING "HWLOC patch command output:\n${UMF_HWLOC_PATCH_ERROR}") + endif() +endif() + # This build type check is not possible on Windows when CMAKE_BUILD_TYPE is not # set, because in this case the build type is determined after a CMake # configuration is done (at the build time) From 8923281e0f9b1dc905f7fb0f2c4ca155dfd49a3d Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Thu, 3 Oct 2024 16:11:36 +0200 Subject: [PATCH 03/82] Fix disabling of pci support in hwloc The flag that hwloc recognize is --disable-pci, not --disable-pciaccess. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb40457764..0fe20025a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,7 +189,7 @@ elseif(NOT UMF_DISABLE_HWLOC) add_custom_command( COMMAND ./configure --prefix=${hwloc_targ_BINARY_DIR} --enable-static=yes - --enable-shared=no --disable-libxml2 --disable-pciaccess + --enable-shared=no --disable-libxml2 --disable-pci --disable-levelzero --disable-opencl --disable-cuda --disable-nvml CFLAGS=-fPIC CXXFLAGS=-fPIC WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} From d107b05f379414e5c90ac903efd29dfbab6497d9 Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Fri, 8 Nov 2024 10:14:50 +0100 Subject: [PATCH 04/82] Add a RUNPATH to installed libraries --- src/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0ebd1160f8..76479926c1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -153,6 +153,9 @@ elseif(MACOSX) endif() if(UMF_BUILD_SHARED_LIBRARY) + # Set the runtime search path to the directory containing hwloc library + set(CMAKE_INSTALL_RPATH "\$ORIGIN") + if(NOT UMF_DISABLE_HWLOC) set(HWLOC_LIB ${UMF_HWLOC_NAME}) endif() From 23298fe3c4a643880bd5aac52ee3364c4f94a636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 12 Nov 2024 10:57:04 +0100 Subject: [PATCH 05/82] 0.9.1 release --- .github/workflows/basic.yml | 2 +- ChangeLog | 6 ++++++ scripts/docs_config/conf.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index bf4f31f909..4652a7a1d1 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -8,7 +8,7 @@ permissions: env: # for installation testing - it should match with version set in CMake - UMF_VERSION: 0.9.0 + UMF_VERSION: 0.9.1 BUILD_DIR : "${{github.workspace}}/build" INSTL_DIR : "${{github.workspace}}/../install-dir" diff --git a/ChangeLog b/ChangeLog index 867e59f0ff..2b41d7d9b8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Tue Nov 12 2024 Łukasz Stolarczuk + + * Version 0.9.1 + + This patch release contains only 3 small fixes in build system of UMF. + Thu Sep 12 2024 Łukasz Stolarczuk * Version 0.9.0 diff --git a/scripts/docs_config/conf.py b/scripts/docs_config/conf.py index b93d7d977f..13a9759836 100644 --- a/scripts/docs_config/conf.py +++ b/scripts/docs_config/conf.py @@ -22,7 +22,7 @@ author = "Intel" # The full version, including alpha/beta/rc tags -release = "0.9.0" +release = "0.9.1" # -- General configuration --------------------------------------------------- From 0d118afc5be9393ecadf42f68c2720153a5bf623 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 29 Nov 2024 11:26:19 +0100 Subject: [PATCH 06/82] Clean up target_*() calls in test/CMakeLists.txt Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 64 ++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bf9884dc98..71a9a46e2a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -40,12 +40,26 @@ function(build_umf_test) set(LIB_DIRS ${LIB_DIRS} ${LIBHWLOC_LIBRARY_DIRS}) - if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) + if(UMF_BUILD_CUDA_PROVIDER) + set(INC_DIRS ${INC_DIRS} ${CUDA_INCLUDE_DIRS}) + set(LIB_DIRS ${LIB_DIRS} ${CUDA_LIBRARY_DIRS}) + endif() + + if(UMF_BUILD_LEVEL_ZERO_PROVIDER) + set(INC_DIRS ${INC_DIRS} ${LEVEL_ZERO_INCLUDE_DIRS}) + endif() + + if(UMF_POOL_JEMALLOC_ENABLED) set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS}) + set(CPL_DEFS ${CPL_DEFS} UMF_POOL_JEMALLOC_ENABLED=1) endif() - if(UMF_BUILD_CUDA_PROVIDER) - set(LIB_DIRS ${LIB_DIRS} ${CUDA_LIBRARY_DIRS}) + if(UMF_POOL_SCALABLE_ENABLED) + set(CPL_DEFS ${CPL_DEFS} UMF_POOL_SCALABLE_ENABLED=1) + endif() + + if(UMF_BUILD_LIBUMF_POOL_DISJOINT) + set(CPL_DEFS ${CPL_DEFS} UMF_POOL_DISJOINT_ENABLED=1) endif() set(TEST_LIBS @@ -60,15 +74,7 @@ function(build_umf_test) SRCS ${ARG_SRCS} LIBS ${TEST_LIBS}) - if(UMF_POOL_JEMALLOC_ENABLED) - target_compile_definitions(${TEST_TARGET_NAME} - PRIVATE UMF_POOL_JEMALLOC_ENABLED=1) - endif() - - if(UMF_POOL_SCALABLE_ENABLED) - target_compile_definitions(${TEST_TARGET_NAME} - PRIVATE UMF_POOL_SCALABLE_ENABLED=1) - endif() + target_compile_definitions(${TEST_TARGET_NAME} PRIVATE ${CPL_DEFS}) if(NOT MSVC) # Suppress 'cast discards const qualifier' warnings. Parametrized GTEST @@ -80,6 +86,7 @@ function(build_umf_test) target_compile_options(${TEST_TARGET_NAME} PRIVATE -Werror) endif() endif() + target_link_directories(${TEST_TARGET_NAME} PRIVATE ${LIB_DIRS}) target_include_directories( @@ -89,7 +96,8 @@ function(build_umf_test) ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc ${UMF_CMAKE_SOURCE_DIR}/src/utils ${UMF_TEST_DIR}/common - ${UMF_TEST_DIR}) + ${UMF_TEST_DIR} + ${INC_DIRS}) endfunction() function(add_umf_test) @@ -157,6 +165,10 @@ if(UMF_POOL_JEMALLOC_ENABLED) set(LIB_JEMALLOC_POOL jemalloc_pool) endif() +if(UMF_BUILD_LIBUMF_POOL_DISJOINT) + set(LIB_DISJOINT_POOL disjoint_pool) +endif() + if(UMF_BUILD_SHARED_LIBRARY) # if build as shared library, ba symbols won't be visible in tests set(BA_SOURCES_FOR_TEST ${BA_SOURCES}) @@ -245,13 +257,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_os_memory SRCS provider_os_memory.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) - if(UMF_BUILD_LIBUMF_POOL_DISJOINT) - target_compile_definitions(umf_test-provider_os_memory - PRIVATE UMF_POOL_DISJOINT_ENABLED=1) - target_link_libraries(umf_test-provider_os_memory PRIVATE disjoint_pool) - endif() - + LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL} ${LIB_DISJOINT_POOL}) add_umf_test( NAME provider_os_memory_multiple_numa_nodes SRCS provider_os_memory_multiple_numa_nodes.cpp @@ -365,8 +371,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) SRCS providers/provider_level_zero.cpp providers/level_zero_helpers.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST} ze_loader) - target_include_directories(umf_test-provider_level_zero - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) add_umf_test( NAME provider_level_zero_dlopen @@ -375,8 +379,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST}) target_compile_definitions(umf_test-provider_level_zero_dlopen PUBLIC USE_DLOPEN=1) - target_include_directories(umf_test-provider_level_zero_dlopen - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) @@ -396,10 +398,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) SRCS providers/provider_cuda.cpp providers/cuda_helpers.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST} cuda) - target_include_directories(umf_test-provider_cuda - PRIVATE ${CUDA_INCLUDE_DIRS}) - target_link_directories(umf_test-provider_cuda PRIVATE - ${CUDA_LIBRARY_DIRS}) add_umf_test( NAME provider_cuda_dlopen @@ -408,8 +406,6 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST}) target_compile_definitions(umf_test-provider_cuda_dlopen PUBLIC USE_DLOPEN=1) - target_include_directories(umf_test-provider_cuda_dlopen - PRIVATE ${CUDA_INCLUDE_DIRS}) else() message( STATUS @@ -601,10 +597,6 @@ if(LINUX) ze_loader disjoint_pool ${UMF_UTILS_FOR_TEST}) - target_include_directories(umf_test-ipc_level_zero_prov_producer - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) - target_include_directories(umf_test-ipc_level_zero_prov_consumer - PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) add_umf_ipc_test(TEST ipc_level_zero_prov SRC_DIR providers) endif() @@ -635,10 +627,6 @@ if(LINUX) cuda disjoint_pool ${UMF_UTILS_FOR_TEST}) - target_include_directories(umf_test-ipc_cuda_prov_producer - PRIVATE ${CUDA_INCLUDE_DIRS}) - target_include_directories(umf_test-ipc_cuda_prov_consumer - PRIVATE ${CUDA_INCLUDE_DIRS}) add_umf_ipc_test(TEST ipc_cuda_prov SRC_DIR providers) endif() else() From 2d28e062793c681f5523000923eb0a1bd91db4ce Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 25 Nov 2024 11:21:10 +0100 Subject: [PATCH 07/82] Add the coarse library Add the coarse library that will replace the coarse provider. Signed-off-by: Lukasz Dorau --- cmake/helpers.cmake | 3 +- src/CMakeLists.txt | 11 +- src/coarse/CMakeLists.txt | 26 + src/coarse/coarse.c | 1351 +++++++++++++++++++++++++++++++++++++ src/coarse/coarse.h | 112 +++ 5 files changed, 1498 insertions(+), 5 deletions(-) create mode 100644 src/coarse/CMakeLists.txt create mode 100644 src/coarse/coarse.c create mode 100644 src/coarse/coarse.h diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 2544a15186..0a165bc3af 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -387,7 +387,8 @@ function(add_umf_library) ${ARG_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src/utils - ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc) + ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc + ${UMF_CMAKE_SOURCE_DIR}/src/coarse) add_umf_target_compile_options(${ARG_NAME}) add_umf_target_link_options(${ARG_NAME}) endfunction() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4736ed0f9..ffd928f7cc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,15 +16,16 @@ set(UMF_CUDA_INCLUDE_DIR # TODO: Cleanup the compile definitions across all the CMake files set(UMF_COMMON_COMPILE_DEFINITIONS UMF_VERSION=${UMF_VERSION}) -add_subdirectory(utils) - -set(UMF_LIBS $) - set(BA_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc.c ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_linear.c ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_global.c) +add_subdirectory(utils) +add_subdirectory(coarse) + +set(UMF_LIBS $ $) + if(LINUX) set(BA_SOURCES ${BA_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_linux.c) @@ -145,6 +146,8 @@ else() LIBS ${UMF_LIBS}) endif() +add_dependencies(umf coarse) + if(UMF_LINK_HWLOC_STATICALLY) add_dependencies(umf ${UMF_HWLOC_NAME}) endif() diff --git a/src/coarse/CMakeLists.txt b/src/coarse/CMakeLists.txt new file mode 100644 index 0000000000..8806b6b557 --- /dev/null +++ b/src/coarse/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) + +set(COARSE_SOURCES coarse.c ../ravl/ravl.c) + +if(UMF_BUILD_SHARED_LIBRARY) + set(COARSE_EXTRA_SRCS ${BA_SOURCES}) + set(COARSE_EXTRA_LIBS $) +endif() + +add_umf_library( + NAME coarse + TYPE STATIC + SRCS ${COARSE_SOURCES} ${COARSE_EXTRA_SRCS} + LIBS ${COARSE_EXTRA_LIBS}) + +target_include_directories( + coarse + PRIVATE $ + $ + $) + +add_library(${PROJECT_NAME}::coarse ALIAS coarse) diff --git a/src/coarse/coarse.c b/src/coarse/coarse.c new file mode 100644 index 0000000000..7294801544 --- /dev/null +++ b/src/coarse/coarse.c @@ -0,0 +1,1351 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include +#include +#include + +#include + +#include "base_alloc_global.h" +#include "coarse.h" +#include "libumf.h" +#include "ravl.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" + +#ifdef _WIN32 +UTIL_ONCE_FLAG Log_initialized = UTIL_ONCE_FLAG_INIT; +#else +void __attribute__((constructor)) coarse_init(void) { utils_log_init(); } +void __attribute__((destructor)) coarse_destroy(void) {} +#endif /* _WIN32 */ + +typedef struct coarse_t { + // handle of the memory provider + void *provider; + + // coarse callbacks + coarse_callbacks_t cb; + + // memory allocation strategy + coarse_strategy_t allocation_strategy; + + // page size of the memory provider + size_t page_size; + + // all_blocks - tree of all blocks - sorted by an address of data + struct ravl *all_blocks; + + // free_blocks - tree of free blocks - sorted by a size of data, + // each node contains a pointer (ravl_free_blocks_head_t) + // to the head of the list of free blocks of the same size + struct ravl *free_blocks; + + struct utils_mutex_t lock; + + // statistics + size_t used_size; + size_t alloc_size; +} coarse_t; + +typedef struct ravl_node ravl_node_t; + +typedef enum check_free_blocks_t { + CHECK_ONLY_THE_FIRST_BLOCK = 0, + CHECK_ALL_BLOCKS_OF_SIZE, +} check_free_blocks_t; + +typedef struct block_t { + size_t size; + unsigned char *data; + bool used; + + // Node in the list of free blocks of the same size pointing to this block. + // The list is located in the (coarse->free_blocks) RAVL tree. + struct ravl_free_blocks_elem_t *free_list_ptr; +} block_t; + +// A general node in a RAVL tree. +// 1) coarse->all_blocks RAVL tree (tree of all blocks - sorted by an address of data): +// key - pointer (block_t->data) to the beginning of the block data +// value - pointer (block_t) to the block of the allocation +// 2) coarse->free_blocks RAVL tree (tree of free blocks - sorted by a size of data): +// key - size of the allocation (block_t->size) +// value - pointer (ravl_free_blocks_head_t) to the head of the list of free blocks of the same size +typedef struct ravl_data_t { + uintptr_t key; + void *value; +} ravl_data_t; + +// The head of the list of free blocks of the same size. +typedef struct ravl_free_blocks_head_t { + struct ravl_free_blocks_elem_t *head; +} ravl_free_blocks_head_t; + +// The node of the list of free blocks of the same size +typedef struct ravl_free_blocks_elem_t { + struct block_t *block; + struct ravl_free_blocks_elem_t *next; + struct ravl_free_blocks_elem_t *prev; +} ravl_free_blocks_elem_t; + +// The compare function of a RAVL tree +static int coarse_ravl_comp(const void *lhs, const void *rhs) { + const ravl_data_t *lhs_ravl = (const ravl_data_t *)lhs; + const ravl_data_t *rhs_ravl = (const ravl_data_t *)rhs; + + if (lhs_ravl->key < rhs_ravl->key) { + return -1; + } + + if (lhs_ravl->key > rhs_ravl->key) { + return 1; + } + + // lhs_ravl->key == rhs_ravl->key + return 0; +} + +static inline block_t *get_node_block(ravl_node_t *node) { + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + assert(node_data->value); + return node_data->value; +} + +static inline ravl_node_t *get_node_prev(ravl_node_t *node) { + return ravl_node_predecessor(node); +} + +static inline ravl_node_t *get_node_next(ravl_node_t *node) { + return ravl_node_successor(node); +} + +#ifndef NDEBUG +static block_t *get_block_prev(ravl_node_t *node) { + ravl_node_t *ravl_prev = ravl_node_predecessor(node); + if (!ravl_prev) { + return NULL; + } + + return get_node_block(ravl_prev); +} + +static block_t *get_block_next(ravl_node_t *node) { + ravl_node_t *ravl_next = ravl_node_successor(node); + if (!ravl_next) { + return NULL; + } + + return get_node_block(ravl_next); +} +#endif /* NDEBUG */ + +// The functions "coarse_ravl_*" handles the coarse->all_blocks list of blocks +// sorted by a pointer (block_t->data) to the beginning of the block data. +// +// coarse_ravl_add_new - allocate and add a new block to the tree +// and link this block to the next and the previous one. +static block_t *coarse_ravl_add_new(struct ravl *rtree, unsigned char *data, + size_t size, ravl_node_t **node) { + assert(rtree); + assert(data); + assert(size); + + // TODO add valgrind annotations + block_t *block = umf_ba_global_alloc(sizeof(*block)); + if (block == NULL) { + return NULL; + } + + block->data = data; + block->size = size; + block->free_list_ptr = NULL; + + ravl_data_t rdata = {(uintptr_t)block->data, block}; + assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); + int ret = ravl_emplace_copy(rtree, &rdata); + if (ret) { + umf_ba_global_free(block); + return NULL; + } + + ravl_node_t *new_node = ravl_find(rtree, &rdata, RAVL_PREDICATE_EQUAL); + assert(NULL != new_node); + + if (node) { + *node = new_node; + } + + return block; +} + +// coarse_ravl_find_node - find the node in the tree +static ravl_node_t *coarse_ravl_find_node(struct ravl *rtree, void *ptr) { + ravl_data_t data = {(uintptr_t)ptr, NULL}; + return ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); +} + +// coarse_ravl_rm - remove the block from the tree +static block_t *coarse_ravl_rm(struct ravl *rtree, void *ptr) { + ravl_data_t data = {(uintptr_t)ptr, NULL}; + ravl_node_t *node; + node = ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); + if (node) { + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + block_t *block = node_data->value; + assert(block); + ravl_remove(rtree, node); + assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); + return block; + } + return NULL; +} + +// The functions "node_list_*" handle lists of free blocks of the same size. +// The heads (ravl_free_blocks_head_t) of those lists are stored in nodes of +// the coarse->free_blocks RAVL tree. +// +// node_list_add - add a free block to the list of free blocks of the same size +static ravl_free_blocks_elem_t * +node_list_add(ravl_free_blocks_head_t *head_node, struct block_t *block) { + assert(head_node); + assert(block); + + ravl_free_blocks_elem_t *node = umf_ba_global_alloc(sizeof(*node)); + if (node == NULL) { + return NULL; + } + + if (head_node->head) { + head_node->head->prev = node; + } + + node->block = block; + node->next = head_node->head; + node->prev = NULL; + head_node->head = node; + + return node; +} + +// node_list_rm - remove the given free block from the list of free blocks of the same size +static block_t *node_list_rm(ravl_free_blocks_head_t *head_node, + ravl_free_blocks_elem_t *node) { + assert(head_node); + assert(node); + assert(head_node->head); + + if (node == head_node->head) { + assert(node->prev == NULL); + head_node->head = node->next; + } + + ravl_free_blocks_elem_t *node_next = node->next; + ravl_free_blocks_elem_t *node_prev = node->prev; + if (node_next) { + node_next->prev = node_prev; + } + + if (node_prev) { + node_prev->next = node_next; + } + + struct block_t *block = node->block; + block->free_list_ptr = NULL; + umf_ba_global_free(node); + + return block; +} + +// node_list_rm_first - remove the first free block from the list of free blocks of the same size only if it can be properly aligned +static block_t *node_list_rm_first(ravl_free_blocks_head_t *head_node, + size_t alignment) { + assert(head_node); + assert(head_node->head); + + ravl_free_blocks_elem_t *node = head_node->head; + assert(node->prev == NULL); + struct block_t *block = node->block; + + if (IS_NOT_ALIGNED(block->size, alignment)) { + return NULL; + } + + if (node->next) { + node->next->prev = NULL; + } + + head_node->head = node->next; + block->free_list_ptr = NULL; + umf_ba_global_free(node); + + return block; +} + +// node_list_rm_with_alignment - remove the first free block with the correct alignment from the list of free blocks of the same size +static block_t *node_list_rm_with_alignment(ravl_free_blocks_head_t *head_node, + size_t alignment) { + assert(head_node); + assert(head_node->head); + + assert(((ravl_free_blocks_elem_t *)head_node->head)->prev == NULL); + + ravl_free_blocks_elem_t *node; + for (node = head_node->head; node != NULL; node = node->next) { + if (IS_ALIGNED(node->block->size, alignment)) { + return node_list_rm(head_node, node); + } + } + + return NULL; +} + +// The functions "free_blocks_*" handle the coarse->free_blocks RAVL tree +// sorted by a size of the allocation (block_t->size). +// This is a tree of heads (ravl_free_blocks_head_t) of lists of free blocks of the same size. +// +// free_blocks_add - add a free block to the list of free blocks of the same size +static int free_blocks_add(struct ravl *free_blocks, block_t *block) { + ravl_free_blocks_head_t *head_node = NULL; + int rv; + + ravl_data_t head_node_data = {(uintptr_t)block->size, NULL}; + ravl_node_t *node; + node = ravl_find(free_blocks, &head_node_data, RAVL_PREDICATE_EQUAL); + if (node) { + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + head_node = node_data->value; + assert(head_node); + } else { // no head_node + head_node = umf_ba_global_alloc(sizeof(*head_node)); + if (!head_node) { + return -1; + } + + head_node->head = NULL; + + ravl_data_t data = {(uintptr_t)block->size, head_node}; + rv = ravl_emplace_copy(free_blocks, &data); + if (rv) { + umf_ba_global_free(head_node); + return -1; + } + } + + block->free_list_ptr = node_list_add(head_node, block); + if (!block->free_list_ptr) { + return -1; // out of memory + } + + assert(block->free_list_ptr->block->size == block->size); + + return 0; +} + +// free_blocks_rm_ge - remove the first free block of a size greater or equal to the given size only if it can be properly aligned +// If it was the last block, the head node is freed and removed from the tree. +// It is used during memory allocation (looking for a free block). +static block_t *free_blocks_rm_ge(struct ravl *free_blocks, size_t size, + size_t alignment, + check_free_blocks_t check_blocks) { + ravl_data_t data = {(uintptr_t)size, NULL}; + ravl_node_t *node; + node = ravl_find(free_blocks, &data, RAVL_PREDICATE_GREATER_EQUAL); + if (!node) { + return NULL; + } + + ravl_data_t *node_data = ravl_data(node); + assert(node_data); + assert(node_data->key >= size); + + ravl_free_blocks_head_t *head_node = node_data->value; + assert(head_node); + + block_t *block = NULL; + switch (check_blocks) { + case CHECK_ONLY_THE_FIRST_BLOCK: + block = node_list_rm_first(head_node, alignment); + break; + case CHECK_ALL_BLOCKS_OF_SIZE: + block = node_list_rm_with_alignment(head_node, alignment); + break; + } + + if (head_node->head == NULL) { + umf_ba_global_free(head_node); + ravl_remove(free_blocks, node); + } + + return block; +} + +// free_blocks_rm_node - remove the free block pointed by the given node. +// If it was the last block, the head node is freed and removed from the tree. +// It is used during merging free blocks and destroying the coarse->free_blocks tree. +static block_t *free_blocks_rm_node(struct ravl *free_blocks, + ravl_free_blocks_elem_t *node) { + assert(free_blocks); + assert(node); + size_t size = node->block->size; + ravl_data_t data = {(uintptr_t)size, NULL}; + ravl_node_t *ravl_node; + ravl_node = ravl_find(free_blocks, &data, RAVL_PREDICATE_EQUAL); + assert(ravl_node); + + ravl_data_t *node_data = ravl_data(ravl_node); + assert(node_data); + assert(node_data->key == size); + + ravl_free_blocks_head_t *head_node = node_data->value; + assert(head_node); + + block_t *block = node_list_rm(head_node, node); + + if (head_node->head == NULL) { + umf_ba_global_free(head_node); + ravl_remove(free_blocks, ravl_node); + } + + return block; +} + +// user_block_merge - merge two blocks from one of two lists of user blocks: all_blocks or free_blocks +static umf_result_t user_block_merge(coarse_t *coarse, ravl_node_t *node1, + ravl_node_t *node2, bool used, + ravl_node_t **merged_node) { + assert(node1); + assert(node2); + assert(node1 == get_node_prev(node2)); + assert(node2 == get_node_next(node1)); + assert(merged_node); + + *merged_node = NULL; + + struct ravl *all_blocks = coarse->all_blocks; + struct ravl *free_blocks = coarse->free_blocks; + + block_t *block1 = get_node_block(node1); + block_t *block2 = get_node_block(node2); + assert(block1->data < block2->data); + + bool same_used = ((block1->used == used) && (block2->used == used)); + bool contignous_data = (block1->data + block1->size == block2->data); + + // check if blocks can be merged + if (!same_used || !contignous_data) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // check if blocks can be merged + umf_result_t umf_result = + coarse->cb.merge(coarse->provider, block1->data, block2->data, + block1->size + block2->size); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_merge_cb(lowPtr=%p, highPtr=%p, totalSize=%zu) failed", + (void *)block1->data, (void *)block2->data, + block1->size + block2->size); + return umf_result; + } + + if (block1->free_list_ptr) { + free_blocks_rm_node(free_blocks, block1->free_list_ptr); + block1->free_list_ptr = NULL; + } + + if (block2->free_list_ptr) { + free_blocks_rm_node(free_blocks, block2->free_list_ptr); + block2->free_list_ptr = NULL; + } + + // update the size + block1->size += block2->size; + + block_t *block_rm = coarse_ravl_rm(all_blocks, block2->data); + assert(block_rm == block2); + (void)block_rm; // WA for unused variable error + umf_ba_global_free(block2); + + *merged_node = node1; + + return UMF_RESULT_SUCCESS; +} + +// free_block_merge_with_prev - merge the given free block +// with the previous one if both are unused and have continuous data. +// Remove the merged block from the tree of free blocks. +static ravl_node_t *free_block_merge_with_prev(coarse_t *coarse, + ravl_node_t *node) { + ravl_node_t *node_prev = get_node_prev(node); + if (!node_prev) { + return node; + } + + ravl_node_t *merged_node = NULL; + umf_result_t umf_result = + user_block_merge(coarse, node_prev, node, false, &merged_node); + if (umf_result != UMF_RESULT_SUCCESS) { + return node; + } + + assert(merged_node != NULL); + + return merged_node; +} + +// free_block_merge_with_next - merge the given free block +// with the next one if both are unused and have continuous data. +// Remove the merged block from the tree of free blocks. +static ravl_node_t *free_block_merge_with_next(coarse_t *coarse, + ravl_node_t *node) { + ravl_node_t *node_next = get_node_next(node); + if (!node_next) { + return node; + } + + ravl_node_t *merged_node = NULL; + umf_result_t umf_result = + user_block_merge(coarse, node, node_next, false, &merged_node); + if (umf_result != UMF_RESULT_SUCCESS) { + return node; + } + + assert(merged_node != NULL); + + return merged_node; +} + +#ifndef NDEBUG // begin of DEBUG code + +typedef struct debug_cb_args_t { + coarse_t *provider; + size_t sum_used; + size_t sum_blocks_size; + size_t num_all_blocks; + size_t num_free_blocks; +} debug_cb_args_t; + +static void debug_verify_all_blocks_cb(void *data, void *arg) { + assert(data); + assert(arg); + + ravl_data_t *node_data = data; + block_t *block = node_data->value; + assert(block); + + debug_cb_args_t *cb_args = (debug_cb_args_t *)arg; + coarse_t *provider = cb_args->provider; + + ravl_node_t *node = + ravl_find(provider->all_blocks, data, RAVL_PREDICATE_EQUAL); + assert(node); + + block_t *block_next = get_block_next(node); + block_t *block_prev = get_block_prev(node); + + cb_args->num_all_blocks++; + if (!block->used) { + cb_args->num_free_blocks++; + } + + assert(block->data); + assert(block->size > 0); + + // data addresses in the list are in ascending order + if (block_prev) { + assert(block_prev->data < block->data); + } + + if (block_next) { + assert(block->data < block_next->data); + } + + // two block's data should not overlap + if (block_next) { + assert((block->data + block->size) <= block_next->data); + } + + cb_args->sum_blocks_size += block->size; + if (block->used) { + cb_args->sum_used += block->size; + } +} + +static umf_result_t coarse_get_stats_no_lock(coarse_t *coarse, + coarse_stats_t *stats); + +static bool debug_check(coarse_t *provider) { + assert(provider); + + coarse_stats_t stats = {0}; + coarse_get_stats_no_lock(provider, &stats); + + debug_cb_args_t cb_args = {0}; + cb_args.provider = provider; + + // verify the all_blocks list + ravl_foreach(provider->all_blocks, debug_verify_all_blocks_cb, &cb_args); + + assert(cb_args.num_all_blocks == stats.num_all_blocks); + assert(cb_args.num_free_blocks == stats.num_free_blocks); + assert(cb_args.sum_used == provider->used_size); + assert(cb_args.sum_blocks_size == provider->alloc_size); + assert(provider->alloc_size >= provider->used_size); + + return true; +} +#endif /* NDEBUG */ // end of DEBUG code + +static umf_result_t coarse_add_used_block(coarse_t *coarse, void *addr, + size_t size) { + block_t *new_block = + coarse_ravl_add_new(coarse->all_blocks, addr, size, NULL); + if (new_block == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + new_block->used = true; + coarse->alloc_size += size; + coarse->used_size += size; + + return UMF_RESULT_SUCCESS; +} + +static void coarse_ravl_cb_rm_all_blocks_node(void *data, void *arg) { + assert(data); + assert(arg); + + coarse_t *coarse = (struct coarse_t *)arg; + ravl_data_t *node_data = data; + block_t *block = node_data->value; + assert(block); + + if (block->used) { +#ifndef NDEBUG + LOG_WARN("not freed block (addr: %p, size: %zu)", (void *)block->data, + block->size); +#endif + assert(coarse->used_size >= block->size); + coarse->used_size -= block->size; + } + + if (block->free_list_ptr) { + free_blocks_rm_node(coarse->free_blocks, block->free_list_ptr); + } + + if (coarse->cb.free) { + coarse->cb.free(coarse->provider, block->data, block->size); + } + + assert(coarse->alloc_size >= block->size); + coarse->alloc_size -= block->size; + + umf_ba_global_free(block); +} + +static umf_result_t can_provider_split(coarse_t *coarse, void *ptr, + size_t totalSize, size_t firstSize) { + // check if the block can be split + umf_result_t umf_result = + coarse->cb.split(coarse->provider, ptr, totalSize, firstSize); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR( + "coarse_split_cb->(ptr=%p, totalSize = %zu = (%zu + %zu)) failed", + ptr, totalSize, firstSize, totalSize - firstSize); + } + + return umf_result; +} + +static umf_result_t create_aligned_block(coarse_t *coarse, size_t orig_size, + size_t alignment, block_t **current) { + (void)orig_size; // unused in the Release version + int rv; + + block_t *curr = *current; + + // In case of non-zero alignment create an aligned block what would be further used. + uintptr_t orig_data = (uintptr_t)curr->data; + uintptr_t aligned_data = ALIGN_UP(orig_data, alignment); + size_t padding = aligned_data - orig_data; + if (alignment > 0 && padding > 0) { + // check if block can be split by the upstream provider + umf_result_t umf_result = + can_provider_split(coarse, curr->data, curr->size, padding); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + block_t *aligned_block = + coarse_ravl_add_new(coarse->all_blocks, curr->data + padding, + curr->size - padding, NULL); + if (aligned_block == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + curr->used = false; + curr->size = padding; + + rv = free_blocks_add(coarse->free_blocks, curr); + if (rv) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + // use aligned block + *current = aligned_block; + assert((*current)->size >= orig_size); + } + + return UMF_RESULT_SUCCESS; +} + +// Split the current block and put the new block after the one that we use. +static umf_result_t split_current_block(coarse_t *coarse, block_t *curr, + size_t size) { + + // check if block can be split by the upstream provider + umf_result_t umf_result = + can_provider_split(coarse, curr->data, curr->size, size); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + ravl_node_t *new_node = NULL; + + block_t *new_block = coarse_ravl_add_new( + coarse->all_blocks, curr->data + size, curr->size - size, &new_node); + if (new_block == NULL) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + new_block->used = false; + + int rv = free_blocks_add(coarse->free_blocks, get_node_block(new_node)); + if (rv) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + return UMF_RESULT_SUCCESS; +} + +static block_t *find_free_block(struct ravl *free_blocks, size_t size, + size_t alignment, + coarse_strategy_t allocation_strategy) { + block_t *block; + + switch (allocation_strategy) { + case UMF_COARSE_MEMORY_STRATEGY_FASTEST: + // Always allocate a free block of the (size + alignment) size + // and later cut out the properly aligned part leaving two remaining parts. + return free_blocks_rm_ge(free_blocks, size + alignment, 0, + CHECK_ONLY_THE_FIRST_BLOCK); + + case UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE: + // First check if the first free block of the 'size' size has the correct alignment. + block = free_blocks_rm_ge(free_blocks, size, alignment, + CHECK_ONLY_THE_FIRST_BLOCK); + if (block) { + return block; + } + + // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + return free_blocks_rm_ge(free_blocks, size + alignment, 0, + CHECK_ONLY_THE_FIRST_BLOCK); + + case UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE: + // First look through all free blocks of the 'size' size + // and choose the first one with the correct alignment. + block = free_blocks_rm_ge(free_blocks, size, alignment, + CHECK_ALL_BLOCKS_OF_SIZE); + if (block) { + return block; + } + + // If none of them had the correct alignment, + // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + return free_blocks_rm_ge(free_blocks, size + alignment, 0, + CHECK_ONLY_THE_FIRST_BLOCK); + } + + return NULL; +} + +static int free_blocks_re_add(coarse_t *coarse, block_t *block) { + assert(coarse); + + ravl_node_t *node = coarse_ravl_find_node(coarse->all_blocks, block->data); + assert(node); + + // merge with prev and/or next block if they are unused and have continuous data + node = free_block_merge_with_prev(coarse, node); + node = free_block_merge_with_next(coarse, node); + + return free_blocks_add(coarse->free_blocks, get_node_block(node)); +} + +static void ravl_cb_count(void *data, void *arg) { + assert(arg); + (void)data; // unused + + size_t *num_all_blocks = arg; + (*num_all_blocks)++; +} + +static void ravl_cb_count_free(void *data, void *arg) { + assert(data); + assert(arg); + + ravl_data_t *node_data = data; + assert(node_data); + ravl_free_blocks_head_t *head_node = node_data->value; + assert(head_node); + struct ravl_free_blocks_elem_t *free_block = head_node->head; + assert(free_block); + + size_t *num_all_blocks = arg; + while (free_block) { + (*num_all_blocks)++; + free_block = free_block->next; + } +} + +static umf_result_t coarse_get_stats_no_lock(coarse_t *coarse, + coarse_stats_t *stats) { + assert(coarse); + + size_t num_all_blocks = 0; + ravl_foreach(coarse->all_blocks, ravl_cb_count, &num_all_blocks); + + size_t num_free_blocks = 0; + ravl_foreach(coarse->free_blocks, ravl_cb_count_free, &num_free_blocks); + + stats->alloc_size = coarse->alloc_size; + stats->used_size = coarse->used_size; + stats->num_all_blocks = num_all_blocks; + stats->num_free_blocks = num_free_blocks; + + return UMF_RESULT_SUCCESS; +} + +// PUBLIC API + +umf_result_t coarse_new(coarse_params_t *coarse_params, coarse_t **pcoarse) { +#ifdef _WIN32 + utils_init_once(&Log_initialized, utils_log_init); +#endif /* _WIN32 */ + + if (coarse_params == NULL || pcoarse == NULL) { + LOG_ERR("coarse parameters or handle is missing"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->provider) { + LOG_ERR("memory provider is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->page_size) { + LOG_ERR("page size of the memory provider is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->cb.split) { + LOG_ERR("coarse split callback is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse_params->cb.merge) { + LOG_ERR("coarse merge callback is not set"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // alloc() and free() callbacks are optional + + coarse_t *coarse = umf_ba_global_alloc(sizeof(*coarse)); + if (!coarse) { + LOG_ERR("out of the host memory"); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + memset(coarse, 0, sizeof(*coarse)); + + coarse->provider = coarse_params->provider; + coarse->page_size = coarse_params->page_size; + coarse->cb = coarse_params->cb; + coarse->allocation_strategy = coarse_params->allocation_strategy; + + umf_result_t umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + + coarse->free_blocks = ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); + if (coarse->free_blocks == NULL) { + LOG_ERR("out of the host memory"); + goto err_free_coarse; + } + + coarse->all_blocks = ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); + if (coarse->all_blocks == NULL) { + LOG_ERR("out of the host memory"); + goto err_delete_ravl_free_blocks; + } + + coarse->alloc_size = 0; + coarse->used_size = 0; + + umf_result = UMF_RESULT_ERROR_UNKNOWN; + + if (utils_mutex_init(&coarse->lock) == NULL) { + LOG_ERR("lock initialization failed"); + goto err_delete_ravl_all_blocks; + } + + assert(coarse->used_size == 0); + assert(coarse->alloc_size == 0); + assert(debug_check(coarse)); + + *pcoarse = coarse; + + return UMF_RESULT_SUCCESS; + +err_delete_ravl_all_blocks: + ravl_delete(coarse->all_blocks); +err_delete_ravl_free_blocks: + ravl_delete(coarse->free_blocks); +err_free_coarse: + umf_ba_global_free(coarse); + return umf_result; +} + +void coarse_delete(coarse_t *coarse) { + if (coarse == NULL) { + LOG_ERR("coarse handle is missing"); + return; + } + + utils_mutex_destroy_not_free(&coarse->lock); + + ravl_foreach(coarse->all_blocks, coarse_ravl_cb_rm_all_blocks_node, coarse); + assert(coarse->used_size == 0); + assert(coarse->alloc_size == 0); + + ravl_delete(coarse->all_blocks); + ravl_delete(coarse->free_blocks); + + umf_ba_global_free(coarse); +} + +umf_result_t coarse_add_memory_from_provider(coarse_t *coarse, size_t size) { + umf_result_t umf_result; + void *ptr = NULL; + + if (coarse == NULL || size == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (!coarse->cb.alloc) { + LOG_ERR("error: alloc callback is not set"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + umf_result = coarse_alloc(coarse, size, coarse->page_size, &ptr); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + assert(ptr); + + return coarse_free(coarse, ptr, size); +} + +umf_result_t coarse_add_memory_fixed(coarse_t *coarse, void *addr, + size_t size) { + umf_result_t umf_result; + + if (coarse == NULL || addr == NULL || size == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (coarse->cb.alloc || coarse->cb.free) { + LOG_ERR("error: alloc or free callback is set"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + umf_result = coarse_add_used_block(coarse, addr, size); + + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + umf_result = coarse_free(coarse, addr, size); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + LOG_DEBUG("coarse_ALLOC (add_memory_block) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t coarse_alloc(coarse_t *coarse, size_t size, size_t alignment, + void **resultPtr) { + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + + if (coarse == NULL || resultPtr == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // alignment must be a power of two and a multiple or a divider of the page size + if (alignment && + ((alignment & (alignment - 1)) || ((alignment % coarse->page_size) && + (coarse->page_size % alignment)))) { + LOG_ERR("wrong alignment: %zu (not a power of 2 or a multiple or a " + "divider of the page size (%zu))", + alignment, coarse->page_size); + return UMF_RESULT_ERROR_INVALID_ALIGNMENT; + } + + if (IS_NOT_ALIGNED(alignment, coarse->page_size)) { + alignment = ALIGN_UP(alignment, coarse->page_size); + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + // Find a block with greater or equal size using the given memory allocation strategy + block_t *curr = find_free_block(coarse->free_blocks, size, alignment, + coarse->allocation_strategy); + + // If the block that we want to reuse has a greater size, split it. + // Try to merge the split part with the successor if it is not used. + enum { ACTION_NONE = 0, ACTION_USE, ACTION_SPLIT } action = ACTION_NONE; + + if (curr && curr->size > size) { + action = ACTION_SPLIT; + } else if (curr && curr->size == size) { + action = ACTION_USE; + } + + if (action) { // ACTION_SPLIT or ACTION_USE + assert(curr->used == false); + + // In case of non-zero alignment create an aligned block what would be further used. + if (alignment > 0) { + umf_result = create_aligned_block(coarse, size, alignment, &curr); + if (umf_result != UMF_RESULT_SUCCESS) { + (void)free_blocks_re_add(coarse, curr); + goto err_unlock; + } + } + + if (action == ACTION_SPLIT) { + // Split the current block and put the new block after the one that we use. + umf_result = split_current_block(coarse, curr, size); + if (umf_result != UMF_RESULT_SUCCESS) { + (void)free_blocks_re_add(coarse, curr); + goto err_unlock; + } + + curr->size = size; + + LOG_DEBUG("coarse_ALLOC (split_block) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + + } else { // action == ACTION_USE + LOG_DEBUG("coarse_ALLOC (same_block) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + } + + curr->used = true; + *resultPtr = curr->data; + coarse->used_size += size; + + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return UMF_RESULT_SUCCESS; + } + + // no suitable block found - try to get more memory from the upstream provider + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + + *resultPtr = NULL; + + if (!coarse->cb.alloc) { + LOG_ERR("out of memory"); + goto err_unlock; + } + + umf_result = coarse->cb.alloc(coarse->provider, size, alignment, resultPtr); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_alloc_cb() failed: out of memory"); + goto err_unlock; + } + + ASSERT_IS_ALIGNED(((uintptr_t)(*resultPtr)), alignment); + + umf_result = coarse_add_used_block(coarse, *resultPtr, size); + if (umf_result != UMF_RESULT_SUCCESS) { + if (coarse->cb.free) { + coarse->cb.free(coarse->provider, *resultPtr, size); + } + goto err_unlock; + } + + LOG_DEBUG("coarse_ALLOC (memory_provider) %zu used %zu alloc %zu", size, + coarse->used_size, coarse->alloc_size); + + umf_result = UMF_RESULT_SUCCESS; + +err_unlock: + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return umf_result; +} + +umf_result_t coarse_free(coarse_t *coarse, void *ptr, size_t bytes) { + if (coarse == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ptr == NULL) { + return UMF_RESULT_SUCCESS; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + ravl_node_t *node = coarse_ravl_find_node(coarse->all_blocks, ptr); + if (node == NULL) { + // the block was not found + LOG_ERR("memory block not found (ptr = %p, size = %zu)", ptr, bytes); + utils_mutex_unlock(&coarse->lock); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + block_t *block = get_node_block(node); + assert(block->used); + + if (bytes > 0 && bytes != block->size) { + // wrong size of allocation + LOG_ERR("wrong size of allocation"); + utils_mutex_unlock(&coarse->lock); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + LOG_DEBUG("coarse_FREE (return_block_to_pool) %zu used %zu alloc %zu", + block->size, coarse->used_size - block->size, coarse->alloc_size); + + assert(coarse->used_size >= block->size); + coarse->used_size -= block->size; + + block->used = false; + + // Merge with prev and/or next block if they are unused and have continuous data. + node = free_block_merge_with_prev(coarse, node); + node = free_block_merge_with_next(coarse, node); + + int rv = free_blocks_add(coarse->free_blocks, get_node_block(node)); + if (rv) { + utils_mutex_unlock(&coarse->lock); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t coarse_merge(coarse_t *coarse, void *lowPtr, void *highPtr, + size_t totalSize) { + umf_result_t umf_result; + + if (coarse == NULL || lowPtr == NULL || highPtr == NULL || totalSize == 0 || + ((uintptr_t)highPtr <= (uintptr_t)lowPtr) || + ((uintptr_t)highPtr - (uintptr_t)lowPtr >= totalSize)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; + + ravl_node_t *low_node = coarse_ravl_find_node(coarse->all_blocks, lowPtr); + if (low_node == NULL) { + LOG_ERR("the lowPtr memory block not found"); + goto err_mutex_unlock; + } + + block_t *low_block = get_node_block(low_node); + if (!low_block->used) { + LOG_ERR("the lowPtr block is not allocated"); + goto err_mutex_unlock; + } + + ravl_node_t *high_node = coarse_ravl_find_node(coarse->all_blocks, highPtr); + if (high_node == NULL) { + LOG_ERR("the highPtr memory block not found"); + goto err_mutex_unlock; + } + + block_t *high_block = get_node_block(high_node); + if (!high_block->used) { + LOG_ERR("the highPtr block is not allocated"); + goto err_mutex_unlock; + } + + if (get_node_next(low_node) != high_node || + ((uintptr_t)highPtr != ((uintptr_t)lowPtr + low_block->size))) { + LOG_ERR("given allocations are not adjacent"); + goto err_mutex_unlock; + } + + assert(get_node_prev(high_node) == low_node); + + if (low_block->size + high_block->size != totalSize) { + LOG_ERR("wrong totalSize: %zu != %zu", totalSize, + low_block->size + high_block->size); + goto err_mutex_unlock; + } + + ravl_node_t *merged_node = NULL; + + umf_result = + user_block_merge(coarse, low_node, high_node, true, &merged_node); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("merging a block failed"); + goto err_mutex_unlock; + } + + assert(merged_node == low_node); + assert(low_block->size == totalSize); + + umf_result = UMF_RESULT_SUCCESS; + +err_mutex_unlock: + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return umf_result; +} + +umf_result_t coarse_split(coarse_t *coarse, void *ptr, size_t totalSize, + size_t firstSize) { + umf_result_t umf_result; + + if (coarse == NULL || ptr == NULL || (firstSize >= totalSize) || + firstSize == 0 || totalSize == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + assert(debug_check(coarse)); + + umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; + + ravl_node_t *node = coarse_ravl_find_node(coarse->all_blocks, ptr); + if (node == NULL) { + LOG_ERR("memory block not found"); + goto err_mutex_unlock; + } + + block_t *block = get_node_block(node); + + if (block->size != totalSize) { + LOG_ERR("wrong totalSize: %zu != %zu", totalSize, block->size); + goto err_mutex_unlock; + } + + if (!block->used) { + LOG_ERR("block is not allocated"); + goto err_mutex_unlock; + } + + // check if block can be split by the memory provider + umf_result = can_provider_split(coarse, ptr, totalSize, firstSize); + if (umf_result != UMF_RESULT_SUCCESS) { + LOG_ERR("memory provider cannot split a memory block"); + goto err_mutex_unlock; + } + + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + + block_t *new_block = + coarse_ravl_add_new(coarse->all_blocks, block->data + firstSize, + block->size - firstSize, NULL); + if (new_block == NULL) { + goto err_mutex_unlock; + } + + block->size = firstSize; + new_block->used = true; + + assert(new_block->size == (totalSize - firstSize)); + + umf_result = UMF_RESULT_SUCCESS; + +err_mutex_unlock: + assert(debug_check(coarse)); + utils_mutex_unlock(&coarse->lock); + + return umf_result; +} + +coarse_stats_t coarse_get_stats(coarse_t *coarse) { + coarse_stats_t stats = {0}; + + if (coarse == NULL) { + return stats; + } + + if (utils_mutex_lock(&coarse->lock) != 0) { + LOG_ERR("locking the lock failed"); + return stats; + } + + coarse_get_stats_no_lock(coarse, &stats); + + utils_mutex_unlock(&coarse->lock); + + return stats; +} diff --git a/src/coarse/coarse.h b/src/coarse/coarse.h new file mode 100644 index 0000000000..cd151ca27a --- /dev/null +++ b/src/coarse/coarse.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_COARSE_H +#define UMF_COARSE_H + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct coarse_t coarse_t; + +// coarse callbacks implement provider-specific actions +typedef struct coarse_callbacks_t { + // alloc() is optional (can be NULL for the fixed-size memory provider) + umf_result_t (*alloc)(void *provider, size_t size, size_t alignment, + void **ptr); + // free() is optional (can be NULL if the provider does not support the free() op) + umf_result_t (*free)(void *provider, void *ptr, size_t size); + umf_result_t (*split)(void *provider, void *ptr, size_t totalSize, + size_t firstSize); + umf_result_t (*merge)(void *provider, void *lowPtr, void *highPtr, + size_t totalSize); +} coarse_callbacks_t; + +// coarse library allocation strategy +typedef enum coarse_strategy_t { + // Always allocate a free block of the (size + alignment) size + // and cut out the properly aligned part leaving two remaining parts. + // It is the fastest strategy but causes memory fragmentation + // when alignment is greater than 0. + // It is the best strategy when alignment always equals 0. + UMF_COARSE_MEMORY_STRATEGY_FASTEST = 0, + + // Check if the first free block of the 'size' size has the correct alignment. + // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, + + // Look through all free blocks of the 'size' size + // and choose the first one with the correct alignment. + // If none of them had the correct alignment, + // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE, +} coarse_strategy_t; + +// coarse library settings structure +typedef struct coarse_params_t { + // handle of the memory provider + void *provider; + + // coarse callbacks + coarse_callbacks_t cb; + + // memory allocation strategy, + // see coarse_strategy_t for details + coarse_strategy_t allocation_strategy; + + // page size of the memory provider + size_t page_size; +} coarse_params_t; + +// coarse library statistics +typedef struct coarse_stats_t { + // total allocation size + size_t alloc_size; + + // size of used memory + size_t used_size; + + // total number of allocated memory blocks + size_t num_all_blocks; + + // number of free memory blocks + size_t num_free_blocks; +} coarse_stats_t; + +umf_result_t coarse_new(coarse_params_t *coarse_params, coarse_t **pcoarse); +void coarse_delete(coarse_t *coarse); + +umf_result_t coarse_alloc(coarse_t *coarse, size_t size, size_t alignment, + void **resultPtr); +umf_result_t coarse_free(coarse_t *coarse, void *ptr, size_t bytes); + +umf_result_t coarse_merge(coarse_t *coarse, void *lowPtr, void *highPtr, + size_t totalSize); +umf_result_t coarse_split(coarse_t *coarse, void *ptr, size_t totalSize, + size_t firstSize); + +// supported only if the alloc callback is set, +// returns UMF_RESULT_ERROR_NOT_SUPPORTED otherwise +umf_result_t coarse_add_memory_from_provider(coarse_t *coarse, size_t size); + +// supported only if the alloc and the free callbacks are NOT set +// returns UMF_RESULT_ERROR_NOT_SUPPORTED otherwise +umf_result_t coarse_add_memory_fixed(coarse_t *coarse, void *addr, size_t size); + +coarse_stats_t coarse_get_stats(coarse_t *coarse); + +#ifdef __cplusplus +} +#endif + +#endif // UMF_COARSE_H From 958b6905d7d5160af9328dc745d480025fb74945 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 2 Dec 2024 08:38:32 +0100 Subject: [PATCH 08/82] Add tests for the coarse library Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 6 + test/coarse_lib.cpp | 1319 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1325 insertions(+) create mode 100644 test/coarse_lib.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c8b854ba5d..ffa4bd20da 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -88,6 +88,7 @@ function(build_umf_test) PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_CMAKE_SOURCE_DIR}/src ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc + ${UMF_CMAKE_SOURCE_DIR}/src/coarse ${UMF_CMAKE_SOURCE_DIR}/src/utils ${UMF_TEST_DIR}/common ${UMF_TEST_DIR}) @@ -196,6 +197,11 @@ add_umf_test( SRCS provider_coarse.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST}) +add_umf_test( + NAME coarse_lib + SRCS coarse_lib.cpp ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST} coarse) + if(UMF_BUILD_LIBUMF_POOL_DISJOINT) add_umf_test( NAME disjointPool diff --git a/test/coarse_lib.cpp b/test/coarse_lib.cpp new file mode 100644 index 0000000000..6a3d9637ec --- /dev/null +++ b/test/coarse_lib.cpp @@ -0,0 +1,1319 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "coarse.h" +#include "provider.hpp" + +using umf_test::KB; +using umf_test::MB; +using umf_test::test; + +#define MOCKED_COARSE ((coarse_t *)0x01) +#define MOCKED_PROVIDER ((umf_memory_provider_handle_t)0x02) +#define INVALID_PTR ((void *)0x03) + +static umf_result_t alloc_cb(void *provider, size_t size, size_t alignment, + void **ptr) { + return umfMemoryProviderAlloc((umf_memory_provider_handle_t)provider, size, + alignment, ptr); +} + +static umf_result_t free_cb(void *provider, void *ptr, size_t size) { + return umfMemoryProviderFree((umf_memory_provider_handle_t)provider, ptr, + size); +} + +static umf_result_t split_cb(void *provider, void *ptr, size_t totalSize, + size_t firstSize) { + if (provider == NULL || ptr == NULL || (firstSize >= totalSize) || + firstSize == 0 || totalSize == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t merge_cb(void *provider, void *lowPtr, void *highPtr, + size_t totalSize) { + if (provider == NULL || lowPtr == NULL || highPtr == NULL || + totalSize == 0 || ((uintptr_t)highPtr <= (uintptr_t)lowPtr) || + ((uintptr_t)highPtr - (uintptr_t)lowPtr >= totalSize)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t alloc_cb_fails(void *provider, size_t size, + size_t alignment, void **ptr) { + (void)provider; //unused + (void)size; //unused + (void)alignment; //unused + (void)ptr; //unused + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; +} + +static umf_result_t free_cb_fails(void *provider, void *ptr, size_t size) { + (void)provider; //unused + (void)ptr; //unused + (void)size; //unused + return UMF_RESULT_ERROR_USER_SPECIFIC; +} + +static umf_result_t split_cb_fails(void *provider, void *ptr, size_t totalSize, + size_t firstSize) { + (void)provider; //unused + (void)ptr; //unused + (void)totalSize; //unused + (void)firstSize; //unused + return UMF_RESULT_ERROR_USER_SPECIFIC; +} + +static umf_result_t merge_cb_fails(void *provider, void *lowPtr, void *highPtr, + size_t totalSize) { + (void)provider; //unused + (void)lowPtr; //unused + (void)highPtr; //unused + (void)totalSize; //unused + return UMF_RESULT_ERROR_USER_SPECIFIC; +} + +static void coarse_params_set_default(coarse_params_t *coarse_params, + umf_memory_provider_handle_t provider, + coarse_strategy_t allocation_strategy) { + memset(coarse_params, 0, sizeof(*coarse_params)); + coarse_params->provider = provider; + coarse_params->allocation_strategy = allocation_strategy; + coarse_params->cb.split = split_cb; + coarse_params->cb.merge = merge_cb; + coarse_params->page_size = utils_get_page_size(); + + if (provider) { + coarse_params->cb.alloc = alloc_cb; + coarse_params->cb.free = free_cb; + } +} + +umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = + umf::providerMakeCOps(); + +struct CoarseWithMemoryStrategyTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + allocation_strategy = this->GetParam(); + coarse_params_set_default(&coarse_params, MOCKED_PROVIDER, + allocation_strategy); + } + + coarse_t *coarse_handle = nullptr; + coarse_params_t coarse_params; + coarse_strategy_t allocation_strategy; + umf_result_t umf_result; +}; + +INSTANTIATE_TEST_SUITE_P( + CoarseWithMemoryStrategyTest, CoarseWithMemoryStrategyTest, + ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, + UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, + UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_provider) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + void *ptr; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_fixed_memory) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_various) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // free NULL + umf_result = coarse_free(ch, nullptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // free invalid pointer + umf_result = coarse_free(ch, INVALID_PTR, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // wrong alignment (3 bytes) + ptr = nullptr; + umf_result = coarse_alloc(ch, 2 * MB, 3, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ALIGNMENT); + ASSERT_EQ(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // not freed allocation + // coarse_delete() prints LOG_WARN() in Debug mode + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_merge) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_split */ + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_split(ch, ptr, 2 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, (ptr + 1 * MB), 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 1 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_merge */ + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_split(ch, ptr, 2 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_free(ch, ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(coarse_handle); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +// negative tests + +// NULL parameters +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_params) { + umf_result = coarse_new(nullptr, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no provider +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_provider) { + coarse_params.provider = NULL; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no page size +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_page_size) { + coarse_params.page_size = 0; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no split callback +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_split_cb) { + coarse_params.cb.split = NULL; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +// no merge callback +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_no_merge_cb) { + coarse_params.cb.merge = NULL; + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_handle, nullptr); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alloc_invalid) { + void *ptr = nullptr; + + umf_result = coarse_alloc(nullptr, MB, 0, nullptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(ptr, nullptr); + + umf_result = coarse_alloc(nullptr, MB, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(ptr, nullptr); + + umf_result = coarse_alloc(MOCKED_COARSE, MB, 0, nullptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(ptr, nullptr); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_free_invalid) { + // coarse handle is NULL + umf_result = coarse_free(nullptr, nullptr, MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_delete_null) { + coarse_delete(nullptr); +} + +TEST_P(CoarseWithMemoryStrategyTest, + coarseTest_add_memory_from_provider_null_0) { + umf_result = coarse_add_memory_from_provider(nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_add_memory_fixed_null_0) { + umf_result = coarse_add_memory_fixed(nullptr, nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_null_stats) { + ASSERT_EQ(coarse_get_stats(nullptr).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(nullptr).used_size, 0); + ASSERT_EQ(coarse_get_stats(nullptr).num_all_blocks, 0); + ASSERT_EQ(coarse_get_stats(nullptr).num_free_blocks, 0); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_merge_negative) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_split */ + + umf_result = coarse_alloc(ch, 6 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + // firstSize >= totalSize + umf_result = coarse_split(ch, ptr, 6 * MB, 6 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // firstSize == 0 + umf_result = coarse_split(ch, ptr, 6 * MB, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // totalSize == 0 + umf_result = coarse_split(ch, ptr, 0, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // wrong totalSize + umf_result = coarse_split(ch, ptr, 5 * MB, 1 * KB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // memory block not found + umf_result = coarse_split(ch, ptr + 1, 6 * MB, 1 * KB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result = coarse_free(ch, ptr, 6 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // split freed block + umf_result = coarse_split(ch, ptr, alloc_size, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_merge */ + + umf_result = coarse_alloc(ch, 6 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + // split (6 * MB) block into (1 * MB) + (5 * MB) + umf_result = coarse_split(ch, ptr, 6 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + // split (5 * MB) block into (2 * MB) + (3 * MB) + umf_result = coarse_split(ch, (ptr + 1 * MB), 5 * MB, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 6 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 4); + + // now we have 3 used blocks: (1 * MB) + (2 * MB) + (3 * MB) + + // highPtr <= lowPtr + umf_result = coarse_merge(ch, (ptr + 1 * MB), ptr, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // highPtr - lowPtr >= totalSize + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // low ptr does not exist + umf_result = coarse_merge(ch, ptr + 1, (ptr + 1 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // high ptr does not exist + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB + 1), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // low_block->size + high_block->size != totalSize + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 5 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // not adjacent blocks + umf_result = coarse_merge(ch, ptr, (ptr + 3 * MB), 4 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // free the 2 MB block in the middle + umf_result = coarse_free(ch, (ptr + 1 * MB), 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 4); + + // now we have 3 blocks: (1 * MB) used + (2 * MB) freed + (3 * MB) used + + // the low ptr block is not allocated + umf_result = coarse_merge(ch, (ptr + 1 * MB), (ptr + 3 * MB), 5 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // the high ptr block is not allocated + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result = coarse_free(ch, ptr, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, (ptr + 3 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(coarse_handle); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_alloc_cb_fails) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.alloc = alloc_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_free_cb_fails) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.free = free_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_cb_fails) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.split = split_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + void *ptr = nullptr; + const size_t alloc_size = 20 * MB; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // coarse_alloc(alloc_size / 2, alignment = 0) + umf_result = coarse_alloc(ch, alloc_size / 2, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // coarse_alloc(alloc_size / 2, alignment = 2 * MB) + umf_result = coarse_alloc(ch, alloc_size / 2, 2 * MB, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // coarse_alloc(alloc_size, alignment = 0) - OK + umf_result = coarse_alloc(ch, alloc_size, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_split(ch, ptr, alloc_size, alloc_size / 2); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + + ASSERT_EQ(coarse_get_stats(ch).used_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + umf_result = coarse_free(ch, ptr, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(coarse_handle); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_merge_cb_fails) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 10 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + coarse_params.cb.merge = merge_cb_fails; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + char *ptr = nullptr; + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + /* test coarse_merge */ + umf_result = coarse_alloc(ch, 3 * MB, 0, (void **)&ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_split(ch, ptr, 3 * MB, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_merge(ch, ptr, (ptr + 1 * MB), 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_USER_SPECIFIC); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, ptr, 3 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(coarse_get_stats(ch).used_size, 3 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, ptr, 1 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_free(ch, (ptr + 1 * MB), 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, buff_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + coarse_delete(coarse_handle); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_alloc_set) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_free_set) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_alloc_free_set) { + // preallocate some memory and initialize the vector with zeros + const size_t buff_size = 20 * MB; + std::vector buffer(buff_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_fixed(ch, buf, buff_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_provider_alloc_not_set) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + coarse_params.cb.alloc = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 20 * MB; + void *ptr; + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_alloc(ch, 2 * MB, 0, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + umf_result = coarse_alloc(ch, 2 * MB, 2 * MB, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY); + ASSERT_EQ(ptr, nullptr); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 0); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t init_buffer_size = 20 * MB; + void *p1, *p2; + + umf_result = coarse_add_memory_from_provider(ch, init_buffer_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // alloc 2x 2MB + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p2, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + ASSERT_NE(p1, p2); + + // swap pointers to get p1 < p2 + if (p1 > p2) { + std::swap(p1, p2); + } + + // free + alloc first block + // the block should be reused + // currently there is no purging, so the alloc size shouldn't change + // there should be no block merging between used and not-used blocks + umf_result = coarse_free(ch, p1, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + + // free all allocs + // overall alloc size shouldn't change + // block p2 should merge with the prev free block p1 + // and the remaining init block + umf_result = coarse_free(ch, p1, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + umf_result = coarse_free(ch, p2, 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // test allocations with alignment + // TODO: what about holes? + umf_result = coarse_alloc(ch, 1 * MB - 4, 128, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ((uintptr_t)p1 & 127, 0); + + umf_result = coarse_alloc(ch, 1 * MB - 4, 128, (void **)&p2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p2, nullptr); + ASSERT_EQ((uintptr_t)p2 & 127, 0); + + umf_result = coarse_free(ch, p1, 1 * MB - 4); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + umf_result = coarse_free(ch, p2, 1 * MB - 4); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + // alloc whole buffer + // after this, there should be one single block + umf_result = coarse_alloc(ch, init_buffer_size, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // free all memory + umf_result = coarse_free(ch, p1, init_buffer_size); + + // alloc 2 MB block - the init block should be split + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p1, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 2); + + // alloc additional 2 MB + // the non-used block should be used + umf_result = coarse_alloc(ch, 2 * MB, 0, (void **)&p2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(p2, nullptr); + ASSERT_EQ(coarse_get_stats(ch).used_size, 4 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 3); + ASSERT_NE(p1, p2); + + // make sure that p1 < p2 + if (p1 > p2) { + std::swap(p1, p2); + } + + // free blocks in order: p2, p1 + // block p1 should merge with the next block p2 + // swap pointers to get p1 < p2 + coarse_free(ch, p2, 2 * MB); + coarse_free(ch, p1, 2 * MB); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // alloc 10x 2 MB - this should occupy all allocated memory + constexpr int allocs_size = 10; + void *allocs[allocs_size] = {0}; + for (int i = 0; i < allocs_size; i++) { + ASSERT_EQ(coarse_get_stats(ch).used_size, i * 2 * MB); + umf_result = coarse_alloc(ch, 2 * MB, 0, &allocs[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(allocs[i], nullptr); + } + ASSERT_EQ(coarse_get_stats(ch).used_size, 20 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + // there should be no block with the free memory + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, allocs_size); + + // free all memory + for (int i = 0; i < allocs_size; i++) { + umf_result = coarse_free(ch, allocs[i], 2 * MB); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_simple1) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t init_buffer_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, init_buffer_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // test 1 + + size_t s1 = 74659 * KB; + size_t s2 = 8206 * KB; + + size_t max_alloc_size = 0; + + const int nreps = 2; + const int nptrs = 6; + + // s1 + for (int j = 0; j < nreps; j++) { + void *t[nptrs] = {0}; + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_alloc(ch, s1, 0, &t[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(t[i], nullptr); + } + + if (max_alloc_size == 0) { + max_alloc_size = coarse_get_stats(ch).alloc_size; + } + + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_free(ch, t[i], s1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + // s2 + for (int j = 0; j < nreps; j++) { + void *t[nptrs] = {0}; + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_alloc(ch, s2, 0, &t[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(t[i], nullptr); + } + + // all s2 should fit into single block leaved after freeing s1 + ASSERT_LE(coarse_get_stats(ch).alloc_size, max_alloc_size); + + for (int i = 0; i < nptrs; i++) { + umf_result = coarse_free(ch, t[i], s2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_simple2) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t init_buffer_size = 20 * MB; + + umf_result = coarse_add_memory_from_provider(ch, init_buffer_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, init_buffer_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + // test + double sizes[] = {2, 4, 0.5, 1, 8, 0.25}; + size_t alignment[] = {0, 4, 0, 16, 32, 128}; + for (int i = 0; i < 6; i++) { + size_t s = (size_t)(sizes[i] * MB); + void *t[8] = {0}; + for (int j = 0; j < 8; j++) { + umf_result = coarse_alloc(ch, s, alignment[i], &t[j]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(t[j], nullptr); + } + + for (int j = 0; j < 8; j++) { + umf_result = coarse_free(ch, t[j], s); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + } + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alignment_provider) { + umf_memory_provider_handle_t malloc_memory_provider; + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, + &malloc_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(malloc_memory_provider, nullptr); + + coarse_params.provider = malloc_memory_provider; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + const size_t alloc_size = 40 * MB; + + umf_result = coarse_add_memory_from_provider(ch, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + const int niter = 10; + const int size = 1 * MB; + void *ptr[niter] = {0}; + + for (int i = 0; i < niter; i++) { + umf_result = coarse_alloc(ch, size, 0, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ptr[i] = nullptr; + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size / 2); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + ASSERT_EQ(ptr[i], nullptr); + umf_result = coarse_alloc(ch, size, 2 * MB, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + + for (int i = 0; i < niter; i++) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); + umfMemoryProviderDestroy(malloc_memory_provider); +} + +TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alignment_fixed_memory) { + // preallocate some memory and initialize the vector with zeros + const size_t alloc_size = 40 * MB; + std::vector buffer(alloc_size, 0); + void *buf = (void *)buffer.data(); + ASSERT_NE(buf, nullptr); + + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; + + umf_result = coarse_new(&coarse_params, &coarse_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(coarse_handle, nullptr); + + coarse_t *ch = coarse_handle; + + umf_result = coarse_add_memory_fixed(ch, buf, alloc_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0 * MB); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + const int niter = 10; + const int size = 1 * MB; + void *ptr[niter] = {0}; + + for (int i = 0; i < niter; i++) { + umf_result = coarse_alloc(ch, size, 0, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ptr[i] = nullptr; + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size / 2); + ASSERT_EQ(coarse_get_stats(ch).alloc_size, alloc_size); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, niter + 1); + + for (int i = 0; i < niter; i += 2) { + ASSERT_EQ(ptr[i], nullptr); + umf_result = coarse_alloc(ch, size, 2 * MB, &ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr[i], nullptr); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, niter * size); + + for (int i = 0; i < niter; i++) { + umf_result = coarse_free(ch, ptr[i], size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + ASSERT_EQ(coarse_get_stats(ch).used_size, 0); + ASSERT_EQ(coarse_get_stats(ch).num_all_blocks, 1); + + coarse_delete(ch); +} From aff5982afcd53455fa881575059b7a4ed9f8bc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 2 Dec 2024 17:06:11 +0100 Subject: [PATCH 09/82] Bump version on main to 0.11.0 --- .github/workflows/reusable_basic.yml | 4 ++-- RELEASE_STEPS.md | 2 +- include/umf/base.h | 2 +- scripts/docs_config/conf.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 1c13a771b1..9b71c7d1b8 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -7,8 +7,8 @@ permissions: contents: read env: - # for installation testing - it should match with version set in CMake - UMF_VERSION: 0.10.0 + # for installation testing - it should match with version set in git + UMF_VERSION: 0.11.0 BUILD_DIR : "${{github.workspace}}/build" INSTL_DIR : "${{github.workspace}}/../install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index e88ca9c2d8..fb46f156b9 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -42,7 +42,7 @@ Do changes for a release: - Update project's version in a few places: - For major and minor releases: `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - `release` variable in `scripts/docs_config/conf.py` (for docs) - - `UMF_VERSION` variable in `.github/workflows/basic.yml` (for installation test) + - `UMF_VERSION` variable in `.github/workflows/reusable_basic.yml` (for installation test) - For major releases update ABI version in `.map` and `.def` files - These files are defined for all public libraries (`libumf` and `proxy_lib`, at the moment) - Commit these changes and tag the release: diff --git a/include/umf/base.h b/include/umf/base.h index 53378195d2..32d84771f3 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -28,7 +28,7 @@ extern "C" { #define UMF_MINOR_VERSION(_ver) (_ver & 0x0000ffff) /// @brief Current version of the UMF headers -#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(0, 10) +#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(0, 11) /// @brief Operation results typedef enum umf_result_t { diff --git a/scripts/docs_config/conf.py b/scripts/docs_config/conf.py index 28c9b5f9f8..577bc0b484 100644 --- a/scripts/docs_config/conf.py +++ b/scripts/docs_config/conf.py @@ -22,7 +22,7 @@ author = "Intel" # The full version, including alpha/beta/rc tags -release = "0.10.0" +release = "0.11.0" # -- General configuration --------------------------------------------------- From dcc1ec1ae4ad932800d160b1ddb1ac61f326f08d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 3 Dec 2024 10:11:51 +0100 Subject: [PATCH 10/82] Add info about the `PTRACE_MODE_ATTACH_REALCREDS` permission Add info about the `PTRACE_MODE_ATTACH_REALCREDS` permission required by the OS and the L0 providers to `README.md`. Signed-off-by: Lukasz Dorau --- README.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6f1233c639..3379132e7f 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,15 @@ OS memory provider supports two types of memory mappings (set by the `visibility IPC API requires the `UMF_MEM_MAP_SHARED` memory `visibility` mode (`UMF_RESULT_ERROR_INVALID_ARGUMENT` is returned otherwise). +IPC API uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain +a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported since Linux 5.6). +Permission to duplicate another process's file descriptor is governed by a ptrace access mode +`PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using +the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: +```sh +$ sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" +``` + There are available two mechanisms for the shared memory mapping: 1) a named shared memory object (used if the `shm_name` parameter is not NULL) or 2) an anonymous file descriptor (used if the `shm_name` parameter is NULL) @@ -162,23 +171,37 @@ An anonymous file descriptor for the shared memory mapping will be created using ##### Requirements -Required packages for tests (Linux-only yet): +IPC API on Linux requires the `PTRACE_MODE_ATTACH_REALCREDS` permission (see `ptrace(2)`) +to duplicate another process's file descriptor (see above). + +Packages required for tests (Linux-only yet): - libnuma-dev #### Level Zero memory provider A memory provider that provides memory from L0 device. +IPC API uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain +a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported since Linux 5.6). +Permission to duplicate another process's file descriptor is governed by a ptrace access mode +`PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using +the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: +```sh +$ sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" +``` + ##### Requirements 1) Linux or Windows OS 2) The `UMF_BUILD_LEVEL_ZERO_PROVIDER` option turned `ON` (by default) +3) IPC API on Linux requires the `PTRACE_MODE_ATTACH_REALCREDS` permission (see `ptrace(2)`) + to duplicate another process's file descriptor (see above). Additionally, required for tests: -3) The `UMF_BUILD_GPU_TESTS` option turned `ON` -4) System with Level Zero compatible GPU -5) Required packages: +4) The `UMF_BUILD_GPU_TESTS` option turned `ON` +5) System with Level Zero compatible GPU +6) Required packages: - liblevel-zero-dev (Linux) or level-zero-sdk (Windows) #### DevDax memory provider (Linux only) From 73d012e4d626c08b739d952710f0287ceb0b62ba Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 21 Nov 2024 08:38:47 +0100 Subject: [PATCH 11/82] Use libcoarse in the file provider Signed-off-by: Lukasz Dorau --- src/provider/provider_file_memory.c | 55 ++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 32383a5ecc..9d332fd467 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -63,6 +63,7 @@ umf_result_t umfFileMemoryProviderParamsSetVisibility( #else // !defined(_WIN32) && !defined(UMF_NO_HWLOC) #include "base_alloc_global.h" +#include "coarse.h" #include "critnib.h" #include "libumf.h" #include "utils_common.h" @@ -101,6 +102,8 @@ typedef struct file_memory_provider_t { // It is needed mainly in the get_ipc_handle and open_ipc_handle hooks // to mmap a specific part of a file. critnib *fd_offset_map; + + coarse_t *coarse; // coarse library handle } file_memory_provider_t; // File Memory Provider settings struct @@ -166,6 +169,14 @@ file_translate_params(umf_file_memory_provider_params_t *in_params, return UMF_RESULT_SUCCESS; } +static umf_result_t file_alloc_cb(void *provider, size_t size, size_t alignment, + void **resultPtr); +static umf_result_t file_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize); +static umf_result_t file_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize); + static umf_result_t file_initialize(void *params, void **provider) { umf_result_t ret; @@ -233,10 +244,27 @@ static umf_result_t file_initialize(void *params, void **provider) { file_provider->page_size = utils_get_page_size(); } + coarse_params_t coarse_params = {0}; + coarse_params.provider = file_provider; + coarse_params.page_size = file_provider->page_size; + coarse_params.cb.alloc = file_alloc_cb; + coarse_params.cb.free = NULL; // not available for the file provider + coarse_params.cb.split = file_allocation_split_cb; + coarse_params.cb.merge = file_allocation_merge_cb; + + coarse_t *coarse = NULL; + ret = coarse_new(&coarse_params, &coarse); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_new() failed"); + goto err_close_fd; + } + + file_provider->coarse = coarse; + if (utils_mutex_init(&file_provider->lock) == NULL) { LOG_ERR("lock init failed"); ret = UMF_RESULT_ERROR_UNKNOWN; - goto err_close_fd; + goto err_coarse_delete; } file_provider->fd_offset_map = critnib_new(); @@ -261,6 +289,8 @@ static umf_result_t file_initialize(void *params, void **provider) { critnib_delete(file_provider->fd_offset_map); err_mutex_destroy_not_free: utils_mutex_destroy_not_free(&file_provider->lock); +err_coarse_delete: + coarse_delete(file_provider->coarse); err_close_fd: utils_close_fd(file_provider->fd); err_free_file_provider: @@ -285,6 +315,7 @@ static void file_finalize(void *provider) { utils_close_fd(file_provider->fd); critnib_delete(file_provider->fd_offset_map); critnib_delete(file_provider->mmaps); + coarse_delete(file_provider->coarse); umf_ba_global_free(file_provider); } @@ -443,6 +474,12 @@ static umf_result_t file_alloc_aligned(file_memory_provider_t *file_provider, static umf_result_t file_alloc(void *provider, size_t size, size_t alignment, void **resultPtr) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_alloc(file_provider->coarse, size, alignment, resultPtr); +} + +static umf_result_t file_alloc_cb(void *provider, size_t size, size_t alignment, + void **resultPtr) { umf_result_t umf_result; int ret; @@ -568,10 +605,15 @@ static const char *file_get_name(void *provider) { return "FILE"; } -// This function is supposed to be thread-safe, so it should NOT be called concurrently -// with file_allocation_merge() with the same pointer. static umf_result_t file_allocation_split(void *provider, void *ptr, size_t totalSize, size_t firstSize) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_split(file_provider->coarse, ptr, totalSize, firstSize); +} + +static umf_result_t file_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { (void)totalSize; file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; @@ -601,9 +643,14 @@ static umf_result_t file_allocation_split(void *provider, void *ptr, return UMF_RESULT_SUCCESS; } -// It should NOT be called concurrently with file_allocation_split() with the same pointer. static umf_result_t file_allocation_merge(void *provider, void *lowPtr, void *highPtr, size_t totalSize) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_merge(file_provider->coarse, lowPtr, highPtr, totalSize); +} + +static umf_result_t file_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { (void)lowPtr; (void)totalSize; From f81c64702db468a9bde7d13d2d2df039fe04e7e4 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 25 Nov 2024 10:05:23 +0100 Subject: [PATCH 12/82] Add free() op to the file provider Signed-off-by: Lukasz Dorau --- src/provider/provider_file_memory.c | 6 ++++++ test/provider_file_memory.cpp | 18 +++++++++--------- test/provider_file_memory_ipc.cpp | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 9d332fd467..558b1062a5 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -815,6 +815,11 @@ static umf_result_t file_close_ipc_handle(void *provider, void *ptr, return UMF_RESULT_SUCCESS; } +static umf_result_t file_free(void *provider, void *ptr, size_t size) { + file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + return coarse_free(file_provider->coarse, ptr, size); +} + static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .version = UMF_VERSION_CURRENT, .initialize = file_initialize, @@ -824,6 +829,7 @@ static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = file_get_recommended_page_size, .get_min_page_size = file_get_min_page_size, .get_name = file_get_name, + .ext.free = file_free, .ext.purge_lazy = file_purge_lazy, .ext.purge_force = file_purge_force, .ext.allocation_merge = file_allocation_merge, diff --git a/test/provider_file_memory.cpp b/test/provider_file_memory.cpp index d3124aa11f..0d54c287cb 100644 --- a/test/provider_file_memory.cpp +++ b/test/provider_file_memory.cpp @@ -98,7 +98,7 @@ static void test_alloc_free_success(umf_memory_provider_handle_t provider, } umf_result = umfMemoryProviderFree(provider, ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } static void verify_last_native_error(umf_memory_provider_handle_t provider, @@ -159,7 +159,7 @@ TEST_F(test, test_if_mapped_with_MAP_SYNC) { bool flag_found = is_mapped_with_MAP_SYNC(path, buf, size); umf_result = umfMemoryProviderFree(hProvider, buf, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); @@ -244,10 +244,10 @@ TEST_P(FileProviderParamsDefault, two_allocations) { memset(ptr2, 0x22, size); umf_result = umfMemoryProviderFree(provider.get(), ptr1, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } TEST_P(FileProviderParamsDefault, alloc_page64_align_0) { @@ -366,12 +366,12 @@ TEST_P(FileProviderParamsDefault, get_name) { TEST_P(FileProviderParamsDefault, free_size_0_ptr_not_null) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(FileProviderParamsDefault, free_NULL) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } // other negative tests @@ -449,7 +449,7 @@ TEST_F(test, set_null_path) { TEST_P(FileProviderParamsDefault, free_INVALID_POINTER_SIZE_GT_0) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(FileProviderParamsDefault, purge_lazy_INVALID_POINTER) { @@ -512,7 +512,7 @@ TEST_P(FileProviderParamsShared, IPC_base_success_test) { ASSERT_EQ(ret, 0); umf_result = umfMemoryProviderFree(provider.get(), ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } TEST_P(FileProviderParamsShared, IPC_file_not_exist) { @@ -552,5 +552,5 @@ TEST_P(FileProviderParamsShared, IPC_file_not_exist) { ASSERT_EQ(new_ptr, nullptr); umf_result = umfMemoryProviderFree(provider.get(), ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } diff --git a/test/provider_file_memory_ipc.cpp b/test/provider_file_memory_ipc.cpp index ee7ab6c8ff..115322a476 100644 --- a/test/provider_file_memory_ipc.cpp +++ b/test/provider_file_memory_ipc.cpp @@ -73,7 +73,7 @@ HostMemoryAccessor hostAccessor; static std::vector ipcManyPoolsTestParamsList = { // TODO: enable it when sizes of allocations in ipcFixtures.hpp are fixed // {umfProxyPoolOps(), nullptr, umfFileMemoryProviderOps(), -// file_params_shared.get(), &hostAccessor, true}, +// file_params_shared.get(), &hostAccessor, false}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfFileMemoryProviderOps(), file_params_shared.get(), &hostAccessor, false}, From a27c45fa76ef8d0528e98b818b32e87a8a52efad Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 3 Dec 2024 11:46:55 +0100 Subject: [PATCH 13/82] Use libcoarse in the devdax provider Signed-off-by: Lukasz Dorau --- src/provider/provider_devdax_memory.c | 141 ++++++++++++-------------- test/provider_devdax_memory.cpp | 11 +- 2 files changed, 70 insertions(+), 82 deletions(-) diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index 32407acbba..c013c7ffba 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -58,6 +58,7 @@ umf_result_t umfDevDaxMemoryProviderParamsSetProtection( #else // !defined(_WIN32) && !defined(UMF_NO_HWLOC) #include "base_alloc_global.h" +#include "coarse.h" #include "libumf.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -74,6 +75,7 @@ typedef struct devdax_memory_provider_t { size_t offset; // offset in the file used for memory mapping utils_mutex_t lock; // lock of ptr and offset unsigned protection; // combination of OS-specific protection flags + coarse_t *coarse; // coarse library handle } devdax_memory_provider_t; // DevDax Memory provider settings struct @@ -133,6 +135,12 @@ devdax_translate_params(umf_devdax_memory_provider_params_t *in_params, return UMF_RESULT_SUCCESS; } +static umf_result_t devdax_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize); +static umf_result_t devdax_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize); + static umf_result_t devdax_initialize(void *params, void **provider) { umf_result_t ret; @@ -161,21 +169,42 @@ static umf_result_t devdax_initialize(void *params, void **provider) { memset(devdax_provider, 0, sizeof(*devdax_provider)); - ret = devdax_translate_params(in_params, devdax_provider); + coarse_params_t coarse_params = {0}; + coarse_params.provider = devdax_provider; + coarse_params.page_size = DEVDAX_PAGE_SIZE_2MB; + // The alloc callback is not available in case of the devdax provider + // because it is a fixed-size memory provider + // and the entire devdax memory is added as a single block + // to the coarse library. + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; // not available for the devdax provider + coarse_params.cb.split = devdax_allocation_split_cb; + coarse_params.cb.merge = devdax_allocation_merge_cb; + + coarse_t *coarse = NULL; + ret = coarse_new(&coarse_params, &coarse); if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_new() failed"); goto err_free_devdax_provider; } + devdax_provider->coarse = coarse; + + ret = devdax_translate_params(in_params, devdax_provider); + if (ret != UMF_RESULT_SUCCESS) { + goto err_coarse_delete; + } + devdax_provider->size = in_params->size; if (utils_copy_path(in_params->path, devdax_provider->path, PATH_MAX)) { - goto err_free_devdax_provider; + goto err_coarse_delete; } int fd = utils_devdax_open(in_params->path); if (fd == -1) { LOG_ERR("cannot open the device DAX: %s", in_params->path); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_free_devdax_provider; + goto err_coarse_delete; } bool is_dax = false; @@ -189,23 +218,26 @@ static umf_result_t devdax_initialize(void *params, void **provider) { LOG_PDEBUG("mapping the devdax failed (path=%s, size=%zu)", in_params->path, devdax_provider->size); ret = UMF_RESULT_ERROR_UNKNOWN; - goto err_free_devdax_provider; + goto err_coarse_delete; } if (!is_dax) { LOG_ERR("mapping the devdax with MAP_SYNC failed: %s", in_params->path); ret = UMF_RESULT_ERROR_UNKNOWN; - - if (devdax_provider->base) { - utils_munmap(devdax_provider->base, devdax_provider->size); - } - - goto err_free_devdax_provider; + goto err_unmap_devdax; } LOG_DEBUG("devdax memory mapped (path=%s, size=%zu, addr=%p)", in_params->path, devdax_provider->size, devdax_provider->base); + // add the entire devdax memory as a single block + ret = coarse_add_memory_fixed(coarse, devdax_provider->base, + devdax_provider->size); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("adding memory block failed"); + goto err_unmap_devdax; + } + if (utils_mutex_init(&devdax_provider->lock) == NULL) { LOG_ERR("lock init failed"); ret = UMF_RESULT_ERROR_UNKNOWN; @@ -217,7 +249,11 @@ static umf_result_t devdax_initialize(void *params, void **provider) { return UMF_RESULT_SUCCESS; err_unmap_devdax: - utils_munmap(devdax_provider->base, devdax_provider->size); + if (devdax_provider->base) { + utils_munmap(devdax_provider->base, devdax_provider->size); + } +err_coarse_delete: + coarse_delete(devdax_provider->coarse); err_free_devdax_provider: umf_ba_global_free(devdax_provider); return ret; @@ -227,78 +263,15 @@ static void devdax_finalize(void *provider) { devdax_memory_provider_t *devdax_provider = provider; utils_mutex_destroy_not_free(&devdax_provider->lock); utils_munmap(devdax_provider->base, devdax_provider->size); + coarse_delete(devdax_provider->coarse); umf_ba_global_free(devdax_provider); } -static int devdax_alloc_aligned(size_t length, size_t alignment, void *base, - size_t size, utils_mutex_t *lock, - void **out_addr, size_t *offset) { - assert(out_addr); - - if (utils_mutex_lock(lock)) { - LOG_ERR("locking file offset failed"); - return -1; - } - - uintptr_t ptr = (uintptr_t)base + *offset; - uintptr_t rest_of_div = alignment ? (ptr % alignment) : 0; - - if (alignment > 0 && rest_of_div > 0) { - ptr += alignment - rest_of_div; - } - - size_t new_offset = ptr - (uintptr_t)base + length; - - if (new_offset > size) { - utils_mutex_unlock(lock); - LOG_ERR("cannot allocate more memory than the device DAX size: %zu", - size); - return -1; - } - - *offset = new_offset; - *out_addr = (void *)ptr; - - utils_mutex_unlock(lock); - - return 0; -} - static umf_result_t devdax_alloc(void *provider, size_t size, size_t alignment, void **resultPtr) { - int ret; - - // alignment must be a power of two and a multiple or a divider of the page size - if (alignment && ((alignment & (alignment - 1)) || - ((alignment % DEVDAX_PAGE_SIZE_2MB) && - (DEVDAX_PAGE_SIZE_2MB % alignment)))) { - LOG_ERR("wrong alignment: %zu (not a power of 2 or a multiple or a " - "divider of the page size (%zu))", - alignment, DEVDAX_PAGE_SIZE_2MB); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (IS_NOT_ALIGNED(alignment, DEVDAX_PAGE_SIZE_2MB)) { - alignment = ALIGN_UP(alignment, DEVDAX_PAGE_SIZE_2MB); - } - devdax_memory_provider_t *devdax_provider = (devdax_memory_provider_t *)provider; - - void *addr = NULL; - errno = 0; - ret = devdax_alloc_aligned(size, alignment, devdax_provider->base, - devdax_provider->size, &devdax_provider->lock, - &addr, &devdax_provider->offset); - if (ret) { - devdax_store_last_native_error(UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED, 0); - LOG_ERR("memory allocation failed"); - return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; - } - - *resultPtr = addr; - - return UMF_RESULT_SUCCESS; + return coarse_alloc(devdax_provider->coarse, size, alignment, resultPtr); } static void devdax_get_last_native_error(void *provider, const char **ppMessage, @@ -384,6 +357,14 @@ static const char *devdax_get_name(void *provider) { static umf_result_t devdax_allocation_split(void *provider, void *ptr, size_t totalSize, size_t firstSize) { + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + return coarse_split(devdax_provider->coarse, ptr, totalSize, firstSize); +} + +static umf_result_t devdax_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { (void)provider; (void)ptr; (void)totalSize; @@ -393,6 +374,14 @@ static umf_result_t devdax_allocation_split(void *provider, void *ptr, static umf_result_t devdax_allocation_merge(void *provider, void *lowPtr, void *highPtr, size_t totalSize) { + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + return coarse_merge(devdax_provider->coarse, lowPtr, highPtr, totalSize); +} + +static umf_result_t devdax_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, + size_t totalSize) { (void)provider; (void)lowPtr; (void)highPtr; diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index 0fd0705da4..bb00f205a6 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -237,30 +237,29 @@ TEST_P(umfProviderTest, purge_force) { TEST_P(umfProviderTest, alloc_page64_align_page_minus_1_WRONG_ALIGNMENT_1) { test_alloc_failure(provider.get(), page_plus_64, page_size - 1, - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_page64_align_one_half_pages_WRONG_ALIGNMENT_2) { test_alloc_failure(provider.get(), page_plus_64, page_size + (page_size / 2), - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_page64_WRONG_ALIGNMENT_3_pages) { test_alloc_failure(provider.get(), page_plus_64, 3 * page_size, - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_3_pages_WRONG_ALIGNMENT_3_pages) { test_alloc_failure(provider.get(), 3 * page_size, 3 * page_size, - UMF_RESULT_ERROR_INVALID_ARGUMENT, 0); + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); } TEST_P(umfProviderTest, alloc_WRONG_SIZE) { size_t size = (size_t)(-1) & ~(page_size - 1); test_alloc_failure(provider.get(), size, 0, - UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, - UMF_DEVDAX_RESULT_ERROR_ALLOC_FAILED); + UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, 0); } // other positive tests From 96605c4a3a962d698c5871cf40ae861585abfc7e Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 25 Nov 2024 10:15:34 +0100 Subject: [PATCH 14/82] Add free() to the devdax provider --- src/provider/provider_devdax_memory.c | 7 +++++++ test/provider_devdax_memory.cpp | 10 +++++----- test/provider_devdax_memory_ipc.cpp | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index c013c7ffba..463b796ec0 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -516,6 +516,12 @@ static umf_result_t devdax_close_ipc_handle(void *provider, void *ptr, return UMF_RESULT_SUCCESS; } +static umf_result_t devdax_free(void *provider, void *ptr, size_t size) { + devdax_memory_provider_t *devdax_provider = + (devdax_memory_provider_t *)provider; + return coarse_free(devdax_provider->coarse, ptr, size); +} + static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .version = UMF_VERSION_CURRENT, .initialize = devdax_initialize, @@ -525,6 +531,7 @@ static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .get_recommended_page_size = devdax_get_recommended_page_size, .get_min_page_size = devdax_get_min_page_size, .get_name = devdax_get_name, + .ext.free = devdax_free, .ext.purge_lazy = devdax_purge_lazy, .ext.purge_force = devdax_purge_force, .ext.allocation_merge = devdax_allocation_merge, diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index bb00f205a6..afff1de4f7 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -100,7 +100,7 @@ static void test_alloc_free_success(umf_memory_provider_handle_t provider, } umf_result = umfMemoryProviderFree(provider, ptr, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } static void verify_last_native_error(umf_memory_provider_handle_t provider, @@ -162,7 +162,7 @@ TEST_F(test, test_if_mapped_with_MAP_SYNC) { bool flag_found = is_mapped_with_MAP_SYNC(path, buf, size); umf_result = umfMemoryProviderFree(hProvider, buf, size); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); @@ -294,12 +294,12 @@ TEST_P(umfProviderTest, get_name) { TEST_P(umfProviderTest, free_size_0_ptr_not_null) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(umfProviderTest, free_NULL) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } // other negative tests @@ -307,7 +307,7 @@ TEST_P(umfProviderTest, free_NULL) { TEST_P(umfProviderTest, free_INVALID_POINTER_SIZE_GT_0) { umf_result_t umf_result = umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(umfProviderTest, purge_lazy_INVALID_POINTER) { diff --git a/test/provider_devdax_memory_ipc.cpp b/test/provider_devdax_memory_ipc.cpp index 3941f66e90..921347f400 100644 --- a/test/provider_devdax_memory_ipc.cpp +++ b/test/provider_devdax_memory_ipc.cpp @@ -53,7 +53,7 @@ static std::vector getIpcProxyPoolTestParamsList(void) { ipcProxyPoolTestParamsList = { {umfProxyPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, true}, + defaultDevDaxParams.get(), &hostAccessor, false}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), defaultDevDaxParams.get(), &hostAccessor, false}, From d9f1feec2d314784b7bd2da0281fd4411544cf9d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 3 Dec 2024 11:47:23 +0100 Subject: [PATCH 15/82] Remove free_not_supp from the ipcTestParams tuple Remove free_not_supp from the ipcTestParams tuple. It is not needed any more. Signed-off-by: Lukasz Dorau --- test/ipcAPI.cpp | 2 +- test/ipcFixtures.hpp | 44 +++++++------------------- test/provider_devdax_memory_ipc.cpp | 6 ++-- test/provider_file_memory_ipc.cpp | 12 +++---- test/provider_os_memory.cpp | 4 +-- test/providers/provider_level_zero.cpp | 10 +++--- 6 files changed, 29 insertions(+), 49 deletions(-) diff --git a/test/ipcAPI.cpp b/test/ipcAPI.cpp index 4df32a1c9b..aa22f353dc 100644 --- a/test/ipcAPI.cpp +++ b/test/ipcAPI.cpp @@ -116,4 +116,4 @@ HostMemoryAccessor hostMemoryAccessor; INSTANTIATE_TEST_SUITE_P(umfIpcTestSuite, umfIpcTest, ::testing::Values(ipcTestParams{ umfProxyPoolOps(), nullptr, &IPC_MOCK_PROVIDER_OPS, - nullptr, &hostMemoryAccessor, false})); + nullptr, &hostMemoryAccessor})); diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index 161a84844c..6fbacfa222 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -47,25 +47,23 @@ class HostMemoryAccessor : public MemoryAccessor { }; // ipcTestParams: -// pool_ops, pool_params, provider_ops, provider_params, memoryAccessor, free_not_supp -// free_not_supp (bool) - provider does not support the free() op +// pool_ops, pool_params, provider_ops, provider_params, memoryAccessor using ipcTestParams = std::tuple; + void *, MemoryAccessor *>; struct umfIpcTest : umf_test::test, ::testing::WithParamInterface { umfIpcTest() {} void SetUp() override { test::SetUp(); - auto [pool_ops, pool_params, provider_ops, provider_params, accessor, - free_not_supp] = this->GetParam(); + auto [pool_ops, pool_params, provider_ops, provider_params, accessor] = + this->GetParam(); poolOps = pool_ops; poolParams = pool_params; providerOps = provider_ops; providerParams = provider_params; memAccessor = accessor; - freeNotSupported = free_not_supp; } void TearDown() override { test::TearDown(); } @@ -124,18 +122,8 @@ struct umfIpcTest : umf_test::test, void *poolParams = nullptr; umf_memory_provider_ops_t *providerOps = nullptr; void *providerParams = nullptr; - bool freeNotSupported = false; }; -static inline umf_result_t -get_umf_result_of_free(bool freeNotSupported, umf_result_t expected_result) { - if (freeNotSupported) { - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - return expected_result; -} - TEST_P(umfIpcTest, GetIPCHandleSize) { size_t size = 0; umf::pool_unique_handle_t pool = makePool(); @@ -177,8 +165,7 @@ TEST_P(umfIpcTest, GetIPCHandleInvalidArgs) { EXPECT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); ret = umfFree(ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } TEST_P(umfIpcTest, CloseIPCHandleInvalidPtr) { @@ -239,8 +226,7 @@ TEST_P(umfIpcTest, BasicFlow) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); pool.reset(nullptr); EXPECT_EQ(stat.getCount, 1); @@ -303,8 +289,7 @@ TEST_P(umfIpcTest, GetPoolByOpenedHandle) { for (size_t i = 0; i < NUM_ALLOCS; ++i) { umf_result_t ret = umfFree(ptrs[i]); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } } @@ -330,8 +315,7 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ptr = umfPoolMalloc(pool.get(), SIZE); ASSERT_NE(ptr, nullptr); @@ -353,8 +337,7 @@ TEST_P(umfIpcTest, AllocFreeAllocTest) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); pool.reset(nullptr); EXPECT_EQ(stat.getCount, stat.putCount); @@ -405,8 +388,7 @@ TEST_P(umfIpcTest, openInTwoPools) { EXPECT_EQ(ret, UMF_RESULT_SUCCESS); ret = umfPoolFree(pool1.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); pool1.reset(nullptr); pool2.reset(nullptr); @@ -457,8 +439,7 @@ TEST_P(umfIpcTest, ConcurrentGetPutHandles) { for (void *ptr : ptrs) { umf_result_t ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } pool.reset(nullptr); @@ -520,8 +501,7 @@ TEST_P(umfIpcTest, ConcurrentOpenCloseHandles) { for (void *ptr : ptrs) { umf_result_t ret = umfPoolFree(pool.get(), ptr); - EXPECT_EQ(ret, - get_umf_result_of_free(freeNotSupported, UMF_RESULT_SUCCESS)); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); } pool.reset(nullptr); diff --git a/test/provider_devdax_memory_ipc.cpp b/test/provider_devdax_memory_ipc.cpp index 921347f400..ed4f1a5f8d 100644 --- a/test/provider_devdax_memory_ipc.cpp +++ b/test/provider_devdax_memory_ipc.cpp @@ -53,14 +53,14 @@ static std::vector getIpcProxyPoolTestParamsList(void) { ipcProxyPoolTestParamsList = { {umfProxyPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, false}, + defaultDevDaxParams.get(), &hostAccessor}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, false}, + defaultDevDaxParams.get(), &hostAccessor}, #endif #ifdef UMF_POOL_SCALABLE_ENABLED {umfScalablePoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - defaultDevDaxParams.get(), &hostAccessor, false}, + defaultDevDaxParams.get(), &hostAccessor}, #endif }; diff --git a/test/provider_file_memory_ipc.cpp b/test/provider_file_memory_ipc.cpp index 115322a476..70c1acd8f5 100644 --- a/test/provider_file_memory_ipc.cpp +++ b/test/provider_file_memory_ipc.cpp @@ -73,14 +73,14 @@ HostMemoryAccessor hostAccessor; static std::vector ipcManyPoolsTestParamsList = { // TODO: enable it when sizes of allocations in ipcFixtures.hpp are fixed // {umfProxyPoolOps(), nullptr, umfFileMemoryProviderOps(), -// file_params_shared.get(), &hostAccessor, false}, +// file_params_shared.get(), &hostAccessor}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_shared.get(), &hostAccessor, false}, + file_params_shared.get(), &hostAccessor}, #endif #ifdef UMF_POOL_SCALABLE_ENABLED {umfScalablePoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_shared.get(), &hostAccessor, false}, + file_params_shared.get(), &hostAccessor}, #endif }; @@ -96,14 +96,14 @@ static std::vector getIpcFsDaxTestParamsList(void) { ipcFsDaxTestParamsList = { // TODO: enable it when sizes of allocations in ipcFixtures.hpp are fixed // {umfProxyPoolOps(), nullptr, umfFileMemoryProviderOps(), -// file_params_fsdax.get(), &hostAccessor, true}, +// file_params_fsdax.get(), &hostAccessor}, #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_fsdax.get(), &hostAccessor, false}, + file_params_fsdax.get(), &hostAccessor}, #endif #ifdef UMF_POOL_SCALABLE_ENABLED {umfScalablePoolOps(), nullptr, umfFileMemoryProviderOps(), - file_params_fsdax.get(), &hostAccessor, false}, + file_params_fsdax.get(), &hostAccessor}, #endif }; diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 57bce46d24..687db08053 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -465,11 +465,11 @@ disjoint_params_unique_handle_t disjointParams = disjointPoolParams(); static std::vector ipcTestParamsList = { #if (defined UMF_POOL_DISJOINT_ENABLED) {umfDisjointPoolOps(), disjointParams.get(), umfOsMemoryProviderOps(), - os_params.get(), &hostAccessor, false}, + os_params.get(), &hostAccessor}, #endif #ifdef UMF_POOL_JEMALLOC_ENABLED {umfJemallocPoolOps(), nullptr, umfOsMemoryProviderOps(), os_params.get(), - &hostAccessor, false}, + &hostAccessor}, #endif }; diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index d0584777be..78b5e4847a 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -454,9 +454,9 @@ INSTANTIATE_TEST_SUITE_P( #ifdef _WIN32 GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfIpcTest); #else -INSTANTIATE_TEST_SUITE_P( - umfLevelZeroProviderTestSuite, umfIpcTest, - ::testing::Values(ipcTestParams{ - umfProxyPoolOps(), nullptr, umfLevelZeroMemoryProviderOps(), - l0Params_device_memory.get(), &l0Accessor, false})); +INSTANTIATE_TEST_SUITE_P(umfLevelZeroProviderTestSuite, umfIpcTest, + ::testing::Values(ipcTestParams{ + umfProxyPoolOps(), nullptr, + umfLevelZeroMemoryProviderOps(), + l0Params_device_memory.get(), &l0Accessor})); #endif From d8a340e06749ca6943472e6ac33fb443bcdb791e Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 27 Nov 2024 14:25:28 +0100 Subject: [PATCH 16/82] Disable jemalloc pool on RHEL Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_multi_numa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_multi_numa.yml b/.github/workflows/reusable_multi_numa.yml index c012f3e19e..2ccb2d8f32 100644 --- a/.github/workflows/reusable_multi_numa.yml +++ b/.github/workflows/reusable_multi_numa.yml @@ -45,7 +45,7 @@ jobs: -DUMF_BUILD_TESTS=ON -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=${{ matrix.os == 'rhel-9.1' && 'OFF' || 'ON' }} -DUMF_TESTS_FAIL_ON_SKIP=ON ${{ matrix.build_type == 'Debug' && matrix.os == 'ubuntu-22.04' && '-DUMF_USE_COVERAGE=ON' || '' }} From e95d92edb5faa3aa7ae99e56e3f2dc0a4153e641 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 4 Dec 2024 12:25:20 +0100 Subject: [PATCH 17/82] Link statically with custom jemalloc with disabled initial TLS Link statically with custom jemalloc built from sources with the following non-default options enabled: --with-jemalloc-prefix=je_ - add je_ prefix to all public APIs --disable-cxx - Disable C++ integration. This will cause new and delete operators implementations to be omitted. --disable-initial-exec-tls - Disable the initial-exec TLS model for jemalloc's internal thread-local storage (on those platforms that support explicit settings). This can allow jemalloc to be dynamically loaded after program startup (e.g. using dlopen). Fixes: #891 Fixes: #894 Fixes: #903 Signed-off-by: Lukasz Dorau --- .github/docker/ubuntu-20.04.Dockerfile | 2 +- .github/docker/ubuntu-22.04.Dockerfile | 2 +- .github/workflows/coverity.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/reusable_basic.yml | 4 +- .github/workflows/reusable_benchmarks.yml | 2 +- .github/workflows/reusable_codeql.yml | 2 +- .github/workflows/reusable_fast.yml | 4 +- .github/workflows/reusable_proxy_lib.yml | 2 +- .github/workflows/reusable_sanitizers.yml | 2 +- .github/workflows/reusable_valgrind.yml | 2 +- CMakeLists.txt | 94 ++++++++++++++++--- README.md | 16 +++- benchmark/ubench.c | 4 +- examples/CMakeLists.txt | 7 +- examples/cmake/FindJEMALLOC.cmake | 6 +- examples/dram_and_fsdax/CMakeLists.txt | 6 +- scripts/qemu/run-build.sh | 2 +- src/pool/CMakeLists.txt | 5 +- src/pool/pool_jemalloc.c | 10 -- test/CMakeLists.txt | 7 ++ .../drd-umf_test-jemalloc_coarse_devdax.supp | 23 +---- .../drd-umf_test-jemalloc_coarse_file.supp | 23 +---- test/supp/drd-umf_test-jemalloc_pool.supp | 5 +- ...grind-umf_test-jemalloc_coarse_devdax.supp | 23 +---- ...elgrind-umf_test-jemalloc_coarse_file.supp | 23 +---- .../supp/helgrind-umf_test-jemalloc_pool.supp | 5 +- 27 files changed, 148 insertions(+), 137 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 069deeac93..a6a45a8c1b 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -24,7 +24,6 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libjemalloc-dev \ libhwloc-dev \ libtbb-dev" @@ -34,6 +33,7 @@ ARG TEST_DEPS="\ # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ + automake \ clang \ g++-7 \ python3-pip \ diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 08d546083d..75c71c526c 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -24,7 +24,6 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libjemalloc-dev \ libhwloc-dev \ libtbb-dev" @@ -34,6 +33,7 @@ ARG TEST_DEPS="\ # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ + automake \ clang \ python3-pip \ sudo \ diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index dfa03fc4f4..531a463c77 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -31,7 +31,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev + sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev - name: Download Coverity run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 281ae00615..46543fac83 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -67,7 +67,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind + sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind - name: Configure CMake run: > diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 9b71c7d1b8..0d27fb9c3d 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -124,7 +124,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y clang cmake libnuma-dev libjemalloc-dev lcov + sudo apt-get install -y clang cmake libnuma-dev lcov - name: Install TBB apt package if: matrix.install_tbb == 'ON' @@ -469,7 +469,7 @@ jobs: python3 -m pip install -r third_party/requirements.txt - name: Install hwloc - run: brew install hwloc jemalloc tbb + run: brew install hwloc tbb automake - name: Configure build run: > diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index 41710029c8..ed6a482947 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -34,7 +34,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - name: Initialize vcpkg if: matrix.os == 'windows-latest' diff --git a/.github/workflows/reusable_codeql.yml b/.github/workflows/reusable_codeql.yml index e764563103..046c320817 100644 --- a/.github/workflows/reusable_codeql.yml +++ b/.github/workflows/reusable_codeql.yml @@ -62,7 +62,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev + sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libtbb-dev # Latest distros do not allow global pip installation - name: "[Lin] Install Python requirements in venv" diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index e25de68a1b..5673727ac1 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -79,13 +79,13 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt-get install -y cmake libjemalloc-dev libhwloc-dev libnuma-dev libtbb-dev + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - name: Install dependencies (ubuntu-20.04) if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - sudo apt-get install -y cmake libjemalloc-dev libnuma-dev libtbb-dev + sudo apt-get install -y cmake libnuma-dev libtbb-dev .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package - name: Set ptrace value for IPC test (on Linux only) diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index 2a27161b3e..103c4a5163 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -32,7 +32,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev lcov + sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov - name: Set ptrace value for IPC test run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" diff --git a/.github/workflows/reusable_sanitizers.yml b/.github/workflows/reusable_sanitizers.yml index 3acda6833e..f9e121f889 100644 --- a/.github/workflows/reusable_sanitizers.yml +++ b/.github/workflows/reusable_sanitizers.yml @@ -29,7 +29,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev + sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libtbb-dev - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' diff --git a/.github/workflows/reusable_valgrind.yml b/.github/workflows/reusable_valgrind.yml index 86ceb68c68..3e0af273a5 100644 --- a/.github/workflows/reusable_valgrind.yml +++ b/.github/workflows/reusable_valgrind.yml @@ -20,7 +20,7 @@ jobs: - name: Install apt packages run: | sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind + sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind - name: Configure CMake run: > diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dcc293d2e..fefe646851 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,87 @@ else() message(FATAL_ERROR "Unknown OS type") endif() +if(NOT UMF_BUILD_LIBUMF_POOL_JEMALLOC) + set(UMF_POOL_JEMALLOC_ENABLED FALSE) +elseif(WINDOWS) + pkg_check_modules(JEMALLOC jemalloc) + if(NOT JEMALLOC_FOUND) + find_package(JEMALLOC REQUIRED jemalloc) + endif() +else() + if(NOT DEFINED UMF_JEMALLOC_REPO) + set(UMF_JEMALLOC_REPO "https://github.com/jemalloc/jemalloc.git") + endif() + + if(NOT DEFINED UMF_JEMALLOC_TAG) + set(UMF_JEMALLOC_TAG 5.3.0) + endif() + + include(FetchContent) + message( + STATUS + "Will fetch jemalloc from ${UMF_JEMALLOC_REPO} (tag: ${UMF_JEMALLOC_TAG})" + ) + + FetchContent_Declare( + jemalloc_targ + GIT_REPOSITORY ${UMF_JEMALLOC_REPO} + GIT_TAG ${UMF_JEMALLOC_TAG}) + FetchContent_MakeAvailable(jemalloc_targ) + + add_custom_command( + COMMAND ./autogen.sh + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_SOURCE_DIR}/configure) + add_custom_command( + # Custom jemalloc build. Non-default options used: + # --with-jemalloc-prefix=je_ - add je_ prefix to all public APIs + # --disable-cxx - Disable C++ integration. This will cause new and + # delete operators implementations to be omitted. + # --disable-initial-exec-tls - Disable the initial-exec TLS model for + # jemalloc's internal thread-local storage (on those platforms that + # support explicit settings). This can allow jemalloc to be dynamically + # loaded after program startup (e.g. using dlopen). + COMMAND + ./configure --prefix=${jemalloc_targ_BINARY_DIR} + --with-jemalloc-prefix=je_ --disable-cxx --disable-initial-exec-tls + CFLAGS=-fPIC + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_SOURCE_DIR}/Makefile + DEPENDS ${jemalloc_targ_SOURCE_DIR}/configure) + add_custom_command( + COMMAND make + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la + DEPENDS ${jemalloc_targ_SOURCE_DIR}/Makefile) + add_custom_command( + COMMAND make install + WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} + OUTPUT ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a + DEPENDS ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la) + + add_custom_target(jemalloc_prod + DEPENDS ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) + add_library(jemalloc INTERFACE) + target_link_libraries( + jemalloc INTERFACE ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) + add_dependencies(jemalloc jemalloc_prod) + + set(JEMALLOC_LIBRARY_DIRS ${jemalloc_targ_BINARY_DIR}/lib) + set(JEMALLOC_INCLUDE_DIRS ${jemalloc_targ_BINARY_DIR}/include) + set(JEMALLOC_LIBRARIES ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) +endif() + +if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES) + set(UMF_POOL_JEMALLOC_ENABLED TRUE) + # add PATH to DLL on Windows + set(DLL_PATH_LIST + "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") + message(STATUS " JEMALLOC_LIBRARIES = ${JEMALLOC_LIBRARIES}") + message(STATUS " JEMALLOC_INCLUDE_DIRS = ${JEMALLOC_INCLUDE_DIRS}") + message(STATUS " JEMALLOC_LIBRARY_DIRS = ${JEMALLOC_LIBRARY_DIRS}") +endif() + if(UMF_DISABLE_HWLOC) message(STATUS "hwloc is disabled, hence OS provider, memtargets, " "topology discovery, examples won't be available!") @@ -402,19 +483,6 @@ else() set(UMF_POOL_SCALABLE_ENABLED FALSE) endif() -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - pkg_check_modules(JEMALLOC jemalloc) - if(NOT JEMALLOC_FOUND) - find_package(JEMALLOC REQUIRED jemalloc) - endif() - if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES) - set(UMF_POOL_JEMALLOC_ENABLED TRUE) - # add PATH to DLL on Windows - set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") - endif() -endif() - if(WINDOWS) # TODO: enable the proxy library in the Debug build on Windows # diff --git a/README.md b/README.md index 3379132e7f..c04d7d22ec 100644 --- a/README.md +++ b/README.md @@ -281,11 +281,25 @@ pool manager built as a separate static library: libjemalloc_pool.a on Linux and jemalloc_pool.lib on Windows. The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option has to be turned `ON` to build this library. +[jemalloc](https://github.com/jemalloc/jemalloc) is required to build the jemalloc pool. + +In case of Linux OS jemalloc is built from the (fetched) sources with the following +non-default options enabled: +- `--with-jemalloc-prefix=je_` - adds the `je_` prefix to all public APIs, +- `--disable-cxx` - disables C++ integration, it will cause the `new` and the `delete` + operators implementations to be omitted. +- `--disable-initial-exec-tls` - disables the initial-exec TLS model for jemalloc's + internal thread-local storage (on those platforms that support + explicit settings), it can allow jemalloc to be dynamically + loaded after program startup (e.g. using `dlopen()`). + +The default jemalloc package is required on Windows. + ##### Requirements 1) The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned `ON` 2) Required packages: - - libjemalloc-dev (Linux) or jemalloc (Windows) + - jemalloc (Windows only) #### Scalable Pool (part of libumf) diff --git a/benchmark/ubench.c b/benchmark/ubench.c index 5f1bfe9e48..845dc881db 100644 --- a/benchmark/ubench.c +++ b/benchmark/ubench.c @@ -445,8 +445,8 @@ static void do_ipc_get_put_benchmark(alloc_t *allocs, size_t num_allocs, } } -int create_level_zero_params(ze_context_handle_t *context, - ze_device_handle_t *device) { +static int create_level_zero_params(ze_context_handle_t *context, + ze_device_handle_t *device) { uint32_t driver_idx = 0; ze_driver_handle_t driver = NULL; diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 942579a303..ee99fb07d5 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -284,8 +284,11 @@ if(LINUX) SRCS dram_and_fsdax/dram_and_fsdax.c LIBS umf jemalloc_pool) - target_link_directories(${EXAMPLE_NAME} PRIVATE - ${LIBHWLOC_LIBRARY_DIRS}) + target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--no-as-needed,-ldl") + + target_link_directories( + ${EXAMPLE_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS} + ${JEMALLOC_LIBRARY_DIRS}) add_test( NAME ${EXAMPLE_NAME} diff --git a/examples/cmake/FindJEMALLOC.cmake b/examples/cmake/FindJEMALLOC.cmake index 89d488ecc0..e6db190d4a 100644 --- a/examples/cmake/FindJEMALLOC.cmake +++ b/examples/cmake/FindJEMALLOC.cmake @@ -2,9 +2,11 @@ # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -message(STATUS "Checking for module 'jemalloc' using find_library()") +message( + STATUS "Looking for the static 'libjemalloc.a' library using find_library()" +) -find_library(JEMALLOC_LIBRARY NAMES libjemalloc jemalloc) +find_library(JEMALLOC_LIBRARY NAMES libjemalloc.a jemalloc.a) set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY}) get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY) diff --git a/examples/dram_and_fsdax/CMakeLists.txt b/examples/dram_and_fsdax/CMakeLists.txt index 0d0bf25935..014a08fcc3 100644 --- a/examples/dram_and_fsdax/CMakeLists.txt +++ b/examples/dram_and_fsdax/CMakeLists.txt @@ -21,10 +21,8 @@ if(NOT LIBHWLOC_FOUND) find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) endif() -pkg_check_modules(JEMALLOC jemalloc) -if(NOT JEMALLOC_FOUND) - find_package(JEMALLOC REQUIRED jemalloc) -endif() +# find the custom jemalloc pointed by CMAKE_PREFIX_PATH +find_package(JEMALLOC REQUIRED jemalloc) # build the example set(EXAMPLE_NAME umf_example_dram_and_fsdax) diff --git a/scripts/qemu/run-build.sh b/scripts/qemu/run-build.sh index 06d6043f6b..b0f4bee1e5 100755 --- a/scripts/qemu/run-build.sh +++ b/scripts/qemu/run-build.sh @@ -14,7 +14,7 @@ pwd echo password | sudo -Sk apt-get update echo password | sudo -Sk apt-get install -y git cmake gcc g++ pkg-config \ - numactl libnuma-dev hwloc libhwloc-dev libjemalloc-dev libtbb-dev valgrind lcov + numactl libnuma-dev hwloc libhwloc-dev libtbb-dev valgrind lcov mkdir build cd build diff --git a/src/pool/CMakeLists.txt b/src/pool/CMakeLists.txt index bdd196b041..9cf9d16654 100644 --- a/src/pool/CMakeLists.txt +++ b/src/pool/CMakeLists.txt @@ -45,10 +45,13 @@ if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) NAME jemalloc_pool TYPE STATIC SRCS pool_jemalloc.c ${POOL_EXTRA_SRCS} - LIBS jemalloc ${POOL_EXTRA_LIBS}) + LIBS ${JEMALLOC_LIBRARIES} ${POOL_EXTRA_LIBS}) target_include_directories(jemalloc_pool PRIVATE ${JEMALLOC_INCLUDE_DIRS}) target_compile_definitions(jemalloc_pool PRIVATE ${POOL_COMPILE_DEFINITIONS}) add_library(${PROJECT_NAME}::jemalloc_pool ALIAS jemalloc_pool) + if(NOT WINDOWS) + add_dependencies(jemalloc_pool jemalloc) + endif() install(TARGETS jemalloc_pool EXPORT ${PROJECT_NAME}-targets) endif() diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index 3ec7c78050..94fd655cc4 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -22,16 +22,6 @@ #include -// The Windows version of jemalloc uses API with je_ prefix, -// while the Linux one does not. -#ifndef _WIN32 -#define je_mallocx mallocx -#define je_dallocx dallocx -#define je_rallocx rallocx -#define je_mallctl mallctl -#define je_malloc_usable_size malloc_usable_size -#endif - #define MALLOCX_ARENA_MAX (MALLCTL_ARENAS_ALL - 1) typedef struct jemalloc_memory_pool_t { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b237428669..b54822b962 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -747,6 +747,13 @@ if(LINUX set(STANDALONE_CMAKE_OPTIONS "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" ) + if(JEMALLOC_INCLUDE_DIRS) + # add custom jemalloc installation + set(STANDALONE_CMAKE_OPTIONS + "${STANDALONE_CMAKE_OPTIONS} -DCMAKE_PREFIX_PATH=${JEMALLOC_INCLUDE_DIRS}/../" + ) + endif() + add_test( NAME umf-standalone_examples COMMAND diff --git a/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp b/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp index fd071432b6..bc4f2295f4 100644 --- a/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp +++ b/test/supp/drd-umf_test-jemalloc_coarse_devdax.supp @@ -1,27 +1,8 @@ { - False-positive ConflictingAccess in libjemalloc.so + False-positive ConflictingAccess in jemalloc drd:ConflictingAccess - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/drd-umf_test-jemalloc_coarse_file.supp b/test/supp/drd-umf_test-jemalloc_coarse_file.supp index fd071432b6..bc4f2295f4 100644 --- a/test/supp/drd-umf_test-jemalloc_coarse_file.supp +++ b/test/supp/drd-umf_test-jemalloc_coarse_file.supp @@ -1,27 +1,8 @@ { - False-positive ConflictingAccess in libjemalloc.so + False-positive ConflictingAccess in jemalloc drd:ConflictingAccess - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive ConflictingAccess in libjemalloc.so - drd:ConflictingAccess - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/drd-umf_test-jemalloc_pool.supp b/test/supp/drd-umf_test-jemalloc_pool.supp index 965ef38844..cb6179f875 100644 --- a/test/supp/drd-umf_test-jemalloc_pool.supp +++ b/test/supp/drd-umf_test-jemalloc_pool.supp @@ -1,6 +1,7 @@ { - Conflicting Access in libjemalloc.so - internal issue of libjemalloc + False-positive ConflictingAccess in jemalloc drd:ConflictingAccess - obj:*libjemalloc.so* + ... + fun:je_* ... } diff --git a/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp b/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp index 18774f387c..ac8969c5ae 100644 --- a/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp +++ b/test/supp/helgrind-umf_test-jemalloc_coarse_devdax.supp @@ -1,27 +1,8 @@ { - False-positive Race in libjemalloc.so + False-positive Race in jemalloc Helgrind:Race - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp b/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp index 18774f387c..ac8969c5ae 100644 --- a/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp +++ b/test/supp/helgrind-umf_test-jemalloc_coarse_file.supp @@ -1,27 +1,8 @@ { - False-positive Race in libjemalloc.so + False-positive Race in jemalloc Helgrind:Race - obj:*/libjemalloc.so* ... - fun:mallocx - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:op_free - ... -} - -{ - False-positive Race in libjemalloc.so - Helgrind:Race - obj:*/libjemalloc.so* - ... - fun:__nptl_deallocate_tsd + fun:je_* ... } diff --git a/test/supp/helgrind-umf_test-jemalloc_pool.supp b/test/supp/helgrind-umf_test-jemalloc_pool.supp index 8068b023dc..98d748feac 100644 --- a/test/supp/helgrind-umf_test-jemalloc_pool.supp +++ b/test/supp/helgrind-umf_test-jemalloc_pool.supp @@ -1,6 +1,7 @@ { - Race in libjemalloc.so - internal issue of libjemalloc + False-positive Race in jemalloc Helgrind:Race - obj:*libjemalloc.so* + ... + fun:je_* ... } From 614c4b54e9b8240ea1f0fd830b35286131c257f1 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 4 Dec 2024 17:01:14 +0100 Subject: [PATCH 18/82] Incorporate jemalloc_pool into libumf Remove the separate static `jemalloc_pool` library. Make the `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned ON by default. Incorporate jemalloc_pool into libumf. Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_basic.yml | 3 --- CMakeLists.txt | 14 +++++++--- README.md | 5 ++-- benchmark/CMakeLists.txt | 5 ++-- benchmark/multithread.cpp | 2 ++ examples/CMakeLists.txt | 2 +- examples/dram_and_fsdax/CMakeLists.txt | 17 +++--------- src/CMakeLists.txt | 21 ++++++++++++--- src/libumf.def | 4 +++ src/libumf.map | 4 +++ src/pool/CMakeLists.txt | 17 ------------ src/pool/pool_jemalloc.c | 27 +++++++++++++++++++ test/CMakeLists.txt | 23 ++++++++-------- ...check-umf_test-jemalloc_coarse_devdax.supp | 7 +++++ ...emcheck-umf_test-jemalloc_coarse_file.supp | 7 +++++ test/test_installation.py | 7 ----- 16 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp create mode 100644 test/supp/memcheck-umf_test-jemalloc_coarse_file.supp diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 0d27fb9c3d..3b573453d8 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -209,7 +209,6 @@ jobs: --install-dir ${{env.INSTL_DIR}} --build-type ${{matrix.build_type}} --disjoint-pool - --jemalloc-pool ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} @@ -300,7 +299,6 @@ jobs: --install-dir ${{env.INSTL_DIR}} --build-type ${{matrix.build_type}} --disjoint-pool - --jemalloc-pool ${{matrix.shared_library == 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} ${{ matrix.shared_library == 'ON' && '--shared-library' || ''}} @@ -495,7 +493,6 @@ jobs: --install-dir ${{env.INSTL_DIR}} --build-type ${{env.BUILD_TYPE}} --disjoint-pool - --jemalloc-pool --proxy --umf-version ${{env.UMF_VERSION}} --shared-library diff --git a/CMakeLists.txt b/CMakeLists.txt index fefe646851..7fcfcbb952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,8 @@ endif() if(NOT UMF_BUILD_LIBUMF_POOL_JEMALLOC) set(UMF_POOL_JEMALLOC_ENABLED FALSE) + set(JEMALLOC_FOUND FALSE) + set(JEMALLOC_LIBRARIES FALSE) elseif(WINDOWS) pkg_check_modules(JEMALLOC jemalloc) if(NOT JEMALLOC_FOUND) @@ -190,6 +192,12 @@ if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES) message(STATUS " JEMALLOC_LIBRARIES = ${JEMALLOC_LIBRARIES}") message(STATUS " JEMALLOC_INCLUDE_DIRS = ${JEMALLOC_INCLUDE_DIRS}") message(STATUS " JEMALLOC_LIBRARY_DIRS = ${JEMALLOC_LIBRARY_DIRS}") +else() + set(UMF_POOL_JEMALLOC_ENABLED FALSE) + message( + STATUS + "Disabling the Jemalloc Pool and tests and benchmarks that use it because jemalloc was not built/found." + ) endif() if(UMF_DISABLE_HWLOC) @@ -523,14 +531,14 @@ elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE) ) endif() elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL JEMALLOC) - if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) + if(UMF_POOL_JEMALLOC_ENABLED) set(UMF_PROXY_LIB_ENABLED ON) set(PROXY_LIB_USES_JEMALLOC_POOL ON) - set(PROXY_LIBS jemalloc_pool umf) + set(PROXY_LIBS umf) else() message( STATUS - "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but UMF_BUILD_LIBUMF_POOL_JEMALLOC is OFF" + "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but the jemalloc pool is disabled" ) endif() else() diff --git a/README.md b/README.md index c04d7d22ec..81a82bfab1 100644 --- a/README.md +++ b/README.md @@ -298,8 +298,9 @@ The default jemalloc package is required on Windows. ##### Requirements 1) The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned `ON` -2) Required packages: - - jemalloc (Windows only) +2) jemalloc is required: +- on Linux and MacOS: jemalloc is fetched and built from sources (a custom build), +- on Windows: the default jemalloc package is required #### Scalable Pool (part of libumf) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 5605519ee2..b2f1299be3 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -51,7 +51,7 @@ function(add_umf_benchmark) set(BENCH_NAME umf-${ARG_NAME}) - set(BENCH_LIBS ${ARG_LIBS} umf) + set(BENCH_LIBS ${ARG_LIBS} umf umf_utils) add_umf_executable( NAME ${BENCH_NAME} @@ -121,8 +121,7 @@ set(LIB_DIRS ${LIBHWLOC_LIBRARY_DIRS}) if(UMF_BUILD_LIBUMF_POOL_DISJOINT) set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool) endif() -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - set(LIBS_OPTIONAL ${LIBS_OPTIONAL} jemalloc_pool ${JEMALLOC_LIBRARIES}) +if(UMF_POOL_JEMALLOC_ENABLED) set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS}) endif() if(LINUX) diff --git a/benchmark/multithread.cpp b/benchmark/multithread.cpp index 4558942ecb..ecc2385292 100644 --- a/benchmark/multithread.cpp +++ b/benchmark/multithread.cpp @@ -139,11 +139,13 @@ int main() { // ctest looks for "PASSED" in the output std::cout << "PASSED" << std::endl; +#if defined(UMF_POOL_DISJOINT_ENABLED) ret = umfDisjointPoolParamsDestroy(hDisjointParams); if (ret != UMF_RESULT_SUCCESS) { std::cerr << "disjoint pool params destroy failed" << std::endl; return -1; } +#endif return 0; } diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index ee99fb07d5..986ad56412 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -282,7 +282,7 @@ if(LINUX) add_umf_executable( NAME ${EXAMPLE_NAME} SRCS dram_and_fsdax/dram_and_fsdax.c - LIBS umf jemalloc_pool) + LIBS umf) target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--no-as-needed,-ldl") diff --git a/examples/dram_and_fsdax/CMakeLists.txt b/examples/dram_and_fsdax/CMakeLists.txt index 014a08fcc3..dcb538085e 100644 --- a/examples/dram_and_fsdax/CMakeLists.txt +++ b/examples/dram_and_fsdax/CMakeLists.txt @@ -21,24 +21,15 @@ if(NOT LIBHWLOC_FOUND) find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) endif() -# find the custom jemalloc pointed by CMAKE_PREFIX_PATH -find_package(JEMALLOC REQUIRED jemalloc) - # build the example set(EXAMPLE_NAME umf_example_dram_and_fsdax) add_executable(${EXAMPLE_NAME} dram_and_fsdax.c) target_include_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS}) -target_link_directories( - ${EXAMPLE_NAME} - PRIVATE - ${LIBUMF_LIBRARY_DIRS} - ${LIBHWLOC_LIBRARY_DIRS} - ${JEMALLOC_LIBRARY_DIRS}) +target_link_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_LIBRARY_DIRS} + ${LIBHWLOC_LIBRARY_DIRS}) -target_link_libraries( - ${EXAMPLE_NAME} PRIVATE hwloc jemalloc_pool ${JEMALLOC_LIBRARIES} - ${LIBUMF_LIBRARIES}) +target_link_libraries(${EXAMPLE_NAME} PRIVATE hwloc ${LIBUMF_LIBRARIES}) # an optional part - adds a test of this example add_test( @@ -54,6 +45,6 @@ if(LINUX) TEST ${EXAMPLE_NAME} PROPERTY ENVIRONMENT_MODIFICATION - "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${JEMALLOC_LIBRARY_DIRS}" + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}" ) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ffd928f7cc..8b1e2248a7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,8 +69,19 @@ set(UMF_SOURCES critnib/critnib.c ravl/ravl.c pool/pool_proxy.c + pool/pool_jemalloc.c pool/pool_scalable.c) +if(UMF_POOL_JEMALLOC_ENABLED) + set(UMF_LIBS ${UMF_LIBS} ${JEMALLOC_LIBRARIES}) + set(UMF_PRIVATE_LIBRARY_DIRS ${UMF_PRIVATE_LIBRARY_DIRS} + ${JEMALLOC_LIBRARY_DIRS}) + set(UMF_PRIVATE_INCLUDE_DIRS ${UMF_PRIVATE_INCLUDE_DIRS} + ${JEMALLOC_INCLUDE_DIRS}) + set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + "UMF_POOL_JEMALLOC_ENABLED=1") +endif() + if(NOT UMF_DISABLE_HWLOC) set(UMF_SOURCES ${UMF_SOURCES} ${HWLOC_DEPENDENT_SOURCES} memtargets/memtarget_numa.c) @@ -146,15 +157,19 @@ else() LIBS ${UMF_LIBS}) endif() +target_include_directories(umf PRIVATE ${UMF_PRIVATE_INCLUDE_DIRS}) +target_link_directories(umf PRIVATE ${UMF_PRIVATE_LIBRARY_DIRS}) +target_compile_definitions(umf PRIVATE ${UMF_COMMON_COMPILE_DEFINITIONS}) + add_dependencies(umf coarse) if(UMF_LINK_HWLOC_STATICALLY) add_dependencies(umf ${UMF_HWLOC_NAME}) endif() -target_link_directories(umf PRIVATE ${UMF_PRIVATE_LIBRARY_DIRS}) - -target_compile_definitions(umf PRIVATE ${UMF_COMMON_COMPILE_DEFINITIONS}) +if(NOT WINDOWS AND UMF_POOL_JEMALLOC_ENABLED) + add_dependencies(umf jemalloc) +endif() if(UMF_BUILD_LEVEL_ZERO_PROVIDER) if(LINUX) diff --git a/src/libumf.def b/src/libumf.def index 0b4588bb81..f2b24be6c2 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -36,6 +36,10 @@ EXPORTS umfFileMemoryProviderParamsSetVisibility umfGetIPCHandle umfGetLastFailedMemoryProvider + umfJemallocPoolOps + umfJemallocPoolParamsCreate + umfJemallocPoolParamsDestroy + umfJemallocPoolParamsSetKeepAllMemory umfLevelZeroMemoryProviderOps umfLevelZeroMemoryProviderParamsCreate umfLevelZeroMemoryProviderParamsDestroy diff --git a/src/libumf.map b/src/libumf.map index 41467bad59..067ec88386 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -30,6 +30,10 @@ UMF_1.0 { umfFileMemoryProviderParamsSetVisibility; umfGetIPCHandle; umfGetLastFailedMemoryProvider; + umfJemallocPoolOps; + umfJemallocPoolParamsCreate; + umfJemallocPoolParamsDestroy; + umfJemallocPoolParamsSetKeepAllMemory; umfLevelZeroMemoryProviderOps; umfLevelZeroMemoryProviderParamsCreate; umfLevelZeroMemoryProviderParamsDestroy; diff --git a/src/pool/CMakeLists.txt b/src/pool/CMakeLists.txt index 9cf9d16654..17be932a45 100644 --- a/src/pool/CMakeLists.txt +++ b/src/pool/CMakeLists.txt @@ -38,20 +38,3 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT) install(TARGETS disjoint_pool EXPORT ${PROJECT_NAME}-targets) endif() - -# libumf_pool_jemalloc -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) - add_umf_library( - NAME jemalloc_pool - TYPE STATIC - SRCS pool_jemalloc.c ${POOL_EXTRA_SRCS} - LIBS ${JEMALLOC_LIBRARIES} ${POOL_EXTRA_LIBS}) - target_include_directories(jemalloc_pool PRIVATE ${JEMALLOC_INCLUDE_DIRS}) - target_compile_definitions(jemalloc_pool - PRIVATE ${POOL_COMPILE_DEFINITIONS}) - add_library(${PROJECT_NAME}::jemalloc_pool ALIAS jemalloc_pool) - if(NOT WINDOWS) - add_dependencies(jemalloc_pool jemalloc) - endif() - install(TARGETS jemalloc_pool EXPORT ${PROJECT_NAME}-targets) -endif() diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index 94fd655cc4..47bc6497fc 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -20,6 +20,32 @@ #include #include +#ifndef UMF_POOL_JEMALLOC_ENABLED + +umf_memory_pool_ops_t *umfJemallocPoolOps(void) { return NULL; } + +umf_result_t +umfJemallocPoolParamsCreate(umf_jemalloc_pool_params_handle_t *hParams) { + (void)hParams; // unused + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t +umfJemallocPoolParamsDestroy(umf_jemalloc_pool_params_handle_t hParams) { + (void)hParams; // unused + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t +umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, + bool keepAllMemory) { + (void)hParams; // unused + (void)keepAllMemory; // unused + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +#else + #include #define MALLOCX_ARENA_MAX (MALLCTL_ARENAS_ALL - 1) @@ -535,3 +561,4 @@ static umf_memory_pool_ops_t UMF_JEMALLOC_POOL_OPS = { umf_memory_pool_ops_t *umfJemallocPoolOps(void) { return &UMF_JEMALLOC_POOL_OPS; } +#endif /* UMF_POOL_JEMALLOC_ENABLED */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b54822b962..58584a4e1b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -165,10 +165,6 @@ if(UMF_BUILD_SHARED_LIBRARY) endif() endif() -if(UMF_POOL_JEMALLOC_ENABLED) - set(LIB_JEMALLOC_POOL jemalloc_pool) -endif() - if(UMF_BUILD_LIBUMF_POOL_DISJOINT) set(LIB_DISJOINT_POOL disjoint_pool) endif() @@ -236,14 +232,15 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT add_umf_test( NAME c_api_multi_pool SRCS c_api/multi_pool.c - LIBS disjoint_pool jemalloc_pool ${JEMALLOC_LIBRARIES}) + LIBS disjoint_pool) endif() if(UMF_POOL_JEMALLOC_ENABLED AND (NOT UMF_DISABLE_HWLOC)) add_umf_test( NAME jemalloc_pool SRCS pools/jemalloc_pool.cpp malloc_compliance_tests.cpp - LIBS jemalloc_pool) + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) endif() if(UMF_POOL_SCALABLE_ENABLED AND (NOT UMF_DISABLE_HWLOC)) @@ -266,7 +263,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_os_memory SRCS provider_os_memory.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL} ${LIB_DISJOINT_POOL}) + LIBS ${UMF_UTILS_FOR_TEST} ${LIB_DISJOINT_POOL}) add_umf_test( NAME provider_os_memory_multiple_numa_nodes SRCS provider_os_memory_multiple_numa_nodes.cpp @@ -314,7 +311,7 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_devdax_memory_ipc SRCS provider_devdax_memory_ipc.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) + LIBS ${UMF_UTILS_FOR_TEST}) add_umf_test( NAME provider_file_memory SRCS provider_file_memory.cpp @@ -322,18 +319,20 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented add_umf_test( NAME provider_file_memory_ipc SRCS provider_file_memory_ipc.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST} ${LIB_JEMALLOC_POOL}) + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) add_umf_test( NAME jemalloc_coarse_file SRCS pools/jemalloc_coarse_file.cpp malloc_compliance_tests.cpp - LIBS jemalloc_pool) + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) add_umf_test( NAME jemalloc_coarse_devdax SRCS pools/jemalloc_coarse_devdax.cpp malloc_compliance_tests.cpp - LIBS jemalloc_pool) + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) endif() # This test requires Linux-only file memory provider @@ -739,7 +738,7 @@ if(LINUX else() message( STATUS - "The dram_and_fsdax example is supported on Linux only and requires UMF_BUILD_LIBUMF_POOL_JEMALLOC to be turned ON - skipping" + "The dram_and_fsdax example is supported on Linux only and requires the jemalloc pool, but it is disabled - skipping" ) endif() diff --git a/test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp b/test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp new file mode 100644 index 0000000000..f719032779 --- /dev/null +++ b/test/supp/memcheck-umf_test-jemalloc_coarse_devdax.supp @@ -0,0 +1,7 @@ +{ + False-positive invalid write of size 8 + Memcheck:Addr8 + ... + fun:je_* + ... +} diff --git a/test/supp/memcheck-umf_test-jemalloc_coarse_file.supp b/test/supp/memcheck-umf_test-jemalloc_coarse_file.supp new file mode 100644 index 0000000000..f719032779 --- /dev/null +++ b/test/supp/memcheck-umf_test-jemalloc_coarse_file.supp @@ -0,0 +1,7 @@ +{ + False-positive invalid write of size 8 + Memcheck:Addr8 + ... + fun:je_* + ... +} diff --git a/test/test_installation.py b/test/test_installation.py index 49a382969c..b5dd676dc1 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -283,11 +283,6 @@ def parse_arguments(self) -> argparse.Namespace: action="store_true", help="Add this argument if the UMF was built with Disjoint Pool enabled", ) - self.parser.add_argument( - "--jemalloc-pool", - action="store_true", - help="Add this argument if the UMF was built with Jemalloc Pool enabled", - ) self.parser.add_argument( "--umf-version", action="store", @@ -306,8 +301,6 @@ def run(self) -> None: pools = [] if self.args.disjoint_pool: pools.append("disjoint_pool") - if self.args.jemalloc_pool: - pools.append("jemalloc_pool") umf_version = Version(self.args.umf_version) From 6b005c8cdd118343e5c985ebbc0d9e9f67bc69fe Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 10:37:00 +0100 Subject: [PATCH 19/82] Remove JEMALLOC_LIBRARY_DIRS from tests and benchmarks Signed-off-by: Lukasz Dorau --- benchmark/CMakeLists.txt | 3 --- test/CMakeLists.txt | 1 - 2 files changed, 4 deletions(-) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index b2f1299be3..efad0baf36 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -121,9 +121,6 @@ set(LIB_DIRS ${LIBHWLOC_LIBRARY_DIRS}) if(UMF_BUILD_LIBUMF_POOL_DISJOINT) set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool) endif() -if(UMF_POOL_JEMALLOC_ENABLED) - set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS}) -endif() if(LINUX) set(LIBS_OPTIONAL ${LIBS_OPTIONAL} m) endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 58584a4e1b..b564789706 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -51,7 +51,6 @@ function(build_umf_test) endif() if(UMF_POOL_JEMALLOC_ENABLED) - set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS}) set(CPL_DEFS ${CPL_DEFS} UMF_POOL_JEMALLOC_ENABLED=1) endif() From d8b63dddde1761902f4fe3970612c6b5f73e370d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 10:37:56 +0100 Subject: [PATCH 20/82] Revert WA for the issue with jemalloc in the proxy library This reverts commit a4fced635067affb6787da41116ae3ffadfc5597. Fixes: #894 Signed-off-by: Lukasz Dorau --- src/proxy_lib/proxy_lib.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/proxy_lib/proxy_lib.c b/src/proxy_lib/proxy_lib.c index f8bae304d8..15ddfca1bc 100644 --- a/src/proxy_lib/proxy_lib.c +++ b/src/proxy_lib/proxy_lib.c @@ -128,13 +128,6 @@ static umf_memory_pool_handle_t Proxy_pool = NULL; // it protects us from recursion in umfPool*() static __TLS int was_called_from_umfPool = 0; -// This WA for the issue: -// https://github.com/oneapi-src/unified-memory-framework/issues/894 -// It protects us from a recursion in malloc_usable_size() -// when the JEMALLOC proxy_lib_pool is used. -// TODO remove this WA when the issue is fixed. -static __TLS int was_called_from_malloc_usable_size = 0; - /*****************************************************************************/ /*** The constructor and destructor of the proxy library *********************/ /*****************************************************************************/ @@ -478,18 +471,15 @@ size_t malloc_usable_size(void *ptr) { return 0; // unsupported in case of the ba_leak allocator } - if (!was_called_from_malloc_usable_size && Proxy_pool && - (umfPoolByPtr(ptr) == Proxy_pool)) { - was_called_from_malloc_usable_size = 1; + if (Proxy_pool && (umfPoolByPtr(ptr) == Proxy_pool)) { was_called_from_umfPool = 1; size_t size = umfPoolMallocUsableSize(Proxy_pool, ptr); was_called_from_umfPool = 0; - was_called_from_malloc_usable_size = 0; return size; } #ifndef _WIN32 - if (!was_called_from_malloc_usable_size && Size_threshold_value) { + if (Size_threshold_value) { return System_malloc_usable_size(ptr); } #endif /* _WIN32 */ From 3112e2b4f1d894a10513712dc020cf5884f269c6 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 11:09:40 +0100 Subject: [PATCH 21/82] Revert moving free() to optional (ext) provider ops This reverts commit b0bfbb7e92b76cc56c3bcd91c7ac063467136376. Remove umfDefaultFree() and umfIsFreeOpDefault(). Remove the `disable_upstream_provider_free` parameter of the Coarse provider. Remove the `upstreamDoesNotFree` argument of the `umfTrackingMemoryProviderCreate()` function. Signed-off-by: Lukasz Dorau --- .../custom_file_provider.c | 2 +- include/umf/memory_provider_ops.h | 18 +++++++-------- src/cpp_helpers.hpp | 2 +- src/memory_pool.c | 5 +---- src/memory_provider.c | 19 ++-------------- src/memory_provider_internal.h | 1 - src/provider/provider_coarse.c | 22 ++++--------------- src/provider/provider_cuda.c | 2 +- src/provider/provider_devdax_memory.c | 2 +- src/provider/provider_file_memory.c | 2 +- src/provider/provider_level_zero.c | 2 +- src/provider/provider_os_memory.c | 2 +- src/provider/provider_tracking.c | 21 ++++++------------ src/provider/provider_tracking.h | 2 +- test/common/provider_null.c | 2 +- test/common/provider_trace.c | 2 +- test/memoryProviderAPI.cpp | 21 +++++++----------- 17 files changed, 41 insertions(+), 86 deletions(-) diff --git a/examples/custom_file_provider/custom_file_provider.c b/examples/custom_file_provider/custom_file_provider.c index ffa61d63f5..ad897fe5ea 100644 --- a/examples/custom_file_provider/custom_file_provider.c +++ b/examples/custom_file_provider/custom_file_provider.c @@ -237,11 +237,11 @@ static umf_memory_provider_ops_t file_ops = { .initialize = file_init, .finalize = file_deinit, .alloc = file_alloc, + .free = file_free, .get_name = file_get_name, .get_last_native_error = file_get_last_native_error, .get_recommended_page_size = file_get_recommended_page_size, .get_min_page_size = file_get_min_page_size, - .ext.free = file_free, }; // Main function diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 0b9c7cfce3..a61e0aad07 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -22,15 +22,6 @@ extern "C" { /// can keep them NULL. /// typedef struct umf_memory_provider_ext_ops_t { - /// - /// @brief Frees the memory space pointed by \p ptr from the memory \p provider - /// @param provider pointer to the memory provider - /// @param ptr pointer to the allocated memory to free - /// @param size size of the allocation - /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure - /// - umf_result_t (*free)(void *provider, void *ptr, size_t size); - /// /// @brief Discard physical pages within the virtual memory mapping associated at the given addr /// and \p size. This call is asynchronous and may delay purging the pages indefinitely. @@ -181,6 +172,15 @@ typedef struct umf_memory_provider_ops_t { umf_result_t (*alloc)(void *provider, size_t size, size_t alignment, void **ptr); + /// + /// @brief Frees the memory space pointed by \p ptr from the memory \p provider + /// @param provider pointer to the memory provider + /// @param ptr pointer to the allocated memory to free + /// @param size size of the allocation + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure + /// + umf_result_t (*free)(void *provider, void *ptr, size_t size); + /// /// @brief Retrieve string representation of the underlying provider specific /// result reported by the last API that returned diff --git a/src/cpp_helpers.hpp b/src/cpp_helpers.hpp index 6316ccbc7e..8789105814 100644 --- a/src/cpp_helpers.hpp +++ b/src/cpp_helpers.hpp @@ -84,7 +84,7 @@ template constexpr umf_memory_provider_ops_t providerOpsBase() { ops.version = UMF_VERSION_CURRENT; ops.finalize = [](void *obj) { delete reinterpret_cast(obj); }; UMF_ASSIGN_OP(ops, T, alloc, UMF_RESULT_ERROR_UNKNOWN); - UMF_ASSIGN_OP(ops.ext, T, free, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, free, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP_NORETURN(ops, T, get_last_native_error); UMF_ASSIGN_OP(ops, T, get_recommended_page_size, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, get_min_page_size, UMF_RESULT_ERROR_UNKNOWN); diff --git a/src/memory_pool.c b/src/memory_pool.c index 4a85955efa..cb1d303f5f 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -42,10 +42,7 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, if (!(flags & UMF_POOL_CREATE_FLAG_DISABLE_TRACKING)) { // Wrap provider with memory tracking provider. - // Check if the provider supports the free() operation. - bool upstreamDoesNotFree = umfIsFreeOpDefault(provider); - ret = umfTrackingMemoryProviderCreate(provider, pool, &pool->provider, - upstreamDoesNotFree); + ret = umfTrackingMemoryProviderCreate(provider, pool, &pool->provider); if (ret != UMF_RESULT_SUCCESS) { goto err_provider_create; } diff --git a/src/memory_provider.c b/src/memory_provider.c index 883f1be263..59f3f12592 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -25,13 +25,6 @@ typedef struct umf_memory_provider_t { void *provider_priv; } umf_memory_provider_t; -static umf_result_t umfDefaultFree(void *provider, void *ptr, size_t size) { - (void)provider; - (void)ptr; - (void)size; - return UMF_RESULT_ERROR_NOT_SUPPORTED; -} - static umf_result_t umfDefaultPurgeLazy(void *provider, void *ptr, size_t size) { (void)provider; @@ -106,9 +99,6 @@ static umf_result_t umfDefaultCloseIPCHandle(void *provider, void *ptr, } void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { - if (!ops->ext.free) { - ops->ext.free = umfDefaultFree; - } if (!ops->ext.purge_lazy) { ops->ext.purge_lazy = umfDefaultPurgeLazy; } @@ -143,7 +133,7 @@ void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { static bool validateOpsMandatory(const umf_memory_provider_ops_t *ops) { // Mandatory ops should be non-NULL - return ops->alloc && ops->get_recommended_page_size && + return ops->alloc && ops->free && ops->get_recommended_page_size && ops->get_min_page_size && ops->initialize && ops->finalize && ops->get_last_native_error && ops->get_name; } @@ -169,10 +159,6 @@ static bool validateOps(const umf_memory_provider_ops_t *ops) { validateOpsIpc(&(ops->ipc)); } -bool umfIsFreeOpDefault(umf_memory_provider_handle_t hProvider) { - return (hProvider->ops.ext.free == umfDefaultFree); -} - umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, void *params, umf_memory_provider_handle_t *hProvider) { @@ -236,8 +222,7 @@ umf_result_t umfMemoryProviderAlloc(umf_memory_provider_handle_t hProvider, umf_result_t umfMemoryProviderFree(umf_memory_provider_handle_t hProvider, void *ptr, size_t size) { UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf_result_t res = - hProvider->ops.ext.free(hProvider->provider_priv, ptr, size); + umf_result_t res = hProvider->ops.free(hProvider->provider_priv, ptr, size); checkErrorAndSetLastProvider(res, hProvider); return res; } diff --git a/src/memory_provider_internal.h b/src/memory_provider_internal.h index 49b2f2e531..60955e0fbd 100644 --- a/src/memory_provider_internal.h +++ b/src/memory_provider_internal.h @@ -20,7 +20,6 @@ extern "C" { void *umfMemoryProviderGetPriv(umf_memory_provider_handle_t hProvider); umf_memory_provider_handle_t *umfGetLastFailedMemoryProviderPtr(void); -bool umfIsFreeOpDefault(umf_memory_provider_handle_t hProvider); #ifdef __cplusplus } diff --git a/src/provider/provider_coarse.c b/src/provider/provider_coarse.c index c3027b91d7..72985faaf2 100644 --- a/src/provider/provider_coarse.c +++ b/src/provider/provider_coarse.c @@ -59,10 +59,6 @@ typedef struct coarse_memory_provider_t { // "coarse ()" // for example: "coarse (L0)" char *name; - - // Set to true if the free() operation of the upstream memory provider is not supported - // (i.e. if (umfMemoryProviderFree(upstream_memory_provider, NULL, 0) == UMF_RESULT_ERROR_NOT_SUPPORTED) - bool disable_upstream_provider_free; } coarse_memory_provider_t; typedef struct ravl_node ravl_node_t; @@ -918,13 +914,6 @@ static umf_result_t coarse_memory_provider_initialize(void *params, coarse_provider->allocation_strategy = coarse_params->allocation_strategy; coarse_provider->init_buffer = coarse_params->init_buffer; - if (coarse_provider->upstream_memory_provider) { - coarse_provider->disable_upstream_provider_free = - umfIsFreeOpDefault(coarse_provider->upstream_memory_provider); - } else { - coarse_provider->disable_upstream_provider_free = false; - } - umf_result_t umf_result = coarse_memory_provider_set_name(coarse_provider); if (umf_result != UMF_RESULT_SUCCESS) { LOG_ERR("name initialization failed"); @@ -1027,8 +1016,7 @@ static void coarse_ravl_cb_rm_upstream_blocks_node(void *data, void *arg) { block_t *alloc = node_data->value; assert(alloc); - if (coarse_provider->upstream_memory_provider && - !coarse_provider->disable_upstream_provider_free) { + if (coarse_provider->upstream_memory_provider) { // We continue to deallocate alloc blocks even if the upstream provider doesn't return success. umfMemoryProviderFree(coarse_provider->upstream_memory_provider, alloc->data, alloc->size); @@ -1288,10 +1276,8 @@ static umf_result_t coarse_memory_provider_alloc(void *provider, size_t size, umf_result = coarse_add_upstream_block(coarse_provider, *resultPtr, size); if (umf_result != UMF_RESULT_SUCCESS) { - if (!coarse_provider->disable_upstream_provider_free) { - umfMemoryProviderFree(coarse_provider->upstream_memory_provider, - *resultPtr, size); - } + umfMemoryProviderFree(coarse_provider->upstream_memory_provider, + *resultPtr, size); goto err_unlock; } @@ -1657,12 +1643,12 @@ umf_memory_provider_ops_t UMF_COARSE_MEMORY_PROVIDER_OPS = { .initialize = coarse_memory_provider_initialize, .finalize = coarse_memory_provider_finalize, .alloc = coarse_memory_provider_alloc, + .free = coarse_memory_provider_free, .get_last_native_error = coarse_memory_provider_get_last_native_error, .get_recommended_page_size = coarse_memory_provider_get_recommended_page_size, .get_min_page_size = coarse_memory_provider_get_min_page_size, .get_name = coarse_memory_provider_get_name, - .ext.free = coarse_memory_provider_free, .ext.purge_lazy = coarse_memory_provider_purge_lazy, .ext.purge_force = coarse_memory_provider_purge_force, .ext.allocation_merge = coarse_memory_provider_allocation_merge, diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index baccbd0235..f46e049723 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -599,11 +599,11 @@ static struct umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .initialize = cu_memory_provider_initialize, .finalize = cu_memory_provider_finalize, .alloc = cu_memory_provider_alloc, + .free = cu_memory_provider_free, .get_last_native_error = cu_memory_provider_get_last_native_error, .get_recommended_page_size = cu_memory_provider_get_recommended_page_size, .get_min_page_size = cu_memory_provider_get_min_page_size, .get_name = cu_memory_provider_get_name, - .ext.free = cu_memory_provider_free, // TODO /* .ext.purge_lazy = cu_memory_provider_purge_lazy, diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index 463b796ec0..79a0662754 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -527,11 +527,11 @@ static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .initialize = devdax_initialize, .finalize = devdax_finalize, .alloc = devdax_alloc, + .free = devdax_free, .get_last_native_error = devdax_get_last_native_error, .get_recommended_page_size = devdax_get_recommended_page_size, .get_min_page_size = devdax_get_min_page_size, .get_name = devdax_get_name, - .ext.free = devdax_free, .ext.purge_lazy = devdax_purge_lazy, .ext.purge_force = devdax_purge_force, .ext.allocation_merge = devdax_allocation_merge, diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 558b1062a5..edf733180b 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -825,11 +825,11 @@ static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .initialize = file_initialize, .finalize = file_finalize, .alloc = file_alloc, + .free = file_free, .get_last_native_error = file_get_last_native_error, .get_recommended_page_size = file_get_recommended_page_size, .get_min_page_size = file_get_min_page_size, .get_name = file_get_name, - .ext.free = file_free, .ext.purge_lazy = file_purge_lazy, .ext.purge_force = file_purge_force, .ext.allocation_merge = file_allocation_merge, diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index f4a3e97c27..70f0acfe54 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -682,11 +682,11 @@ static struct umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .initialize = ze_memory_provider_initialize, .finalize = ze_memory_provider_finalize, .alloc = ze_memory_provider_alloc, + .free = ze_memory_provider_free, .get_last_native_error = ze_memory_provider_get_last_native_error, .get_recommended_page_size = ze_memory_provider_get_recommended_page_size, .get_min_page_size = ze_memory_provider_get_min_page_size, .get_name = ze_memory_provider_get_name, - .ext.free = ze_memory_provider_free, .ext.purge_lazy = ze_memory_provider_purge_lazy, .ext.purge_force = ze_memory_provider_purge_force, .ext.allocation_merge = ze_memory_provider_allocation_merge, diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 4c19944a96..2cc8e9827c 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1408,11 +1408,11 @@ static umf_memory_provider_ops_t UMF_OS_MEMORY_PROVIDER_OPS = { .initialize = os_initialize, .finalize = os_finalize, .alloc = os_alloc, + .free = os_free, .get_last_native_error = os_get_last_native_error, .get_recommended_page_size = os_get_recommended_page_size, .get_min_page_size = os_get_min_page_size, .get_name = os_get_name, - .ext.free = os_free, .ext.purge_lazy = os_purge_lazy, .ext.purge_force = os_purge_force, .ext.allocation_merge = os_allocation_merge, diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index e726feefb0..c4fff4133b 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -154,9 +154,6 @@ typedef struct umf_tracking_memory_provider_t { umf_memory_pool_handle_t pool; critnib *ipcCache; ipc_mapped_handle_cache_handle_t hIpcMappedCache; - - // the upstream provider does not support the free() operation - bool upstreamDoesNotFree; } umf_tracking_memory_provider_t; typedef struct umf_tracking_memory_provider_t umf_tracking_memory_provider_t; @@ -422,8 +419,7 @@ static umf_result_t trackingInitialize(void *params, void **ret) { // TODO clearing the tracker is a temporary solution and should be removed. // The tracker should be cleared using the provider's free() operation. static void clear_tracker_for_the_pool(umf_memory_tracker_handle_t hTracker, - umf_memory_pool_handle_t pool, - bool upstreamDoesNotFree) { + umf_memory_pool_handle_t pool) { uintptr_t rkey; void *rvalue; size_t n_items = 0; @@ -448,7 +444,7 @@ static void clear_tracker_for_the_pool(umf_memory_tracker_handle_t hTracker, #ifndef NDEBUG // print error messages only if provider supports the free() operation - if (n_items && !upstreamDoesNotFree) { + if (n_items) { if (pool) { LOG_ERR( "tracking provider of pool %p is not empty! (%zu items left)", @@ -459,13 +455,12 @@ static void clear_tracker_for_the_pool(umf_memory_tracker_handle_t hTracker, } } #else /* DEBUG */ - (void)upstreamDoesNotFree; // unused in DEBUG build - (void)n_items; // unused in DEBUG build + (void)n_items; // unused in DEBUG build #endif /* DEBUG */ } static void clear_tracker(umf_memory_tracker_handle_t hTracker) { - clear_tracker_for_the_pool(hTracker, NULL, false); + clear_tracker_for_the_pool(hTracker, NULL); } static void trackingFinalize(void *provider) { @@ -480,8 +475,7 @@ static void trackingFinalize(void *provider) { // because it may need those resources till // the very end of exiting the application. if (!utils_is_running_in_proxy_lib()) { - clear_tracker_for_the_pool(p->hTracker, p->pool, - p->upstreamDoesNotFree); + clear_tracker_for_the_pool(p->hTracker, p->pool); } umf_ba_global_free(provider); @@ -760,11 +754,11 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .initialize = trackingInitialize, .finalize = trackingFinalize, .alloc = trackingAlloc, + .free = trackingFree, .get_last_native_error = trackingGetLastError, .get_min_page_size = trackingGetMinPageSize, .get_recommended_page_size = trackingGetRecommendedPageSize, .get_name = trackingName, - .ext.free = trackingFree, .ext.purge_force = trackingPurgeForce, .ext.purge_lazy = trackingPurgeLazy, .ext.allocation_split = trackingAllocationSplit, @@ -777,11 +771,10 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { umf_result_t umfTrackingMemoryProviderCreate( umf_memory_provider_handle_t hUpstream, umf_memory_pool_handle_t hPool, - umf_memory_provider_handle_t *hTrackingProvider, bool upstreamDoesNotFree) { + umf_memory_provider_handle_t *hTrackingProvider) { umf_tracking_memory_provider_t params; params.hUpstream = hUpstream; - params.upstreamDoesNotFree = upstreamDoesNotFree; params.hTracker = TRACKER; if (!params.hTracker) { LOG_ERR("failed, TRACKER is NULL"); diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index 9444ee4757..2abc365051 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -54,7 +54,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, // forwards all requests to hUpstream memory Provider. hUpstream lifetime should be managed by the user of this function. umf_result_t umfTrackingMemoryProviderCreate( umf_memory_provider_handle_t hUpstream, umf_memory_pool_handle_t hPool, - umf_memory_provider_handle_t *hTrackingProvider, bool upstreamDoesNotFree); + umf_memory_provider_handle_t *hTrackingProvider); void umfTrackingMemoryProviderGetUpstreamProvider( umf_memory_provider_handle_t hTrackingProvider, diff --git a/test/common/provider_null.c b/test/common/provider_null.c index 5db389e895..e667bfce40 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -134,11 +134,11 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .initialize = nullInitialize, .finalize = nullFinalize, .alloc = nullAlloc, + .free = nullFree, .get_last_native_error = nullGetLastError, .get_recommended_page_size = nullGetRecommendedPageSize, .get_min_page_size = nullGetPageSize, .get_name = nullName, - .ext.free = nullFree, .ext.purge_lazy = nullPurgeLazy, .ext.purge_force = nullPurgeForce, .ext.allocation_merge = nullAllocationMerge, diff --git a/test/common/provider_trace.c b/test/common/provider_trace.c index 219dde5cd7..9d063b4f56 100644 --- a/test/common/provider_trace.c +++ b/test/common/provider_trace.c @@ -195,11 +195,11 @@ umf_memory_provider_ops_t UMF_TRACE_PROVIDER_OPS = { .initialize = traceInitialize, .finalize = traceFinalize, .alloc = traceAlloc, + .free = traceFree, .get_last_native_error = traceGetLastError, .get_recommended_page_size = traceGetRecommendedPageSize, .get_min_page_size = traceGetPageSize, .get_name = traceName, - .ext.free = traceFree, .ext.purge_lazy = tracePurgeLazy, .ext.purge_force = tracePurgeForce, .ext.allocation_merge = traceAllocationMerge, diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 866ae6dae3..2dc7261f06 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -89,19 +89,6 @@ TEST_F(test, memoryProviderTrace) { ASSERT_EQ(calls.size(), ++call_count); } -TEST_F(test, memoryProviderOpsNullFreeField) { - umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; - provider_ops.ext.free = nullptr; - umf_memory_provider_handle_t hProvider; - auto ret = umfMemoryProviderCreate(&provider_ops, nullptr, &hProvider); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - ret = umfMemoryProviderFree(hProvider, nullptr, 0); - ASSERT_EQ(ret, UMF_RESULT_ERROR_NOT_SUPPORTED); - - umfMemoryProviderDestroy(hProvider); -} - TEST_F(test, memoryProviderOpsNullPurgeLazyField) { umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; provider_ops.ext.purge_lazy = nullptr; @@ -204,6 +191,14 @@ TEST_F(test, memoryProviderOpsNullAllocField) { ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); } +TEST_F(test, memoryProviderOpsNullFreeField) { + umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; + provider_ops.free = nullptr; + umf_memory_provider_handle_t hProvider; + auto ret = umfMemoryProviderCreate(&provider_ops, nullptr, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + TEST_F(test, memoryProviderOpsNullGetLastNativeErrorField) { umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; provider_ops.get_last_native_error = nullptr; From 35332da8007882d97e3aeda8d58a4f9ac44c2fea Mon Sep 17 00:00:00 2001 From: "Vinogradov, Sergei" Date: Thu, 5 Dec 2024 11:29:15 +0100 Subject: [PATCH 22/82] Minor fixes to clean up resources in tests --- test/pools/disjoint_pool.cpp | 4 ++++ test/provider_os_memory.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 319997c823..471e53dc2c 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -29,20 +29,24 @@ disjoint_params_unique_handle_t poolConfig() { res = umfDisjointPoolParamsSetSlabMinSize(config, DEFAULT_DISJOINT_SLAB_MIN_SIZE); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set slab min size"); } res = umfDisjointPoolParamsSetMaxPoolableSize( config, DEFAULT_DISJOINT_MAX_POOLABLE_SIZE); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set max poolable size"); } res = umfDisjointPoolParamsSetCapacity(config, DEFAULT_DISJOINT_CAPACITY); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set capacity"); } res = umfDisjointPoolParamsSetMinBucketSize( config, DEFAULT_DISJOINT_MIN_BUCKET_SIZE); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(config); throw std::runtime_error("Failed to set min bucket size"); } diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 687db08053..4c81b84f9e 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -441,18 +441,22 @@ disjoint_params_unique_handle_t disjointPoolParams() { } res = umfDisjointPoolParamsSetSlabMinSize(params, 4096); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set slab min size"); } res = umfDisjointPoolParamsSetMaxPoolableSize(params, 4096); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set max poolable size"); } res = umfDisjointPoolParamsSetCapacity(params, 4); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set capacity"); } res = umfDisjointPoolParamsSetMinBucketSize(params, 64); if (res != UMF_RESULT_SUCCESS) { + umfDisjointPoolParamsDestroy(params); throw std::runtime_error("Failed to set min bucket size"); } From f311aee1c39c3717b739ec2bc8a5a508d7c5b437 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 14:55:29 +0100 Subject: [PATCH 23/82] Enable all IPC tests Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_dax.yml | 8 ++------ .github/workflows/reusable_proxy_lib.yml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable_dax.yml b/.github/workflows/reusable_dax.yml index f7c5d0d21a..af15226d29 100644 --- a/.github/workflows/reusable_dax.yml +++ b/.github/workflows/reusable_dax.yml @@ -106,8 +106,6 @@ jobs: UMF_TESTS_FSDAX_PATH_2=${{env.UMF_TESTS_FSDAX_PATH_2}} ctest -C ${{matrix.build_type}} -V -R "file|fsdax" - # TODO: enable the provider_devdax_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run the DEVDAX tests with the proxy library # proxy library is built only if libumf is a shared library if: ${{ matrix.shared_library == 'ON' }} @@ -116,10 +114,8 @@ jobs: LD_PRELOAD=./lib/libumf_proxy.so UMF_TESTS_DEVDAX_PATH="/dev/dax${{env.DEVDAX_NAMESPACE}}" UMF_TESTS_DEVDAX_SIZE="$(ndctl list --namespace=namespace${{env.DEVDAX_NAMESPACE}} | grep size | cut -d':' -f2 | cut -d',' -f1)" - ctest -C ${{matrix.build_type}} -V -R devdax -E provider_devdax_memory_ipc + ctest -C ${{matrix.build_type}} -V -R devdax - # TODO: enable the provider_file_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run the FSDAX tests with the proxy library # proxy library is built only if libumf is a shared library if: ${{ matrix.shared_library == 'ON' }} @@ -128,7 +124,7 @@ jobs: LD_PRELOAD=./lib/libumf_proxy.so UMF_TESTS_FSDAX_PATH=${{env.UMF_TESTS_FSDAX_PATH}} UMF_TESTS_FSDAX_PATH_2=${{env.UMF_TESTS_FSDAX_PATH_2}} - ctest -C ${{matrix.build_type}} -V -R "file|fsdax" -E provider_file_memory_ipc + ctest -C ${{matrix.build_type}} -V -R "file|fsdax" - name: Check coverage if: ${{ matrix.build_type == 'Debug' }} diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index 103c4a5163..e73dabe29f 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -59,11 +59,9 @@ jobs: - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) - # TODO enable the provider_file_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run "ctest --output-on-failure" with proxy library working-directory: ${{env.BUILD_DIR}} - run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure -E provider_file_memory_ipc + run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure - name: Run "./test/umf_test-memoryPool" with proxy library working-directory: ${{env.BUILD_DIR}} @@ -77,14 +75,12 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: UMF_PROXY="page.disposition=shared-shm" LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/date - # TODO enable the provider_file_memory_ipc test when the IPC tests with the proxy library are fixed - # see the issue: https://github.com/oneapi-src/unified-memory-framework/issues/864 - name: Run "ctest --output-on-failure" with proxy library and size.threshold=128 working-directory: ${{env.BUILD_DIR}} run: > UMF_PROXY="page.disposition=shared-shm;size.threshold=128" LD_PRELOAD=./lib/libumf_proxy.so - ctest --output-on-failure -E provider_file_memory_ipc + ctest --output-on-failure - name: Check coverage if: ${{ matrix.build_type == 'Debug' }} From c173e56c494f9091f2f2d9450bfe2df7afd14cbc Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 15:39:46 +0100 Subject: [PATCH 24/82] Fix and add error messages Fix two error messages and add some new ones. Signed-off-by: Lukasz Dorau --- src/provider/provider_file_memory.c | 37 ++++++++++++++++++++++------- src/provider/provider_os_memory.c | 4 +--- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 558b1062a5..14e3b6cb0b 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -485,6 +485,8 @@ static umf_result_t file_alloc_cb(void *provider, size_t size, size_t alignment, file_memory_provider_t *file_provider = (file_memory_provider_t *)provider; + *resultPtr = NULL; + // alignment must be a power of two and a multiple or a divider of the page size if (alignment && ((alignment & (alignment - 1)) || ((alignment % file_provider->page_size) && @@ -517,8 +519,15 @@ static umf_result_t file_alloc_cb(void *provider, size_t size, size_t alignment, LOG_ERR("inserting a value to the file descriptor offset map failed " "(addr=%p, offset=%zu)", addr, alloc_offset_fd); + // We cannot undo the file_alloc_aligned() call here, + // because the file memory provider does not support the free operation. + return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("inserted a value to the file descriptor offset map (addr=%p, " + "offset=%zu)", + addr, alloc_offset_fd); + *resultPtr = addr; return UMF_RESULT_SUCCESS; @@ -623,23 +632,31 @@ static umf_result_t file_allocation_split_cb(void *provider, void *ptr, void *value = critnib_get(file_provider->fd_offset_map, (uintptr_t)ptr); if (value == NULL) { - LOG_ERR("file_allocation_split(): getting a value from the file " - "descriptor offset map failed (addr=%p)", + LOG_ERR("getting a value from the file descriptor offset map failed " + "(addr=%p)", ptr); return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("split the value from the file descriptor offset map (addr=%p) " + "from size %zu to %zu + %zu", + ptr, totalSize, firstSize, totalSize - firstSize); + uintptr_t new_key = (uintptr_t)ptr + firstSize; void *new_value = (void *)((uintptr_t)value + firstSize); int ret = critnib_insert(file_provider->fd_offset_map, new_key, new_value, 0 /* update */); if (ret) { - LOG_ERR("file_allocation_split(): inserting a value to the file " - "descriptor offset map failed (addr=%p, offset=%zu)", + LOG_ERR("inserting a value to the file descriptor offset map failed " + "(addr=%p, offset=%zu)", (void *)new_key, (size_t)new_value - 1); return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("inserted a value to the file descriptor offset map (addr=%p, " + "offset=%zu)", + (void *)new_key, (size_t)new_value - 1); + return UMF_RESULT_SUCCESS; } @@ -662,12 +679,16 @@ static umf_result_t file_allocation_merge_cb(void *provider, void *lowPtr, void *value = critnib_remove(file_provider->fd_offset_map, (uintptr_t)highPtr); if (value == NULL) { - LOG_ERR("file_allocation_merge(): removing a value from the file " - "descriptor offset map failed (addr=%p)", + LOG_ERR("removing a value from the file descriptor offset map failed " + "(addr=%p)", highPtr); return UMF_RESULT_ERROR_UNKNOWN; } + LOG_DEBUG("removed a value from the file descriptor offset map (addr=%p) - " + "merged with %p", + highPtr, lowPtr); + return UMF_RESULT_SUCCESS; } @@ -701,9 +722,7 @@ static umf_result_t file_get_ipc_handle(void *provider, const void *ptr, void *value = critnib_get(file_provider->fd_offset_map, (uintptr_t)ptr); if (value == NULL) { - LOG_ERR("file_get_ipc_handle(): getting a value from the IPC cache " - "failed (addr=%p)", - ptr); + LOG_ERR("getting a value from the IPC cache failed (addr=%p)", ptr); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 4c19944a96..04054c20c5 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1286,9 +1286,7 @@ static umf_result_t os_get_ipc_handle(void *provider, const void *ptr, void *value = critnib_get(os_provider->fd_offset_map, (uintptr_t)ptr); if (value == NULL) { - LOG_ERR("os_get_ipc_handle(): getting a value from the IPC cache " - "failed (addr=%p)", - ptr); + LOG_ERR("getting a value from the IPC cache failed (addr=%p)", ptr); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } From c6749f2b7a92f0d11d7b831d984249e5d3d4d786 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 29 Nov 2024 11:54:44 +0100 Subject: [PATCH 25/82] Remove the Coarse provider Fixes: #900 Signed-off-by: Lukasz Dorau --- include/umf/providers/provider_coarse.h | 116 -- src/CMakeLists.txt | 1 - src/libumf.def | 2 - src/libumf.map | 2 - src/provider/provider_coarse.c | 1693 ----------------------- test/CMakeLists.txt | 15 +- test/disjointCoarseMallocPool.cpp | 317 +---- test/memoryPoolAPI.cpp | 8 +- test/poolFixtures.hpp | 25 +- test/pools/disjoint_pool.cpp | 19 +- test/pools/jemalloc_coarse_devdax.cpp | 10 +- test/pools/jemalloc_coarse_file.cpp | 4 +- test/pools/jemalloc_pool.cpp | 8 +- test/pools/pool_base_alloc.cpp | 2 +- test/pools/pool_coarse.hpp | 2 - test/pools/scalable_coarse_devdax.cpp | 10 +- test/pools/scalable_coarse_file.cpp | 4 +- test/pools/scalable_pool.cpp | 3 +- test/provider_coarse.cpp | 668 --------- 19 files changed, 86 insertions(+), 2823 deletions(-) delete mode 100644 include/umf/providers/provider_coarse.h delete mode 100644 src/provider/provider_coarse.c delete mode 100644 test/provider_coarse.cpp diff --git a/include/umf/providers/provider_coarse.h b/include/umf/providers/provider_coarse.h deleted file mode 100644 index 6ed6e0fbc9..0000000000 --- a/include/umf/providers/provider_coarse.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#ifndef UMF_COARSE_PROVIDER_H -#define UMF_COARSE_PROVIDER_H - -#include -#include - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/// @brief Coarse Memory Provider allocation strategy -typedef enum coarse_memory_provider_strategy_t { - /// Always allocate a free block of the (size + alignment) size - /// and cut out the properly aligned part leaving two remaining parts. - /// It is the fastest strategy but causes memory fragmentation - /// when alignment is greater than 0. - /// It is the best strategy when alignment always equals 0. - UMF_COARSE_MEMORY_STRATEGY_FASTEST = 0, - - /// Check if the first free block of the 'size' size has the correct alignment. - /// If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, - - /// Look through all free blocks of the 'size' size - /// and choose the first one with the correct alignment. - /// If none of them had the correct alignment, - /// use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE, - - /// The maximum value (it has to be the last one). - UMF_COARSE_MEMORY_STRATEGY_MAX -} coarse_memory_provider_strategy_t; - -/// @brief Coarse Memory Provider settings struct. -typedef struct coarse_memory_provider_params_t { - /// Handle to the upstream memory provider. - /// It has to be NULL if init_buffer is set - /// (exactly one of them has to be non-NULL). - umf_memory_provider_handle_t upstream_memory_provider; - - /// Memory allocation strategy. - /// See coarse_memory_provider_strategy_t for details. - coarse_memory_provider_strategy_t allocation_strategy; - - /// A pre-allocated buffer that will be the only memory that - /// the coarse provider can provide (the fixed-size memory provider option). - /// If it is non-NULL, `init_buffer_size ` has to contain its size. - /// It has to be NULL if upstream_memory_provider is set - /// (exactly one of them has to be non-NULL). - void *init_buffer; - - /// Size of the initial buffer: - /// 1) `init_buffer` if it is non-NULL xor - /// 2) that will be allocated from the upstream_memory_provider - /// (if it is non-NULL) in the `.initialize` operation. - size_t init_buffer_size; - - /// When it is true and the upstream_memory_provider is given, - /// the init buffer (of `init_buffer_size` bytes) would be pre-allocated - /// during creation time using the `upstream_memory_provider`. - /// If upstream_memory_provider is not given, - /// the init_buffer is always used instead - /// (regardless of the value of this parameter). - bool immediate_init_from_upstream; - - /// Destroy upstream_memory_provider in finalize(). - bool destroy_upstream_memory_provider; -} coarse_memory_provider_params_t; - -/// @brief Coarse Memory Provider stats (TODO move to CTL) -typedef struct coarse_memory_provider_stats_t { - /// Total allocation size. - size_t alloc_size; - - /// Size of used memory. - size_t used_size; - - /// Number of memory blocks allocated from the upstream provider. - size_t num_upstream_blocks; - - /// Total number of allocated memory blocks. - size_t num_all_blocks; - - /// Number of free memory blocks. - size_t num_free_blocks; -} coarse_memory_provider_stats_t; - -umf_memory_provider_ops_t *umfCoarseMemoryProviderOps(void); - -// TODO use CTL -coarse_memory_provider_stats_t -umfCoarseMemoryProviderGetStats(umf_memory_provider_handle_t provider); - -/// @brief Create default params for the coarse memory provider -static inline coarse_memory_provider_params_t -umfCoarseMemoryProviderParamsDefault(void) { - coarse_memory_provider_params_t coarse_memory_provider_params; - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - return coarse_memory_provider_params; -} - -#ifdef __cplusplus -} -#endif - -#endif // UMF_COARSE_PROVIDER_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8b1e2248a7..fb32b6d2eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -59,7 +59,6 @@ set(UMF_SOURCES memspaces/memspace_highest_bandwidth.c memspaces/memspace_lowest_latency.c memspaces/memspace_numa.c - provider/provider_coarse.c provider/provider_cuda.c provider/provider_devdax_memory.c provider/provider_file_memory.c diff --git a/src/libumf.def b/src/libumf.def index f2b24be6c2..c0cd1c90c7 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -14,8 +14,6 @@ EXPORTS umfTearDown umfGetCurrentVersion umfCloseIPCHandle - umfCoarseMemoryProviderGetStats - umfCoarseMemoryProviderOps umfCUDAMemoryProviderOps umfCUDAMemoryProviderParamsCreate umfCUDAMemoryProviderParamsDestroy diff --git a/src/libumf.map b/src/libumf.map index 067ec88386..8a7bdc81ce 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -8,8 +8,6 @@ UMF_1.0 { umfTearDown; umfGetCurrentVersion; umfCloseIPCHandle; - umfCoarseMemoryProviderGetStats; - umfCoarseMemoryProviderOps; umfCUDAMemoryProviderOps; umfCUDAMemoryProviderParamsCreate; umfCUDAMemoryProviderParamsDestroy; diff --git a/src/provider/provider_coarse.c b/src/provider/provider_coarse.c deleted file mode 100644 index 72985faaf2..0000000000 --- a/src/provider/provider_coarse.c +++ /dev/null @@ -1,1693 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#include -#include -#include -#include -#include -#include - -#include - -#include "base_alloc_global.h" -#include "memory_provider_internal.h" -#include "ravl.h" -#include "utils_common.h" -#include "utils_concurrency.h" -#include "utils_log.h" - -#define COARSE_BASE_NAME "coarse" - -#define IS_ORIGIN_OF_BLOCK(origin, block) \ - (((uintptr_t)(block)->data >= (uintptr_t)(origin)->data) && \ - ((uintptr_t)(block)->data + (block)->size <= \ - (uintptr_t)(origin)->data + (origin)->size)) - -typedef struct coarse_memory_provider_t { - umf_memory_provider_handle_t upstream_memory_provider; - - // destroy upstream_memory_provider in finalize() - bool destroy_upstream_memory_provider; - - // memory allocation strategy - coarse_memory_provider_strategy_t allocation_strategy; - - void *init_buffer; - - size_t used_size; - size_t alloc_size; - - // upstream_blocks - tree of all blocks allocated from the upstream provider - struct ravl *upstream_blocks; - - // all_blocks - tree of all blocks - sorted by an address of data - struct ravl *all_blocks; - - // free_blocks - tree of free blocks - sorted by a size of data, - // each node contains a pointer (ravl_free_blocks_head_t) - // to the head of the list of free blocks of the same size - struct ravl *free_blocks; - - struct utils_mutex_t lock; - - // Name of the provider with the upstream provider: - // "coarse ()" - // for example: "coarse (L0)" - char *name; -} coarse_memory_provider_t; - -typedef struct ravl_node ravl_node_t; - -typedef enum check_free_blocks_t { - CHECK_ONLY_THE_FIRST_BLOCK = 0, - CHECK_ALL_BLOCKS_OF_SIZE, -} check_free_blocks_t; - -typedef struct block_t { - size_t size; - unsigned char *data; - bool used; - - // Node in the list of free blocks of the same size pointing to this block. - // The list is located in the (coarse_provider->free_blocks) RAVL tree. - struct ravl_free_blocks_elem_t *free_list_ptr; -} block_t; - -// A general node in a RAVL tree. -// 1) coarse_provider->all_blocks RAVL tree (tree of all blocks - sorted by an address of data): -// key - pointer (block_t->data) to the beginning of the block data -// value - pointer (block_t) to the block of the allocation -// 2) coarse_provider->free_blocks RAVL tree (tree of free blocks - sorted by a size of data): -// key - size of the allocation (block_t->size) -// value - pointer (ravl_free_blocks_head_t) to the head of the list of free blocks of the same size -typedef struct ravl_data_t { - uintptr_t key; - void *value; -} ravl_data_t; - -// The head of the list of free blocks of the same size. -typedef struct ravl_free_blocks_head_t { - struct ravl_free_blocks_elem_t *head; -} ravl_free_blocks_head_t; - -// The node of the list of free blocks of the same size -typedef struct ravl_free_blocks_elem_t { - struct block_t *block; - struct ravl_free_blocks_elem_t *next; - struct ravl_free_blocks_elem_t *prev; -} ravl_free_blocks_elem_t; - -// The compare function of a RAVL tree -static int coarse_ravl_comp(const void *lhs, const void *rhs) { - const ravl_data_t *lhs_ravl = (const ravl_data_t *)lhs; - const ravl_data_t *rhs_ravl = (const ravl_data_t *)rhs; - - if (lhs_ravl->key < rhs_ravl->key) { - return -1; - } - - if (lhs_ravl->key > rhs_ravl->key) { - return 1; - } - - // lhs_ravl->key == rhs_ravl->key - return 0; -} - -static inline block_t *get_node_block(ravl_node_t *node) { - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - assert(node_data->value); - return node_data->value; -} - -static inline ravl_node_t *get_node_prev(ravl_node_t *node) { - return ravl_node_predecessor(node); -} - -static inline ravl_node_t *get_node_next(ravl_node_t *node) { - return ravl_node_successor(node); -} - -#ifndef NDEBUG -static block_t *get_block_prev(ravl_node_t *node) { - ravl_node_t *ravl_prev = ravl_node_predecessor(node); - if (!ravl_prev) { - return NULL; - } - - return get_node_block(ravl_prev); -} - -static block_t *get_block_next(ravl_node_t *node) { - ravl_node_t *ravl_next = ravl_node_successor(node); - if (!ravl_next) { - return NULL; - } - - return get_node_block(ravl_next); -} -#endif /* NDEBUG */ - -static bool is_same_origin(struct ravl *upstream_blocks, block_t *block1, - block_t *block2) { - ravl_data_t rdata1 = {(uintptr_t)block1->data, NULL}; - ravl_node_t *ravl_origin1 = - ravl_find(upstream_blocks, &rdata1, RAVL_PREDICATE_LESS_EQUAL); - assert(ravl_origin1); - - block_t *origin1 = get_node_block(ravl_origin1); - assert(IS_ORIGIN_OF_BLOCK(origin1, block1)); - - return (IS_ORIGIN_OF_BLOCK(origin1, block2)); -} - -// The functions "coarse_ravl_*" handle lists of blocks: -// - coarse_provider->all_blocks and coarse_provider->upstream_blocks -// sorted by a pointer (block_t->data) to the beginning of the block data. -// -// coarse_ravl_add_new - allocate and add a new block to the tree -// and link this block to the next and the previous one. -static block_t *coarse_ravl_add_new(struct ravl *rtree, unsigned char *data, - size_t size, ravl_node_t **node) { - assert(rtree); - assert(data); - assert(size); - - // TODO add valgrind annotations - block_t *block = umf_ba_global_alloc(sizeof(*block)); - if (block == NULL) { - return NULL; - } - - block->data = data; - block->size = size; - block->free_list_ptr = NULL; - - ravl_data_t rdata = {(uintptr_t)block->data, block}; - assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); - int ret = ravl_emplace_copy(rtree, &rdata); - if (ret) { - umf_ba_global_free(block); - return NULL; - } - - ravl_node_t *new_node = ravl_find(rtree, &rdata, RAVL_PREDICATE_EQUAL); - assert(NULL != new_node); - - if (node) { - *node = new_node; - } - - return block; -} - -// coarse_ravl_find_node - find the node in the tree -static ravl_node_t *coarse_ravl_find_node(struct ravl *rtree, void *ptr) { - ravl_data_t data = {(uintptr_t)ptr, NULL}; - return ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); -} - -// coarse_ravl_rm - remove the block from the tree -static block_t *coarse_ravl_rm(struct ravl *rtree, void *ptr) { - ravl_data_t data = {(uintptr_t)ptr, NULL}; - ravl_node_t *node; - node = ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL); - if (node) { - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - block_t *block = node_data->value; - assert(block); - ravl_remove(rtree, node); - assert(NULL == ravl_find(rtree, &data, RAVL_PREDICATE_EQUAL)); - return block; - } - return NULL; -} - -// The functions "node_list_*" handle lists of free blocks of the same size. -// The heads (ravl_free_blocks_head_t) of those lists are stored in nodes of -// the coarse_provider->free_blocks RAVL tree. -// -// node_list_add - add a free block to the list of free blocks of the same size -static ravl_free_blocks_elem_t * -node_list_add(ravl_free_blocks_head_t *head_node, struct block_t *block) { - assert(head_node); - assert(block); - - ravl_free_blocks_elem_t *node = umf_ba_global_alloc(sizeof(*node)); - if (node == NULL) { - return NULL; - } - - if (head_node->head) { - head_node->head->prev = node; - } - - node->block = block; - node->next = head_node->head; - node->prev = NULL; - head_node->head = node; - - return node; -} - -// node_list_rm - remove the given free block from the list of free blocks of the same size -static block_t *node_list_rm(ravl_free_blocks_head_t *head_node, - ravl_free_blocks_elem_t *node) { - assert(head_node); - assert(node); - - if (!head_node->head) { - return NULL; - } - - if (node == head_node->head) { - assert(node->prev == NULL); - head_node->head = node->next; - } - - ravl_free_blocks_elem_t *node_next = node->next; - ravl_free_blocks_elem_t *node_prev = node->prev; - if (node_next) { - node_next->prev = node_prev; - } - - if (node_prev) { - node_prev->next = node_next; - } - - struct block_t *block = node->block; - block->free_list_ptr = NULL; - umf_ba_global_free(node); - - return block; -} - -// node_list_rm_first - remove the first free block from the list of free blocks of the same size only if it can be properly aligned -static block_t *node_list_rm_first(ravl_free_blocks_head_t *head_node, - size_t alignment) { - assert(head_node); - - if (!head_node->head) { - return NULL; - } - - ravl_free_blocks_elem_t *node = head_node->head; - assert(node->prev == NULL); - struct block_t *block = node->block; - - if (IS_NOT_ALIGNED(block->size, alignment)) { - return NULL; - } - - if (node->next) { - node->next->prev = NULL; - } - - head_node->head = node->next; - block->free_list_ptr = NULL; - umf_ba_global_free(node); - - return block; -} - -// node_list_rm_with_alignment - remove the first free block with the correct alignment from the list of free blocks of the same size -static block_t *node_list_rm_with_alignment(ravl_free_blocks_head_t *head_node, - size_t alignment) { - assert(head_node); - - if (!head_node->head) { - return NULL; - } - - assert(((ravl_free_blocks_elem_t *)head_node->head)->prev == NULL); - - ravl_free_blocks_elem_t *node; - for (node = head_node->head; node != NULL; node = node->next) { - if (IS_ALIGNED(node->block->size, alignment)) { - return node_list_rm(head_node, node); - } - } - - return NULL; -} - -// The functions "free_blocks_*" handle the coarse_provider->free_blocks RAVL tree -// sorted by a size of the allocation (block_t->size). -// This is a tree of heads (ravl_free_blocks_head_t) of lists of free blocks of the same size. -// -// free_blocks_add - add a free block to the list of free blocks of the same size -static int free_blocks_add(struct ravl *free_blocks, block_t *block) { - ravl_free_blocks_head_t *head_node = NULL; - int rv; - - ravl_data_t head_node_data = {(uintptr_t)block->size, NULL}; - ravl_node_t *node; - node = ravl_find(free_blocks, &head_node_data, RAVL_PREDICATE_EQUAL); - if (node) { - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - head_node = node_data->value; - assert(head_node); - } else { // no head_node - head_node = umf_ba_global_alloc(sizeof(*head_node)); - if (!head_node) { - return -1; - } - - head_node->head = NULL; - - ravl_data_t data = {(uintptr_t)block->size, head_node}; - rv = ravl_emplace_copy(free_blocks, &data); - if (rv) { - umf_ba_global_free(head_node); - return -1; - } - } - - block->free_list_ptr = node_list_add(head_node, block); - if (!block->free_list_ptr) { - return -1; - } - - assert(block->free_list_ptr->block->size == block->size); - - return 0; -} - -// free_blocks_rm_ge - remove the first free block of a size greater or equal to the given size only if it can be properly aligned -// If it was the last block, the head node is freed and removed from the tree. -// It is used during memory allocation (looking for a free block). -static block_t *free_blocks_rm_ge(struct ravl *free_blocks, size_t size, - size_t alignment, - check_free_blocks_t check_blocks) { - ravl_data_t data = {(uintptr_t)size, NULL}; - ravl_node_t *node; - node = ravl_find(free_blocks, &data, RAVL_PREDICATE_GREATER_EQUAL); - if (!node) { - return NULL; - } - - ravl_data_t *node_data = ravl_data(node); - assert(node_data); - assert(node_data->key >= size); - - ravl_free_blocks_head_t *head_node = node_data->value; - assert(head_node); - - block_t *block; - switch (check_blocks) { - case CHECK_ONLY_THE_FIRST_BLOCK: - block = node_list_rm_first(head_node, alignment); - break; - case CHECK_ALL_BLOCKS_OF_SIZE: - block = node_list_rm_with_alignment(head_node, alignment); - break; - // wrong value of check_blocks - default: - abort(); - } - - if (head_node->head == NULL) { - umf_ba_global_free(head_node); - ravl_remove(free_blocks, node); - } - - return block; -} - -// free_blocks_rm_node - remove the free block pointed by the given node. -// If it was the last block, the head node is freed and removed from the tree. -// It is used during merging free blocks and destroying the coarse_provider->free_blocks tree. -static block_t *free_blocks_rm_node(struct ravl *free_blocks, - ravl_free_blocks_elem_t *node) { - assert(free_blocks); - assert(node); - size_t size = node->block->size; - ravl_data_t data = {(uintptr_t)size, NULL}; - ravl_node_t *ravl_node; - ravl_node = ravl_find(free_blocks, &data, RAVL_PREDICATE_EQUAL); - assert(ravl_node); - - ravl_data_t *node_data = ravl_data(ravl_node); - assert(node_data); - assert(node_data->key == size); - - ravl_free_blocks_head_t *head_node = node_data->value; - assert(head_node); - - block_t *block = node_list_rm(head_node, node); - - if (head_node->head == NULL) { - umf_ba_global_free(head_node); - ravl_remove(free_blocks, ravl_node); - } - - return block; -} - -// user_block_merge - merge two blocks from one of two lists of user blocks: all_blocks or free_blocks -static umf_result_t user_block_merge(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node1, ravl_node_t *node2, - bool used, ravl_node_t **merged_node) { - assert(node1); - assert(node2); - assert(node1 == get_node_prev(node2)); - assert(node2 == get_node_next(node1)); - assert(merged_node); - - *merged_node = NULL; - - struct ravl *upstream_blocks = coarse_provider->upstream_blocks; - struct ravl *all_blocks = coarse_provider->all_blocks; - struct ravl *free_blocks = coarse_provider->free_blocks; - - block_t *block1 = get_node_block(node1); - block_t *block2 = get_node_block(node2); - assert(block1->data < block2->data); - - bool same_used = ((block1->used == used) && (block2->used == used)); - bool contignous_data = (block1->data + block1->size == block2->data); - bool same_origin = is_same_origin(upstream_blocks, block1, block2); - - // check if blocks can be merged - if (!same_used || !contignous_data || !same_origin) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (block1->free_list_ptr) { - free_blocks_rm_node(free_blocks, block1->free_list_ptr); - block1->free_list_ptr = NULL; - } - - if (block2->free_list_ptr) { - free_blocks_rm_node(free_blocks, block2->free_list_ptr); - block2->free_list_ptr = NULL; - } - - // update the size - block1->size += block2->size; - - block_t *block_rm = coarse_ravl_rm(all_blocks, block2->data); - assert(block_rm == block2); - (void)block_rm; // WA for unused variable error - umf_ba_global_free(block2); - - *merged_node = node1; - - return UMF_RESULT_SUCCESS; -} - -// free_block_merge_with_prev - merge the given free block -// with the previous one if both are unused and have continuous data. -// Remove the merged block from the tree of free blocks. -static ravl_node_t * -free_block_merge_with_prev(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - ravl_node_t *node_prev = get_node_prev(node); - if (!node_prev) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - user_block_merge(coarse_provider, node_prev, node, false, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -// free_block_merge_with_next - merge the given free block -// with the next one if both are unused and have continuous data. -// Remove the merged block from the tree of free blocks. -static ravl_node_t * -free_block_merge_with_next(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - ravl_node_t *node_next = get_node_next(node); - if (!node_next) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - user_block_merge(coarse_provider, node, node_next, false, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -// upstream_block_merge - merge the given two upstream blocks -static umf_result_t -upstream_block_merge(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node1, ravl_node_t *node2, - ravl_node_t **merged_node) { - assert(node1); - assert(node2); - assert(merged_node); - - *merged_node = NULL; - - umf_memory_provider_handle_t upstream_provider = - coarse_provider->upstream_memory_provider; - if (!upstream_provider) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - block_t *block1 = get_node_block(node1); - block_t *block2 = get_node_block(node2); - assert(block1->data < block2->data); - - bool contignous_data = (block1->data + block1->size == block2->data); - if (!contignous_data) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - // check if blocks can be merged by the upstream provider - umf_result_t merge_status = umfMemoryProviderAllocationMerge( - coarse_provider->upstream_memory_provider, block1->data, block2->data, - block1->size + block2->size); - if (merge_status != UMF_RESULT_SUCCESS) { - return merge_status; - } - - // update the size - block1->size += block2->size; - - struct ravl *upstream_blocks = coarse_provider->upstream_blocks; - block_t *block_rm = coarse_ravl_rm(upstream_blocks, block2->data); - assert(block_rm == block2); - (void)block_rm; // WA for unused variable error - umf_ba_global_free(block2); - - *merged_node = node1; - - return UMF_RESULT_SUCCESS; -} - -// upstream_block_merge_with_prev - merge the given upstream block -// with the previous one if both have continuous data. -// Remove the merged block from the tree of upstream blocks. -static ravl_node_t * -upstream_block_merge_with_prev(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - assert(node); - - ravl_node_t *node_prev = get_node_prev(node); - if (!node_prev) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - upstream_block_merge(coarse_provider, node_prev, node, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -// upstream_block_merge_with_next - merge the given upstream block -// with the next one if both have continuous data. -// Remove the merged block from the tree of upstream blocks. -static ravl_node_t * -upstream_block_merge_with_next(coarse_memory_provider_t *coarse_provider, - ravl_node_t *node) { - assert(node); - - ravl_node_t *node_next = get_node_next(node); - if (!node_next) { - return node; - } - - ravl_node_t *merged_node = NULL; - umf_result_t umf_result = - upstream_block_merge(coarse_provider, node, node_next, &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - return node; - } - - assert(merged_node != NULL); - - return merged_node; -} - -#ifndef NDEBUG // begin of DEBUG code - -typedef struct debug_cb_args_t { - coarse_memory_provider_t *provider; - size_t sum_used; - size_t sum_blocks_size; - size_t num_all_blocks; - size_t num_free_blocks; - size_t num_alloc_blocks; - size_t sum_alloc_size; -} debug_cb_args_t; - -static void debug_verify_all_blocks_cb(void *data, void *arg) { - assert(data); - assert(arg); - - ravl_data_t *node_data = data; - block_t *block = node_data->value; - assert(block); - - debug_cb_args_t *cb_args = (debug_cb_args_t *)arg; - coarse_memory_provider_t *provider = cb_args->provider; - - ravl_node_t *node = - ravl_find(provider->all_blocks, data, RAVL_PREDICATE_EQUAL); - assert(node); - - block_t *block_next = get_block_next(node); - block_t *block_prev = get_block_prev(node); - - cb_args->num_all_blocks++; - if (!block->used) { - cb_args->num_free_blocks++; - } - - assert(block->data); - assert(block->size > 0); - - // There shouldn't be two adjacent unused blocks - // if they are continuous and have the same origin. - if (block_prev && !block_prev->used && !block->used && - (block_prev->data + block_prev->size == block->data)) { - assert(!is_same_origin(provider->upstream_blocks, block_prev, block)); - } - - if (block_next && !block_next->used && !block->used && - (block->data + block->size == block_next->data)) { - assert(!is_same_origin(provider->upstream_blocks, block, block_next)); - } - - // data addresses in the list are in ascending order - if (block_prev) { - assert(block_prev->data < block->data); - } - - if (block_next) { - assert(block->data < block_next->data); - } - - // two block's data should not overlap - if (block_next) { - assert((block->data + block->size) <= block_next->data); - } - - cb_args->sum_blocks_size += block->size; - if (block->used) { - cb_args->sum_used += block->size; - } -} - -static void debug_verify_upstream_blocks_cb(void *data, void *arg) { - assert(data); - assert(arg); - - ravl_data_t *node_data = data; - block_t *alloc = node_data->value; - assert(alloc); - - debug_cb_args_t *cb_args = (debug_cb_args_t *)arg; - coarse_memory_provider_t *provider = cb_args->provider; - - ravl_node_t *node = - ravl_find(provider->upstream_blocks, data, RAVL_PREDICATE_EQUAL); - assert(node); - - block_t *alloc_next = get_block_next(node); - block_t *alloc_prev = get_block_prev(node); - - cb_args->num_alloc_blocks++; - cb_args->sum_alloc_size += alloc->size; - - assert(alloc->data); - assert(alloc->size > 0); - - // data addresses in the list are in ascending order - if (alloc_prev) { - assert(alloc_prev->data < alloc->data); - } - - if (alloc_next) { - assert(alloc->data < alloc_next->data); - } - - // data should not overlap - if (alloc_next) { - assert((alloc->data + alloc->size) <= alloc_next->data); - } -} - -static umf_result_t -coarse_memory_provider_get_stats(void *provider, - coarse_memory_provider_stats_t *stats); - -static bool debug_check(coarse_memory_provider_t *provider) { - assert(provider); - - coarse_memory_provider_stats_t stats = {0}; - coarse_memory_provider_get_stats(provider, &stats); - - debug_cb_args_t cb_args = {0}; - cb_args.provider = provider; - - // verify the all_blocks list - ravl_foreach(provider->all_blocks, debug_verify_all_blocks_cb, &cb_args); - - assert(cb_args.num_all_blocks == stats.num_all_blocks); - assert(cb_args.num_free_blocks == stats.num_free_blocks); - assert(cb_args.sum_used == provider->used_size); - assert(cb_args.sum_blocks_size == provider->alloc_size); - assert(provider->alloc_size >= provider->used_size); - - // verify the upstream_blocks list - ravl_foreach(provider->upstream_blocks, debug_verify_upstream_blocks_cb, - &cb_args); - - assert(cb_args.sum_alloc_size == provider->alloc_size); - assert(cb_args.num_alloc_blocks == stats.num_upstream_blocks); - - return true; -} -#endif /* NDEBUG */ // end of DEBUG code - -static umf_result_t -coarse_add_upstream_block(coarse_memory_provider_t *coarse_provider, void *addr, - size_t size) { - ravl_node_t *alloc_node = NULL; - - block_t *alloc = coarse_ravl_add_new(coarse_provider->upstream_blocks, addr, - size, &alloc_node); - if (alloc == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - block_t *new_block = - coarse_ravl_add_new(coarse_provider->all_blocks, addr, size, NULL); - if (new_block == NULL) { - coarse_ravl_rm(coarse_provider->upstream_blocks, addr); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - // check if the new upstream block can be merged with its neighbours - alloc_node = upstream_block_merge_with_prev(coarse_provider, alloc_node); - alloc_node = upstream_block_merge_with_next(coarse_provider, alloc_node); - - new_block->used = true; - coarse_provider->alloc_size += size; - coarse_provider->used_size += size; - - return UMF_RESULT_SUCCESS; -} - -static umf_result_t -coarse_memory_provider_set_name(coarse_memory_provider_t *coarse_provider) { - if (coarse_provider->upstream_memory_provider == NULL) { - // COARSE_BASE_NAME will be used - coarse_provider->name = NULL; - return UMF_RESULT_SUCCESS; - } - - const char *up_name = - umfMemoryProviderGetName(coarse_provider->upstream_memory_provider); - if (!up_name) { - return UMF_RESULT_ERROR_UNKNOWN; - } - - size_t length = - strlen(COARSE_BASE_NAME) + strlen(up_name) + 3; // + 3 for " ()" - - coarse_provider->name = umf_ba_global_alloc(length + 1); // + 1 for '\0' - if (coarse_provider->name == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - sprintf(coarse_provider->name, "%s (%s)", COARSE_BASE_NAME, up_name); - - return UMF_RESULT_SUCCESS; -} - -// needed for coarse_memory_provider_initialize() -static umf_result_t coarse_memory_provider_alloc(void *provider, size_t size, - size_t alignment, - void **resultPtr); - -// needed for coarse_memory_provider_initialize() -static umf_result_t coarse_memory_provider_free(void *provider, void *ptr, - size_t bytes); - -static umf_result_t coarse_memory_provider_initialize(void *params, - void **provider) { - assert(provider); - - if (params == NULL) { - LOG_ERR("coarse provider parameters are missing"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - coarse_memory_provider_params_t *coarse_params = - (coarse_memory_provider_params_t *)params; - - // check params - if (!coarse_params->upstream_memory_provider == - !coarse_params->init_buffer) { - LOG_ERR("either upstream provider or init buffer has to be provided in " - "the parameters (exactly one of them)"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (coarse_params->init_buffer_size == 0 && - (coarse_params->immediate_init_from_upstream || - coarse_params->init_buffer != NULL)) { - LOG_ERR("init_buffer_size has to be greater than 0 if " - "immediate_init_from_upstream or init_buffer is set"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (coarse_params->init_buffer_size != 0 && - (!coarse_params->immediate_init_from_upstream && - coarse_params->init_buffer == NULL)) { - LOG_ERR("init_buffer_size is greater than 0 but none of " - "immediate_init_from_upstream nor init_buffer is set"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (coarse_params->destroy_upstream_memory_provider && - !coarse_params->upstream_memory_provider) { - LOG_ERR("destroy_upstream_memory_provider is true, but an upstream " - "provider is not provided"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - coarse_memory_provider_t *coarse_provider = - umf_ba_global_alloc(sizeof(*coarse_provider)); - if (!coarse_provider) { - LOG_ERR("out of the host memory"); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - memset(coarse_provider, 0, sizeof(*coarse_provider)); - - coarse_provider->upstream_memory_provider = - coarse_params->upstream_memory_provider; - coarse_provider->destroy_upstream_memory_provider = - coarse_params->destroy_upstream_memory_provider; - coarse_provider->allocation_strategy = coarse_params->allocation_strategy; - coarse_provider->init_buffer = coarse_params->init_buffer; - - umf_result_t umf_result = coarse_memory_provider_set_name(coarse_provider); - if (umf_result != UMF_RESULT_SUCCESS) { - LOG_ERR("name initialization failed"); - goto err_free_coarse_provider; - } - - // most of the error handling paths below set this error - umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - - coarse_provider->upstream_blocks = - ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); - if (coarse_provider->upstream_blocks == NULL) { - LOG_ERR("out of the host memory"); - goto err_free_name; - } - - coarse_provider->free_blocks = - ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); - if (coarse_provider->free_blocks == NULL) { - LOG_ERR("out of the host memory"); - goto err_delete_ravl_upstream_blocks; - } - - coarse_provider->all_blocks = - ravl_new_sized(coarse_ravl_comp, sizeof(ravl_data_t)); - if (coarse_provider->all_blocks == NULL) { - LOG_ERR("out of the host memory"); - goto err_delete_ravl_free_blocks; - } - - coarse_provider->alloc_size = 0; - coarse_provider->used_size = 0; - - if (utils_mutex_init(&coarse_provider->lock) == NULL) { - LOG_ERR("lock initialization failed"); - umf_result = UMF_RESULT_ERROR_UNKNOWN; - goto err_delete_ravl_all_blocks; - } - - if (coarse_params->upstream_memory_provider && - coarse_params->immediate_init_from_upstream) { - // allocate and immediately deallocate memory using the upstream provider - void *init_buffer = NULL; - coarse_memory_provider_alloc( - coarse_provider, coarse_params->init_buffer_size, 0, &init_buffer); - if (init_buffer == NULL) { - goto err_destroy_mutex; - } - - coarse_memory_provider_free(coarse_provider, init_buffer, - coarse_params->init_buffer_size); - - } else if (coarse_params->init_buffer) { - umf_result = coarse_add_upstream_block(coarse_provider, - coarse_provider->init_buffer, - coarse_params->init_buffer_size); - if (umf_result != UMF_RESULT_SUCCESS) { - goto err_destroy_mutex; - } - - LOG_DEBUG("coarse_ALLOC (init_buffer) %zu used %zu alloc %zu", - coarse_params->init_buffer_size, coarse_provider->used_size, - coarse_provider->alloc_size); - - coarse_memory_provider_free(coarse_provider, - coarse_provider->init_buffer, - coarse_params->init_buffer_size); - } - - assert(coarse_provider->used_size == 0); - assert(coarse_provider->alloc_size == coarse_params->init_buffer_size); - assert(debug_check(coarse_provider)); - - *provider = coarse_provider; - - return UMF_RESULT_SUCCESS; - -err_destroy_mutex: - utils_mutex_destroy_not_free(&coarse_provider->lock); -err_delete_ravl_all_blocks: - ravl_delete(coarse_provider->all_blocks); -err_delete_ravl_free_blocks: - ravl_delete(coarse_provider->free_blocks); -err_delete_ravl_upstream_blocks: - ravl_delete(coarse_provider->upstream_blocks); -err_free_name: - umf_ba_global_free(coarse_provider->name); -err_free_coarse_provider: - umf_ba_global_free(coarse_provider); - return umf_result; -} - -static void coarse_ravl_cb_rm_upstream_blocks_node(void *data, void *arg) { - assert(data); - assert(arg); - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)arg; - ravl_data_t *node_data = data; - block_t *alloc = node_data->value; - assert(alloc); - - if (coarse_provider->upstream_memory_provider) { - // We continue to deallocate alloc blocks even if the upstream provider doesn't return success. - umfMemoryProviderFree(coarse_provider->upstream_memory_provider, - alloc->data, alloc->size); - } - - assert(coarse_provider->alloc_size >= alloc->size); - coarse_provider->alloc_size -= alloc->size; - - umf_ba_global_free(alloc); -} - -static void coarse_ravl_cb_rm_all_blocks_node(void *data, void *arg) { - assert(data); - assert(arg); - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)arg; - ravl_data_t *node_data = data; - block_t *block = node_data->value; - assert(block); - - if (block->used) { - assert(coarse_provider->used_size >= block->size); - coarse_provider->used_size -= block->size; - } - - if (block->free_list_ptr) { - free_blocks_rm_node(coarse_provider->free_blocks, block->free_list_ptr); - } - - umf_ba_global_free(block); -} - -static void coarse_memory_provider_finalize(void *provider) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - utils_mutex_destroy_not_free(&coarse_provider->lock); - - ravl_foreach(coarse_provider->all_blocks, coarse_ravl_cb_rm_all_blocks_node, - coarse_provider); - assert(coarse_provider->used_size == 0); - - ravl_foreach(coarse_provider->upstream_blocks, - coarse_ravl_cb_rm_upstream_blocks_node, coarse_provider); - assert(coarse_provider->alloc_size == 0); - - ravl_delete(coarse_provider->upstream_blocks); - ravl_delete(coarse_provider->all_blocks); - ravl_delete(coarse_provider->free_blocks); - - umf_ba_global_free(coarse_provider->name); - - if (coarse_provider->destroy_upstream_memory_provider && - coarse_provider->upstream_memory_provider) { - umfMemoryProviderDestroy(coarse_provider->upstream_memory_provider); - } - - umf_ba_global_free(coarse_provider); -} - -static umf_result_t -create_aligned_block(coarse_memory_provider_t *coarse_provider, - size_t orig_size, size_t alignment, block_t **current) { - (void)orig_size; // unused in the Release version - int rv; - - block_t *curr = *current; - - // In case of non-zero alignment create an aligned block what would be further used. - uintptr_t orig_data = (uintptr_t)curr->data; - uintptr_t aligned_data = ALIGN_UP(orig_data, alignment); - size_t padding = aligned_data - orig_data; - if (alignment > 0 && padding > 0) { - block_t *aligned_block = coarse_ravl_add_new( - coarse_provider->all_blocks, curr->data + padding, - curr->size - padding, NULL); - if (aligned_block == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - curr->used = false; - curr->size = padding; - - rv = free_blocks_add(coarse_provider->free_blocks, curr); - if (rv) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - // use aligned block - *current = aligned_block; - assert((*current)->size >= orig_size); - } - - return UMF_RESULT_SUCCESS; -} - -// Split the current block and put the new block after the one that we use. -static umf_result_t -split_current_block(coarse_memory_provider_t *coarse_provider, block_t *curr, - size_t size) { - ravl_node_t *new_node = NULL; - - block_t *new_block = - coarse_ravl_add_new(coarse_provider->all_blocks, curr->data + size, - curr->size - size, &new_node); - if (new_block == NULL) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - new_block->used = false; - - int rv = - free_blocks_add(coarse_provider->free_blocks, get_node_block(new_node)); - if (rv) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - return UMF_RESULT_SUCCESS; -} - -static block_t * -find_free_block(struct ravl *free_blocks, size_t size, size_t alignment, - coarse_memory_provider_strategy_t allocation_strategy) { - block_t *block; - - switch (allocation_strategy) { - case UMF_COARSE_MEMORY_STRATEGY_FASTEST: - // Always allocate a free block of the (size + alignment) size - // and later cut out the properly aligned part leaving two remaining parts. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, - CHECK_ONLY_THE_FIRST_BLOCK); - - case UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE: - // First check if the first free block of the 'size' size has the correct alignment. - block = free_blocks_rm_ge(free_blocks, size, alignment, - CHECK_ONLY_THE_FIRST_BLOCK); - if (block) { - return block; - } - - // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, - CHECK_ONLY_THE_FIRST_BLOCK); - - case UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE: - // First look through all free blocks of the 'size' size - // and choose the first one with the correct alignment. - block = free_blocks_rm_ge(free_blocks, size, alignment, - CHECK_ALL_BLOCKS_OF_SIZE); - if (block) { - return block; - } - - // If none of them had the correct alignment, - // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, - CHECK_ONLY_THE_FIRST_BLOCK); - - // unknown memory allocation strategy - default: - abort(); - } -} - -static umf_result_t coarse_memory_provider_alloc(void *provider, size_t size, - size_t alignment, - void **resultPtr) { - umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; - - if (resultPtr == NULL) { - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - // Find a block with greater or equal size using the given memory allocation strategy - block_t *curr = - find_free_block(coarse_provider->free_blocks, size, alignment, - coarse_provider->allocation_strategy); - - // If the block that we want to reuse has a greater size, split it. - // Try to merge the split part with the successor if it is not used. - enum { ACTION_NONE = 0, ACTION_USE, ACTION_SPLIT } action = ACTION_NONE; - - if (curr && curr->size > size) { - action = ACTION_SPLIT; - } else if (curr && curr->size == size) { - action = ACTION_USE; - } - - if (action) { // ACTION_SPLIT or ACTION_USE - assert(curr->used == false); - - // In case of non-zero alignment create an aligned block what would be further used. - if (alignment > 0) { - umf_result = - create_aligned_block(coarse_provider, size, alignment, &curr); - if (umf_result != UMF_RESULT_SUCCESS) { - utils_mutex_unlock(&coarse_provider->lock); - return umf_result; - } - } - - if (action == ACTION_SPLIT) { - // Split the current block and put the new block after the one that we use. - umf_result = split_current_block(coarse_provider, curr, size); - if (umf_result != UMF_RESULT_SUCCESS) { - utils_mutex_unlock(&coarse_provider->lock); - return umf_result; - } - - curr->size = size; - - LOG_DEBUG("coarse_ALLOC (split_block) %zu used %zu alloc %zu", size, - coarse_provider->used_size, coarse_provider->alloc_size); - - } else { // action == ACTION_USE - LOG_DEBUG("coarse_ALLOC (same_block) %zu used %zu alloc %zu", size, - coarse_provider->used_size, coarse_provider->alloc_size); - } - - curr->used = true; - *resultPtr = curr->data; - coarse_provider->used_size += size; - - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return UMF_RESULT_SUCCESS; - } - - // no suitable block found - try to get more memory from the upstream provider - umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - - if (coarse_provider->upstream_memory_provider == NULL) { - LOG_ERR("out of memory - no upstream memory provider given"); - goto err_unlock; - } - - umfMemoryProviderAlloc(coarse_provider->upstream_memory_provider, size, - alignment, resultPtr); - if (*resultPtr == NULL) { - LOG_ERR("out of memory - upstream memory provider allocation failed"); - goto err_unlock; - } - - ASSERT_IS_ALIGNED(((uintptr_t)(*resultPtr)), alignment); - - umf_result = coarse_add_upstream_block(coarse_provider, *resultPtr, size); - if (umf_result != UMF_RESULT_SUCCESS) { - umfMemoryProviderFree(coarse_provider->upstream_memory_provider, - *resultPtr, size); - goto err_unlock; - } - - LOG_DEBUG("coarse_ALLOC (upstream) %zu used %zu alloc %zu", size, - coarse_provider->used_size, coarse_provider->alloc_size); - - umf_result = UMF_RESULT_SUCCESS; - -err_unlock: - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return umf_result; -} - -static umf_result_t coarse_memory_provider_free(void *provider, void *ptr, - size_t bytes) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - ravl_node_t *node = coarse_ravl_find_node(coarse_provider->all_blocks, ptr); - if (node == NULL) { - // the block was not found - utils_mutex_unlock(&coarse_provider->lock); - LOG_ERR("memory block not found (ptr = %p, size = %zu)", ptr, bytes); - return UMF_RESULT_ERROR_UNKNOWN; - } - - block_t *block = get_node_block(node); - if (!block->used) { - // the block is already free - utils_mutex_unlock(&coarse_provider->lock); - LOG_ERR("the block is already free"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - if (bytes > 0 && bytes != block->size) { - // wrong size of allocation - utils_mutex_unlock(&coarse_provider->lock); - LOG_ERR("wrong size of allocation"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - LOG_DEBUG("coarse_FREE (return_block_to_pool) %zu used %zu alloc %zu", - block->size, coarse_provider->used_size - block->size, - coarse_provider->alloc_size); - - assert(coarse_provider->used_size >= block->size); - coarse_provider->used_size -= block->size; - - block->used = false; - - // Merge with prev and/or next block if they are unused and have continuous data. - node = free_block_merge_with_prev(coarse_provider, node); - node = free_block_merge_with_next(coarse_provider, node); - - int rv = - free_blocks_add(coarse_provider->free_blocks, get_node_block(node)); - if (rv) { - utils_mutex_unlock(&coarse_provider->lock); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return UMF_RESULT_SUCCESS; -} - -static void coarse_memory_provider_get_last_native_error(void *provider, - const char **ppMessage, - int32_t *pError) { - (void)provider; // unused - - if (ppMessage == NULL || pError == NULL) { - assert(0); - return; - } - - // Nothing more is needed here, since - // there is no UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC error used. -} - -static umf_result_t coarse_memory_provider_get_min_page_size(void *provider, - void *ptr, - size_t *pageSize) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (!coarse_provider->upstream_memory_provider) { - *pageSize = utils_get_page_size(); - return UMF_RESULT_SUCCESS; - } - - return umfMemoryProviderGetMinPageSize( - coarse_provider->upstream_memory_provider, ptr, pageSize); -} - -static umf_result_t -coarse_memory_provider_get_recommended_page_size(void *provider, size_t size, - size_t *pageSize) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (!coarse_provider->upstream_memory_provider) { - *pageSize = utils_get_page_size(); - return UMF_RESULT_SUCCESS; - } - - return umfMemoryProviderGetRecommendedPageSize( - coarse_provider->upstream_memory_provider, size, pageSize); -} - -static const char *coarse_memory_provider_get_name(void *provider) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (!coarse_provider->name) { - return COARSE_BASE_NAME; - } - - return coarse_provider->name; -} - -static void ravl_cb_count(void *data, void *arg) { - assert(arg); - (void)data; /* unused */ - - size_t *num_all_blocks = arg; - (*num_all_blocks)++; -} - -static void ravl_cb_count_free(void *data, void *arg) { - assert(data); - assert(arg); - - ravl_data_t *node_data = data; - assert(node_data); - ravl_free_blocks_head_t *head_node = node_data->value; - assert(head_node); - struct ravl_free_blocks_elem_t *free_block = head_node->head; - assert(free_block); - - size_t *num_all_blocks = arg; - while (free_block) { - (*num_all_blocks)++; - free_block = free_block->next; - } -} - -static umf_result_t -coarse_memory_provider_get_stats(void *provider, - coarse_memory_provider_stats_t *stats) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - // count blocks - size_t num_upstream_blocks = 0; - ravl_foreach(coarse_provider->upstream_blocks, ravl_cb_count, - &num_upstream_blocks); - - size_t num_all_blocks = 0; - ravl_foreach(coarse_provider->all_blocks, ravl_cb_count, &num_all_blocks); - - size_t num_free_blocks = 0; - ravl_foreach(coarse_provider->free_blocks, ravl_cb_count_free, - &num_free_blocks); - - stats->alloc_size = coarse_provider->alloc_size; - stats->used_size = coarse_provider->used_size; - stats->num_upstream_blocks = num_upstream_blocks; - stats->num_all_blocks = num_all_blocks; - stats->num_free_blocks = num_free_blocks; - - return UMF_RESULT_SUCCESS; -} - -static umf_result_t coarse_memory_provider_purge_lazy(void *provider, void *ptr, - size_t size) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - if (coarse_provider->upstream_memory_provider == NULL) { - LOG_ERR("no upstream memory provider given"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - return umfMemoryProviderPurgeLazy(coarse_provider->upstream_memory_provider, - ptr, size); -} - -static umf_result_t coarse_memory_provider_purge_force(void *provider, - void *ptr, size_t size) { - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - if (coarse_provider->upstream_memory_provider == NULL) { - LOG_ERR("no upstream memory provider given"); - return UMF_RESULT_ERROR_NOT_SUPPORTED; - } - - return umfMemoryProviderPurgeForce( - coarse_provider->upstream_memory_provider, ptr, size); -} - -static umf_result_t coarse_memory_provider_allocation_split(void *provider, - void *ptr, - size_t totalSize, - size_t firstSize) { - umf_result_t umf_result; - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - ravl_node_t *node = coarse_ravl_find_node(coarse_provider->all_blocks, ptr); - if (node == NULL) { - LOG_ERR("memory block not found"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *block = get_node_block(node); - - if (block->size != totalSize) { - LOG_ERR("wrong totalSize"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (!block->used) { - LOG_ERR("block is not allocated"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *new_block = coarse_ravl_add_new(coarse_provider->all_blocks, - block->data + firstSize, - block->size - firstSize, NULL); - if (new_block == NULL) { - umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - goto err_mutex_unlock; - } - - block->size = firstSize; - new_block->used = true; - - assert(new_block->size == (totalSize - firstSize)); - - umf_result = UMF_RESULT_SUCCESS; - -err_mutex_unlock: - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return umf_result; -} - -static umf_result_t coarse_memory_provider_allocation_merge(void *provider, - void *lowPtr, - void *highPtr, - size_t totalSize) { - umf_result_t umf_result; - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)provider; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return UMF_RESULT_ERROR_UNKNOWN; - } - - assert(debug_check(coarse_provider)); - - ravl_node_t *low_node = - coarse_ravl_find_node(coarse_provider->all_blocks, lowPtr); - if (low_node == NULL) { - LOG_ERR("the lowPtr memory block not found"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *low_block = get_node_block(low_node); - if (!low_block->used) { - LOG_ERR("the lowPtr block is not allocated"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - ravl_node_t *high_node = - coarse_ravl_find_node(coarse_provider->all_blocks, highPtr); - if (high_node == NULL) { - LOG_ERR("the highPtr memory block not found"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - block_t *high_block = get_node_block(high_node); - if (!high_block->used) { - LOG_ERR("the highPtr block is not allocated"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (get_node_next(low_node) != high_node) { - LOG_ERR("given pointers cannot be merged"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (get_node_prev(high_node) != low_node) { - LOG_ERR("given pointers cannot be merged"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if (low_block->size + high_block->size != totalSize) { - LOG_ERR("wrong totalSize"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - if ((uintptr_t)highPtr != ((uintptr_t)lowPtr + low_block->size)) { - LOG_ERR("given pointers cannot be merged"); - umf_result = UMF_RESULT_ERROR_INVALID_ARGUMENT; - goto err_mutex_unlock; - } - - ravl_node_t *merged_node = NULL; - - umf_result = user_block_merge(coarse_provider, low_node, high_node, true, - &merged_node); - if (umf_result != UMF_RESULT_SUCCESS) { - LOG_ERR("merging failed"); - goto err_mutex_unlock; - } - - assert(merged_node == low_node); - assert(low_block->size == totalSize); - - umf_result = UMF_RESULT_SUCCESS; - -err_mutex_unlock: - assert(debug_check(coarse_provider)); - utils_mutex_unlock(&coarse_provider->lock); - - return umf_result; -} - -umf_memory_provider_ops_t UMF_COARSE_MEMORY_PROVIDER_OPS = { - .version = UMF_VERSION_CURRENT, - .initialize = coarse_memory_provider_initialize, - .finalize = coarse_memory_provider_finalize, - .alloc = coarse_memory_provider_alloc, - .free = coarse_memory_provider_free, - .get_last_native_error = coarse_memory_provider_get_last_native_error, - .get_recommended_page_size = - coarse_memory_provider_get_recommended_page_size, - .get_min_page_size = coarse_memory_provider_get_min_page_size, - .get_name = coarse_memory_provider_get_name, - .ext.purge_lazy = coarse_memory_provider_purge_lazy, - .ext.purge_force = coarse_memory_provider_purge_force, - .ext.allocation_merge = coarse_memory_provider_allocation_merge, - .ext.allocation_split = coarse_memory_provider_allocation_split, - // TODO - /* - .ipc.get_ipc_handle_size = coarse_memory_provider_get_ipc_handle_size, - .ipc.get_ipc_handle = coarse_memory_provider_get_ipc_handle, - .ipc.put_ipc_handle = coarse_memory_provider_put_ipc_handle, - .ipc.open_ipc_handle = coarse_memory_provider_open_ipc_handle, - .ipc.close_ipc_handle = coarse_memory_provider_close_ipc_handle, - */ -}; - -umf_memory_provider_ops_t *umfCoarseMemoryProviderOps(void) { - return &UMF_COARSE_MEMORY_PROVIDER_OPS; -} - -coarse_memory_provider_stats_t -umfCoarseMemoryProviderGetStats(umf_memory_provider_handle_t provider) { - coarse_memory_provider_stats_t stats = {0}; - - if (provider == NULL) { - return stats; - } - - void *priv = umfMemoryProviderGetPriv(provider); - - coarse_memory_provider_t *coarse_provider = - (struct coarse_memory_provider_t *)priv; - - if (utils_mutex_lock(&coarse_provider->lock) != 0) { - LOG_ERR("locking the lock failed"); - return stats; - } - - coarse_memory_provider_get_stats(priv, &stats); - - utils_mutex_unlock(&coarse_provider->lock); - - return stats; -} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b564789706..593268a52c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -199,11 +199,6 @@ if(LINUX) LIBS ${UMF_UTILS_FOR_TEST}) endif() -add_umf_test( - NAME provider_coarse - SRCS provider_coarse.cpp ${BA_SOURCES_FOR_TEST} - LIBS ${UMF_UTILS_FOR_TEST}) - add_umf_test( NAME coarse_lib SRCS coarse_lib.cpp ${BA_SOURCES_FOR_TEST} @@ -218,10 +213,12 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT) NAME c_api_disjoint_pool SRCS c_api/disjoint_pool.c LIBS disjoint_pool) - add_umf_test( - NAME disjointCoarseMallocPool - SRCS disjointCoarseMallocPool.cpp - LIBS disjoint_pool) + if(LINUX AND (NOT UMF_DISABLE_HWLOC)) + add_umf_test( + NAME disjointCoarseMallocPool + SRCS disjointCoarseMallocPool.cpp + LIBS disjoint_pool) + endif() endif() if(UMF_BUILD_LIBUMF_POOL_DISJOINT diff --git a/test/disjointCoarseMallocPool.cpp b/test/disjointCoarseMallocPool.cpp index 32e1d24f36..45502b1923 100644 --- a/test/disjointCoarseMallocPool.cpp +++ b/test/disjointCoarseMallocPool.cpp @@ -7,39 +7,40 @@ #include -#include "provider.hpp" - #include -#include +#include + +#include "coarse.h" +#include "provider.hpp" using umf_test::KB; using umf_test::MB; using umf_test::test; -#define GetStats umfCoarseMemoryProviderGetStats +#define FILE_PATH ((char *)"tmp_file") umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = umf::providerMakeCOps(); -struct CoarseWithMemoryStrategyTest +struct FileWithMemoryStrategyTest : umf_test::test, - ::testing::WithParamInterface { + ::testing::WithParamInterface { void SetUp() override { test::SetUp(); allocation_strategy = this->GetParam(); } - coarse_memory_provider_strategy_t allocation_strategy; + coarse_strategy_t allocation_strategy; }; INSTANTIATE_TEST_SUITE_P( - CoarseWithMemoryStrategyTest, CoarseWithMemoryStrategyTest, + FileWithMemoryStrategyTest, FileWithMemoryStrategyTest, ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_basic) { - umf_memory_provider_handle_t malloc_memory_provider; +TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple1) { + umf_memory_provider_handle_t malloc_memory_provider = nullptr; umf_result_t umf_result; umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, @@ -47,217 +48,19 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_basic) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(malloc_memory_provider, nullptr); - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.destroy_upstream_memory_provider = true; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; - umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(disjoint_pool_params, nullptr); - umf_result = - umfDisjointPoolParamsSetSlabMinSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMaxPoolableSize(disjoint_pool_params, 4096); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetCapacity(disjoint_pool_params, 4); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = - umfDisjointPoolParamsSetMinBucketSize(disjoint_pool_params, 64); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfDisjointPoolParamsSetTrace(disjoint_pool_params, 1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - umf_memory_pool_handle_t pool; - umf_result = umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, - disjoint_pool_params, - UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(pool, nullptr); - - umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - // test - - umf_memory_provider_handle_t prov = NULL; - umf_result = umfPoolGetMemoryProvider(pool, &prov); + umf_file_memory_provider_params_handle_t file_params = nullptr; + umf_result = umfFileMemoryProviderParamsCreate(&file_params, FILE_PATH); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(prov, nullptr); - - // alloc 2x 2MB - void *p1 = umfPoolMalloc(pool, 2 * MB); - ASSERT_NE(p1, nullptr); - ASSERT_EQ(GetStats(prov).used_size, 2 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 2); - - void *p2 = umfPoolMalloc(pool, 2 * MB); - ASSERT_NE(p2, nullptr); - ASSERT_EQ(GetStats(prov).used_size, 4 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - ASSERT_NE(p1, p2); - - // swap pointers to get p1 < p2 - if (p1 > p2) { - std::swap(p1, p2); - } - - // free + alloc first block - // the block should be reused - // currently there is no purging, so the alloc size shouldn't change - // there should be no block merging between used and not-used blocks - umf_result = umfPoolFree(pool, p1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(prov).used_size, 2 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - - p1 = umfPoolMalloc(pool, 2 * MB); - ASSERT_EQ(GetStats(prov).used_size, 4 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - - // free all allocs - // overall alloc size shouldn't change - // block p2 should merge with the prev free block p1 - // and the remaining init block - umf_result = umfPoolFree(pool, p1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - umf_result = umfPoolFree(pool, p2); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(prov).used_size, 0 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - - // test allocations with alignment - // TODO: what about holes? - p1 = umfPoolAlignedMalloc(pool, 1 * MB - 4, 128); - ASSERT_NE(p1, nullptr); - ASSERT_EQ((uintptr_t)p1 & 127, 0); - p2 = umfPoolAlignedMalloc(pool, 1 * MB - 4, 128); - ASSERT_NE(p2, nullptr); - ASSERT_EQ((uintptr_t)p1 & 127, 0); - umf_result = umfPoolFree(pool, p1); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_result = umfPoolFree(pool, p2); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - - // alloc whole buffer - // after this, there should be one single block - p1 = umfPoolMalloc(pool, init_buffer_size); - ASSERT_EQ(GetStats(prov).used_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - - // free all memory - // alloc 2 MB block - the init block should be split - umf_result = umfPoolFree(pool, p1); - p1 = umfPoolMalloc(pool, 2 * MB); - ASSERT_EQ(GetStats(prov).used_size, 2 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 2); - - // alloc additional 2 MB - // the non-used block should be used - p2 = umfPoolMalloc(pool, 2 * MB); - ASSERT_EQ(GetStats(prov).used_size, 4 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 3); - ASSERT_NE(p1, p2); - - // make sure that p1 < p2 - if (p1 > p2) { - std::swap(p1, p2); - } - - // free blocks in order: p2, p1 - // block p1 should merge with the next block p2 - // swap pointers to get p1 < p2 - umfPoolFree(pool, p2); - umfPoolFree(pool, p1); - ASSERT_EQ(GetStats(prov).used_size, 0 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - - // alloc 10x 2 MB - this should occupy all allocated memory - constexpr int allocs_size = 10; - void *allocs[allocs_size] = {0}; - for (int i = 0; i < allocs_size; i++) { - ASSERT_EQ(GetStats(prov).used_size, i * 2 * MB); - allocs[i] = umfPoolMalloc(pool, 2 * MB); - ASSERT_NE(allocs[i], nullptr); - } - ASSERT_EQ(GetStats(prov).used_size, 20 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - // there should be no block with the free memory - ASSERT_EQ(GetStats(prov).num_all_blocks, allocs_size); - - // free all memory - for (int i = 0; i < allocs_size; i++) { - umf_result = umfPoolFree(pool, allocs[i]); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - } - - ASSERT_EQ(GetStats(prov).num_all_blocks, 1); - ASSERT_EQ(GetStats(prov).used_size, 0 * MB); - ASSERT_EQ(GetStats(prov).alloc_size, init_buffer_size); - - umfPoolDestroy(pool); - // Both coarse_memory_provider and malloc_memory_provider - // have already been destroyed by umfPoolDestroy(), because: - // UMF_POOL_CREATE_FLAG_OWN_PROVIDER was set in umfPoolCreate() and - // coarse_memory_provider_params.destroy_upstream_memory_provider = true; -} - -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; + ASSERT_NE(file_params, nullptr); - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); + umf_memory_provider_handle_t file_memory_provider; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + file_params, &file_memory_provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; + ASSERT_NE(file_memory_provider, nullptr); - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); + umf_result = umfFileMemoryProviderParamsDestroy(file_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); @@ -279,7 +82,7 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { umf_memory_pool_handle_t pool; umf_result = - umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, + umfPoolCreate(umfDisjointPoolOps(), file_memory_provider, disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(pool, nullptr); @@ -295,8 +98,6 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { size_t s1 = 74659 * KB; size_t s2 = 8206 * KB; - size_t max_alloc_size = 0; - const int nreps = 2; const int nptrs = 6; @@ -308,10 +109,6 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { ASSERT_NE(t[i], nullptr); } - if (max_alloc_size == 0) { - max_alloc_size = GetStats(prov).alloc_size; - } - for (int i = 0; i < nptrs; i++) { umf_result = umfPoolFree(pool, t[i]); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); @@ -326,9 +123,6 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { ASSERT_NE(t[i], nullptr); } - // all s2 should fit into single block leaved after freeing s1 - ASSERT_LE(GetStats(prov).alloc_size, max_alloc_size); - for (int i = 0; i < nptrs; i++) { umf_result = umfPoolFree(pool, t[i]); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); @@ -336,12 +130,12 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple1) { } umfPoolDestroy(pool); - umfMemoryProviderDestroy(coarse_memory_provider); + umfMemoryProviderDestroy(file_memory_provider); umfMemoryProviderDestroy(malloc_memory_provider); } -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple2) { - umf_memory_provider_handle_t malloc_memory_provider; +TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple2) { + umf_memory_provider_handle_t malloc_memory_provider = nullptr; umf_result_t umf_result; umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, @@ -349,25 +143,19 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple2) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(malloc_memory_provider, nullptr); - const size_t init_buffer_size = 20 * MB; + umf_file_memory_provider_params_handle_t file_params = nullptr; + umf_result = umfFileMemoryProviderParamsCreate(&file_params, FILE_PATH); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_params, nullptr); - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; + umf_memory_provider_handle_t file_memory_provider; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + file_params, &file_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_memory_provider, nullptr); - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); + umf_result = umfFileMemoryProviderParamsDestroy(file_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); @@ -389,7 +177,7 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple2) { umf_memory_pool_handle_t pool; umf_result = - umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, + umfPoolCreate(umfDisjointPoolOps(), file_memory_provider, disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(pool, nullptr); @@ -415,7 +203,7 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMallocPool_simple2) { } umfPoolDestroy(pool); - umfMemoryProviderDestroy(coarse_memory_provider); + umfMemoryProviderDestroy(file_memory_provider); umfMemoryProviderDestroy(malloc_memory_provider); } @@ -431,7 +219,7 @@ struct alloc_ptr_size { } }; -TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMMapPool_random) { +TEST_P(FileWithMemoryStrategyTest, disjointFileMMapPool_random) { umf_result_t umf_result; const size_t init_buffer_size = 200 * MB; @@ -443,22 +231,19 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMMapPool_random) { const unsigned char alloc_check_val = 11; - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = NULL; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; + umf_file_memory_provider_params_handle_t file_params = nullptr; + umf_result = umfFileMemoryProviderParamsCreate(&file_params, FILE_PATH); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_params, nullptr); + + umf_memory_provider_handle_t file_memory_provider; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), + file_params, &file_memory_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(file_memory_provider, nullptr); - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); + umf_result = umfFileMemoryProviderParamsDestroy(file_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); @@ -480,7 +265,7 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMMapPool_random) { umf_memory_pool_handle_t pool; umf_result = - umfPoolCreate(umfDisjointPoolOps(), coarse_memory_provider, + umfPoolCreate(umfDisjointPoolOps(), file_memory_provider, disjoint_pool_params, UMF_POOL_CREATE_FLAG_NONE, &pool); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(pool, nullptr); @@ -520,9 +305,7 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMMapPool_random) { // alloc for (size_t j = 0; j < count; j++) { - void *ptr = umfPoolMalloc(pool, size); - ASSERT_NE(ptr, nullptr); - + void *ptr = umfPoolCalloc(pool, 1, size); if (ptr == nullptr) { break; } @@ -576,5 +359,5 @@ TEST_P(CoarseWithMemoryStrategyTest, disjointCoarseMMapPool_random) { } umfPoolDestroy(pool); - umfMemoryProviderDestroy(coarse_memory_provider); + umfMemoryProviderDestroy(file_memory_provider); } diff --git a/test/memoryPoolAPI.cpp b/test/memoryPoolAPI.cpp index 1c6d83f2af..95dcfabb23 100644 --- a/test/memoryPoolAPI.cpp +++ b/test/memoryPoolAPI.cpp @@ -181,16 +181,14 @@ TEST_F(test, BasicPoolByPtrTest) { INSTANTIATE_TEST_SUITE_P( mallocPoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{&MALLOC_POOL_OPS, nullptr, - &UMF_NULL_PROVIDER_OPS, nullptr, - nullptr}, + &UMF_NULL_PROVIDER_OPS, nullptr}, poolCreateExtParams{umfProxyPoolOps(), nullptr, - &BA_GLOBAL_PROVIDER_OPS, nullptr, - nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); INSTANTIATE_TEST_SUITE_P(mallocMultiPoolTest, umfMultiPoolTest, ::testing::Values(poolCreateExtParams{ umfProxyPoolOps(), nullptr, - &BA_GLOBAL_PROVIDER_OPS, nullptr, nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); INSTANTIATE_TEST_SUITE_P(umfPoolWithCreateFlagsTest, umfPoolWithCreateFlagsTest, ::testing::Values(0, diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index e5ec85012c..6f54fe1142 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -7,7 +7,6 @@ #include "pool.hpp" #include "provider.hpp" -#include "umf/providers/provider_coarse.h" #include "umf/providers/provider_devdax_memory.h" #include "utils/utils_sanitizers.h" @@ -20,13 +19,11 @@ #include "../malloc_compliance_tests.hpp" -using poolCreateExtParams = - std::tuple; +using poolCreateExtParams = std::tuple; umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { - auto [pool_ops, pool_params, provider_ops, provider_params, coarse_params] = - params; + auto [pool_ops, pool_params, provider_ops, provider_params] = params; umf_memory_provider_handle_t upstream_provider = nullptr; umf_memory_provider_handle_t provider = nullptr; @@ -40,22 +37,6 @@ umf::pool_unique_handle_t poolCreateExtUnique(poolCreateExtParams params) { provider = upstream_provider; - if (coarse_params) { - coarse_memory_provider_params_t *coarse_memory_provider_params = - (coarse_memory_provider_params_t *)coarse_params; - coarse_memory_provider_params->upstream_memory_provider = - upstream_provider; - coarse_memory_provider_params->destroy_upstream_memory_provider = true; - - umf_memory_provider_handle_t coarse_provider = nullptr; - ret = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - coarse_params, &coarse_provider); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - EXPECT_NE(coarse_provider, nullptr); - - provider = coarse_provider; - } - ret = umfPoolCreate(pool_ops, provider, pool_params, UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &hPool); EXPECT_EQ(ret, UMF_RESULT_SUCCESS); diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 471e53dc2c..c254400db3 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -248,19 +248,18 @@ INSTANTIATE_TEST_SUITE_P(disjointPoolTests, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfDisjointPoolOps(), (void *)defaultPoolConfig.get(), - &BA_GLOBAL_PROVIDER_OPS, nullptr, nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); -INSTANTIATE_TEST_SUITE_P(disjointPoolTests, umfMemTest, - ::testing::Values(std::make_tuple( - poolCreateExtParams{ - umfDisjointPoolOps(), - (void *)defaultPoolConfig.get(), - &MOCK_OUT_OF_MEM_PROVIDER_OPS, - (void *)&DEFAULT_DISJOINT_CAPACITY, nullptr}, - static_cast(DEFAULT_DISJOINT_CAPACITY) / 2))); +INSTANTIATE_TEST_SUITE_P( + disjointPoolTests, umfMemTest, + ::testing::Values(std::make_tuple( + poolCreateExtParams{ + umfDisjointPoolOps(), (void *)defaultPoolConfig.get(), + &MOCK_OUT_OF_MEM_PROVIDER_OPS, (void *)&DEFAULT_DISJOINT_CAPACITY}, + static_cast(DEFAULT_DISJOINT_CAPACITY) / 2))); INSTANTIATE_TEST_SUITE_P(disjointMultiPoolTests, umfMultiPoolTest, ::testing::Values(poolCreateExtParams{ umfDisjointPoolOps(), (void *)defaultPoolConfig.get(), - &BA_GLOBAL_PROVIDER_OPS, nullptr, nullptr})); + &BA_GLOBAL_PROVIDER_OPS, nullptr})); diff --git a/test/pools/jemalloc_coarse_devdax.cpp b/test/pools/jemalloc_coarse_devdax.cpp index 350e053ab7..72906e625b 100644 --- a/test/pools/jemalloc_coarse_devdax.cpp +++ b/test/pools/jemalloc_coarse_devdax.cpp @@ -31,15 +31,13 @@ devdax_params_unique_handle_t create_devdax_params() { &umfDevDaxMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); auto devdaxParams = create_devdax_params(); static std::vector poolParamsList = - devdaxParams.get() - ? std::vector{poolCreateExtParams{ - umfJemallocPoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - devdaxParams.get(), &coarseParams}} - : std::vector{}; + devdaxParams.get() ? std::vector{poolCreateExtParams{ + umfJemallocPoolOps(), nullptr, + umfDevDaxMemoryProviderOps(), devdaxParams.get()}} + : std::vector{}; INSTANTIATE_TEST_SUITE_P(jemallocCoarseDevDaxTest, umfPoolTest, ::testing::ValuesIn(poolParamsList)); diff --git a/test/pools/jemalloc_coarse_file.cpp b/test/pools/jemalloc_coarse_file.cpp index 74ad36d56e..68a602df64 100644 --- a/test/pools/jemalloc_coarse_file.cpp +++ b/test/pools/jemalloc_coarse_file.cpp @@ -23,11 +23,9 @@ file_params_unique_handle_t get_file_params_default(char *path) { &umfFileMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); file_params_unique_handle_t fileParams = get_file_params_default(FILE_PATH); INSTANTIATE_TEST_SUITE_P(jemallocCoarseFileTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfJemallocPoolOps(), nullptr, - umfFileMemoryProviderOps(), fileParams.get(), - &coarseParams})); + umfFileMemoryProviderOps(), fileParams.get()})); diff --git a/test/pools/jemalloc_pool.cpp b/test/pools/jemalloc_pool.cpp index 4dddbcd32b..bcc9623c7f 100644 --- a/test/pools/jemalloc_pool.cpp +++ b/test/pools/jemalloc_pool.cpp @@ -29,8 +29,7 @@ auto defaultParams = createOsMemoryProviderParams(); INSTANTIATE_TEST_SUITE_P(jemallocPoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfJemallocPoolOps(), nullptr, - umfOsMemoryProviderOps(), defaultParams.get(), - nullptr})); + umfOsMemoryProviderOps(), defaultParams.get()})); // this test makes sure that jemalloc does not use // memory provider to allocate metadata (and hence @@ -48,9 +47,8 @@ TEST_F(test, metadataNotAllocatedUsingProvider) { res = umfOsMemoryProviderParamsSetProtection(params, UMF_PROTECTION_NONE); ASSERT_EQ(res, UMF_RESULT_SUCCESS); - auto pool = - poolCreateExtUnique({umfJemallocPoolOps(), nullptr, - umfOsMemoryProviderOps(), params, nullptr}); + auto pool = poolCreateExtUnique( + {umfJemallocPoolOps(), nullptr, umfOsMemoryProviderOps(), params}); res = umfOsMemoryProviderParamsDestroy(params); ASSERT_EQ(res, UMF_RESULT_SUCCESS); diff --git a/test/pools/pool_base_alloc.cpp b/test/pools/pool_base_alloc.cpp index 7c9a3701a7..ec07a7c2fe 100644 --- a/test/pools/pool_base_alloc.cpp +++ b/test/pools/pool_base_alloc.cpp @@ -48,4 +48,4 @@ umf_memory_pool_ops_t BA_POOL_OPS = umf::poolMakeCOps(); INSTANTIATE_TEST_SUITE_P(baPool, umfPoolTest, ::testing::Values(poolCreateExtParams{ &BA_POOL_OPS, nullptr, - &umf_test::BASE_PROVIDER_OPS, nullptr, nullptr})); + &umf_test::BASE_PROVIDER_OPS, nullptr})); diff --git a/test/pools/pool_coarse.hpp b/test/pools/pool_coarse.hpp index 7baa612f1e..b1efb4fee9 100644 --- a/test/pools/pool_coarse.hpp +++ b/test/pools/pool_coarse.hpp @@ -5,8 +5,6 @@ #ifndef UMF_TEST_POOL_COARSE_HPP #define UMF_TEST_POOL_COARSE_HPP 1 -#include "umf/providers/provider_coarse.h" - #include "pool.hpp" #include "poolFixtures.hpp" diff --git a/test/pools/scalable_coarse_devdax.cpp b/test/pools/scalable_coarse_devdax.cpp index 1bf77c61ce..970f45ef92 100644 --- a/test/pools/scalable_coarse_devdax.cpp +++ b/test/pools/scalable_coarse_devdax.cpp @@ -31,15 +31,13 @@ devdax_params_unique_handle_t create_devdax_params() { &umfDevDaxMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); auto devdaxParams = create_devdax_params(); static std::vector poolParamsList = - devdaxParams.get() - ? std::vector{poolCreateExtParams{ - umfScalablePoolOps(), nullptr, umfDevDaxMemoryProviderOps(), - devdaxParams.get(), &coarseParams}} - : std::vector{}; + devdaxParams.get() ? std::vector{poolCreateExtParams{ + umfScalablePoolOps(), nullptr, + umfDevDaxMemoryProviderOps(), devdaxParams.get()}} + : std::vector{}; INSTANTIATE_TEST_SUITE_P(scalableCoarseDevDaxTest, umfPoolTest, ::testing::ValuesIn(poolParamsList)); diff --git a/test/pools/scalable_coarse_file.cpp b/test/pools/scalable_coarse_file.cpp index b45c112be2..30134f5eba 100644 --- a/test/pools/scalable_coarse_file.cpp +++ b/test/pools/scalable_coarse_file.cpp @@ -23,11 +23,9 @@ file_params_unique_handle_t get_file_params_default(char *path) { &umfFileMemoryProviderParamsDestroy); } -auto coarseParams = umfCoarseMemoryProviderParamsDefault(); file_params_unique_handle_t fileParams = get_file_params_default(FILE_PATH); INSTANTIATE_TEST_SUITE_P(scalableCoarseFileTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfScalablePoolOps(), nullptr, - umfFileMemoryProviderOps(), fileParams.get(), - &coarseParams})); + umfFileMemoryProviderOps(), fileParams.get()})); diff --git a/test/pools/scalable_pool.cpp b/test/pools/scalable_pool.cpp index 3edacd965c..51cc020305 100644 --- a/test/pools/scalable_pool.cpp +++ b/test/pools/scalable_pool.cpp @@ -27,8 +27,7 @@ auto defaultParams = createOsMemoryProviderParams(); INSTANTIATE_TEST_SUITE_P(scalablePoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{ umfScalablePoolOps(), nullptr, - umfOsMemoryProviderOps(), defaultParams.get(), - nullptr})); + umfOsMemoryProviderOps(), defaultParams.get()})); using scalablePoolParams = std::tuple; struct umfScalablePoolParamsTest diff --git a/test/provider_coarse.cpp b/test/provider_coarse.cpp deleted file mode 100644 index c2de4c06a9..0000000000 --- a/test/provider_coarse.cpp +++ /dev/null @@ -1,668 +0,0 @@ -/* - * Copyright (C) 2023-2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ - -#include - -#include "provider.hpp" - -#include - -using umf_test::KB; -using umf_test::MB; -using umf_test::test; - -#define GetStats umfCoarseMemoryProviderGetStats - -#define UPSTREAM_NAME "umf_ba_global" -#define BASE_NAME "coarse" -#define COARSE_NAME BASE_NAME " (" UPSTREAM_NAME ")" - -umf_memory_provider_ops_t UMF_MALLOC_MEMORY_PROVIDER_OPS = - umf::providerMakeCOps(); - -struct CoarseWithMemoryStrategyTest - : umf_test::test, - ::testing::WithParamInterface { - void SetUp() override { - test::SetUp(); - allocation_strategy = this->GetParam(); - } - - coarse_memory_provider_strategy_t allocation_strategy; -}; - -INSTANTIATE_TEST_SUITE_P( - CoarseWithMemoryStrategyTest, CoarseWithMemoryStrategyTest, - ::testing::Values(UMF_COARSE_MEMORY_STRATEGY_FASTEST, - UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, - UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE)); - -TEST_F(test, coarseProvider_name_upstream) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.destroy_upstream_memory_provider = true; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - size_t minPageSize = 0; - umf_result = umfMemoryProviderGetMinPageSize(coarse_memory_provider, - nullptr, &minPageSize); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - ASSERT_EQ(minPageSize, 0); - - size_t pageSize = 0; - umf_result = umfMemoryProviderGetRecommendedPageSize( - coarse_memory_provider, minPageSize, &pageSize); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - ASSERT_EQ(pageSize, minPageSize); - - ASSERT_EQ( - strcmp(umfMemoryProviderGetName(coarse_memory_provider), COARSE_NAME), - 0); - - umfMemoryProviderDestroy(coarse_memory_provider); - // malloc_memory_provider has already been destroyed - // by umfMemoryProviderDestroy(coarse_memory_provider), because: - // coarse_memory_provider_params.destroy_upstream_memory_provider = true; -} - -TEST_F(test, coarseProvider_name_no_upstream) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - size_t minPageSize = 0; - umf_result = umfMemoryProviderGetMinPageSize(coarse_memory_provider, - nullptr, &minPageSize); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_GT(minPageSize, 0); - - size_t pageSize = 0; - umf_result = umfMemoryProviderGetRecommendedPageSize( - coarse_memory_provider, minPageSize, &pageSize); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_GE(pageSize, minPageSize); - - ASSERT_EQ( - strcmp(umfMemoryProviderGetName(coarse_memory_provider), BASE_NAME), 0); - - umfMemoryProviderDestroy(coarse_memory_provider); -} - -// negative tests - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_null_stats) { - ASSERT_EQ(GetStats(nullptr).alloc_size, 0); - ASSERT_EQ(GetStats(nullptr).used_size, 0); - ASSERT_EQ(GetStats(nullptr).num_upstream_blocks, 0); - ASSERT_EQ(GetStats(nullptr).num_all_blocks, 0); - ASSERT_EQ(GetStats(nullptr).num_free_blocks, 0); -} - -// wrong NULL parameters -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_NULL_params) { - umf_result_t umf_result; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), nullptr, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -// wrong parameters: given no upstream_memory_provider -// nor init_buffer while exactly one of them must be set -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_0) { - umf_result_t umf_result; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = 0; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -// wrong parameters: given both an upstream_memory_provider -// and an init_buffer while only one of them is allowed -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_1) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); - - umfMemoryProviderDestroy(malloc_memory_provider); -} - -// wrong parameters: init_buffer_size must not equal 0 when immediate_init_from_upstream is true -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_2) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = nullptr; - coarse_memory_provider_params.init_buffer_size = 0; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); - - umfMemoryProviderDestroy(malloc_memory_provider); -} - -// wrong parameters: init_buffer_size must not equal 0 when init_buffer is not NULL -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_3) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = 0; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -// wrong parameters: init_buffer_size must equal 0 when init_buffer is NULL and immediate_init_from_upstream is false -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_4) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = 20 * MB; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); - - umfMemoryProviderDestroy(malloc_memory_provider); -} - -// wrong parameters: destroy_upstream_memory_provider is true, but an upstream provider is not provided -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_wrong_params_5) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.destroy_upstream_memory_provider = true; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - ASSERT_EQ(coarse_memory_provider, nullptr); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_split_merge) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_memory_provider_handle_t cp = coarse_memory_provider; - char *ptr = nullptr; - - ASSERT_EQ(GetStats(cp).used_size, 0 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - /* test umfMemoryProviderAllocationSplit */ - umf_result = umfMemoryProviderAlloc(cp, 2 * MB, 0, (void **)&ptr); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 2 * MB, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - umf_result = umfMemoryProviderFree(cp, (ptr + 1 * MB), 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 1 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderFree(cp, ptr, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 0); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - /* test umfMemoryProviderAllocationMerge */ - umf_result = umfMemoryProviderAlloc(cp, 2 * MB, 0, (void **)&ptr); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 2 * MB, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 1 * MB), 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 2 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - umf_result = umfMemoryProviderFree(cp, ptr, 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 0); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_split_merge_negative) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - umf_memory_provider_handle_t cp = coarse_memory_provider; - char *ptr = nullptr; - - ASSERT_EQ(GetStats(cp).used_size, 0 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - /* test umfMemoryProviderAllocationSplit */ - umf_result = umfMemoryProviderAlloc(cp, 6 * MB, 0, (void **)&ptr); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(GetStats(cp).used_size, 6 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 2); - - // firstSize >= totalSize - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 6 * MB, 6 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // firstSize == 0 - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 6 * MB, 0); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // wrong totalSize - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 5 * MB, 1 * KB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - /* test umfMemoryProviderAllocationMerge */ - // split (6 * MB) block into (1 * MB) + (5 * MB) - umf_result = umfMemoryProviderAllocationSplit(cp, ptr, 6 * MB, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 6 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - // split (5 * MB) block into (2 * MB) + (3 * MB) - umf_result = - umfMemoryProviderAllocationSplit(cp, (ptr + 1 * MB), 5 * MB, 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 6 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 4); - - // now we have 3 blocks: (1 * MB) + (2 * MB) + (3 * MB) - - // highPtr <= lowPtr - umf_result = - umfMemoryProviderAllocationMerge(cp, (ptr + 1 * MB), ptr, 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // highPtr - lowPtr >= totalSize - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 1 * MB), 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // low_block->size + high_block->size != totalSize - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 1 * MB), 5 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // not adjacent blocks - umf_result = - umfMemoryProviderAllocationMerge(cp, ptr, (ptr + 3 * MB), 4 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - umf_result = umfMemoryProviderFree(cp, ptr, 1 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 5 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 4); - - umf_result = umfMemoryProviderFree(cp, (ptr + 1 * MB), 2 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 3 * MB); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 3); - - umf_result = umfMemoryProviderFree(cp, (ptr + 3 * MB), 3 * MB); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_EQ(GetStats(cp).used_size, 0); - ASSERT_EQ(GetStats(cp).alloc_size, init_buffer_size); - ASSERT_EQ(GetStats(cp).num_all_blocks, 1); - - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_purge_no_upstream) { - umf_result_t umf_result; - - const size_t init_buffer_size = 20 * MB; - - // preallocate some memory and initialize the vector with zeros - std::vector buffer(init_buffer_size, 0); - void *buf = (void *)buffer.data(); - ASSERT_NE(buf, nullptr); - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.allocation_strategy = allocation_strategy; - coarse_memory_provider_params.upstream_memory_provider = nullptr; - coarse_memory_provider_params.immediate_init_from_upstream = false; - coarse_memory_provider_params.init_buffer = buf; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider = nullptr; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - // umfMemoryProviderPurgeLazy - // provider == NULL - umf_result = umfMemoryProviderPurgeLazy(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = umfMemoryProviderPurgeLazy(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // no upstream_memory_provider - umf_result = - umfMemoryProviderPurgeLazy(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); - - // umfMemoryProviderPurgeForce - // provider == NULL - umf_result = umfMemoryProviderPurgeForce(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // no upstream_memory_provider - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); - - umfMemoryProviderDestroy(coarse_memory_provider); -} - -TEST_P(CoarseWithMemoryStrategyTest, coarseProvider_purge_with_upstream) { - umf_memory_provider_handle_t malloc_memory_provider; - umf_result_t umf_result; - - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(malloc_memory_provider, nullptr); - - const size_t init_buffer_size = 20 * MB; - - coarse_memory_provider_params_t coarse_memory_provider_params; - // make sure there are no undefined members - prevent a UB - memset(&coarse_memory_provider_params, 0, - sizeof(coarse_memory_provider_params)); - coarse_memory_provider_params.upstream_memory_provider = - malloc_memory_provider; - coarse_memory_provider_params.immediate_init_from_upstream = true; - coarse_memory_provider_params.init_buffer = NULL; - coarse_memory_provider_params.init_buffer_size = init_buffer_size; - - umf_memory_provider_handle_t coarse_memory_provider; - umf_result = umfMemoryProviderCreate(umfCoarseMemoryProviderOps(), - &coarse_memory_provider_params, - &coarse_memory_provider); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - ASSERT_NE(coarse_memory_provider, nullptr); - - // umfMemoryProviderPurgeLazy - // provider == NULL - umf_result = umfMemoryProviderPurgeLazy(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = umfMemoryProviderPurgeLazy(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // malloc_memory_provider returns UMF_RESULT_ERROR_UNKNOWN - umf_result = - umfMemoryProviderPurgeLazy(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - - // umfMemoryProviderPurgeForce - // provider == NULL - umf_result = umfMemoryProviderPurgeForce(nullptr, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // ptr == NULL - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, nullptr, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - // malloc_memory_provider returns UMF_RESULT_ERROR_UNKNOWN - umf_result = - umfMemoryProviderPurgeForce(coarse_memory_provider, (void *)0x01, 1); - ASSERT_EQ(umf_result, UMF_RESULT_ERROR_UNKNOWN); - - umfMemoryProviderDestroy(coarse_memory_provider); - umfMemoryProviderDestroy(malloc_memory_provider); -} From 450c5a96d7cc52e29d7c411b0e09b27df0f879f8 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 11:55:03 +0100 Subject: [PATCH 26/82] Replace all NULL with nullptr in disjointCoarseMallocPool.cpp Signed-off-by: Lukasz Dorau --- test/disjointCoarseMallocPool.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/disjointCoarseMallocPool.cpp b/test/disjointCoarseMallocPool.cpp index 45502b1923..383487a87e 100644 --- a/test/disjointCoarseMallocPool.cpp +++ b/test/disjointCoarseMallocPool.cpp @@ -43,8 +43,8 @@ TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple1) { umf_memory_provider_handle_t malloc_memory_provider = nullptr; umf_result_t umf_result; - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, + nullptr, &malloc_memory_provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(malloc_memory_provider, nullptr); @@ -62,7 +62,7 @@ TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple1) { umf_result = umfFileMemoryProviderParamsDestroy(file_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; + umf_disjoint_pool_params_handle_t disjoint_pool_params = nullptr; umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(disjoint_pool_params, nullptr); @@ -89,7 +89,7 @@ TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple1) { umf_result = umfDisjointPoolParamsDestroy(disjoint_pool_params); - umf_memory_provider_handle_t prov = NULL; + umf_memory_provider_handle_t prov = nullptr; umfPoolGetMemoryProvider(pool, &prov); ASSERT_NE(prov, nullptr); @@ -138,8 +138,8 @@ TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple2) { umf_memory_provider_handle_t malloc_memory_provider = nullptr; umf_result_t umf_result; - umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, - &malloc_memory_provider); + umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, + nullptr, &malloc_memory_provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(malloc_memory_provider, nullptr); @@ -157,7 +157,7 @@ TEST_P(FileWithMemoryStrategyTest, disjointFileMallocPool_simple2) { umf_result = umfFileMemoryProviderParamsDestroy(file_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; + umf_disjoint_pool_params_handle_t disjoint_pool_params = nullptr; umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(disjoint_pool_params, nullptr); @@ -245,7 +245,7 @@ TEST_P(FileWithMemoryStrategyTest, disjointFileMMapPool_random) { umf_result = umfFileMemoryProviderParamsDestroy(file_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - umf_disjoint_pool_params_handle_t disjoint_pool_params = NULL; + umf_disjoint_pool_params_handle_t disjoint_pool_params = nullptr; umf_result = umfDisjointPoolParamsCreate(&disjoint_pool_params); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(disjoint_pool_params, nullptr); From 7fec989edd71dd7cc2a4dc26bccd3d12423e0daf Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 5 Dec 2024 15:55:48 +0100 Subject: [PATCH 27/82] Rename disjointCoarseMallocPool test to disjointPoolFileProv Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 5 +++-- ...disjointCoarseMallocPool.cpp => disjointPoolFileProv.cpp} | 0 2 files changed, 3 insertions(+), 2 deletions(-) rename test/{disjointCoarseMallocPool.cpp => disjointPoolFileProv.cpp} (100%) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 593268a52c..d5a07bfbbf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -214,9 +214,10 @@ if(UMF_BUILD_LIBUMF_POOL_DISJOINT) SRCS c_api/disjoint_pool.c LIBS disjoint_pool) if(LINUX AND (NOT UMF_DISABLE_HWLOC)) + # this test uses the file provider add_umf_test( - NAME disjointCoarseMallocPool - SRCS disjointCoarseMallocPool.cpp + NAME disjointPoolFileProv + SRCS disjointPoolFileProv.cpp LIBS disjoint_pool) endif() endif() diff --git a/test/disjointCoarseMallocPool.cpp b/test/disjointPoolFileProv.cpp similarity index 100% rename from test/disjointCoarseMallocPool.cpp rename to test/disjointPoolFileProv.cpp From 86d23413c3374d3020725811422baca770f8962c Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 4 Dec 2024 21:52:56 +0000 Subject: [PATCH 28/82] Implement umfPool[Set/Get]Tag Implements https://github.com/oneapi-src/unified-memory-framework/issues/687 --- include/umf/memory_pool.h | 16 ++++++ src/libumf.def | 2 + src/libumf.map | 2 + src/memory_pool.c | 32 +++++++++++ src/memory_pool_internal.h | 4 ++ test/memoryPoolAPI.cpp | 115 +++++++++++++++++++++++++++++++++++++ 6 files changed, 171 insertions(+) diff --git a/include/umf/memory_pool.h b/include/umf/memory_pool.h index a93d400f92..de045acf49 100644 --- a/include/umf/memory_pool.h +++ b/include/umf/memory_pool.h @@ -170,6 +170,22 @@ umf_memory_pool_handle_t umfPoolByPtr(const void *ptr); umf_result_t umfPoolGetMemoryProvider(umf_memory_pool_handle_t hPool, umf_memory_provider_handle_t *hProvider); +/// +/// @brief Set a custom tag on the memory pool that can be later retrieved using umfPoolGetTag. +/// @param hPool specified memory pool +/// @param tag tag to be set +/// @param oldTag [out][optional] previous tag set on the memory pool +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfPoolSetTag(umf_memory_pool_handle_t hPool, void *tag, + void **oldTag); + +/// +/// @brief Retrieve the tag associated with the memory pool or NULL if no tag is set. +/// @param hPool specified memory pool +/// @param tag [out] tag associated with the memory pool +/// @return UMF_RESULT_SUCCESS on success. +umf_result_t umfPoolGetTag(umf_memory_pool_handle_t hPool, void **tag); + #ifdef __cplusplus } #endif diff --git a/src/libumf.def b/src/libumf.def index 0b4588bb81..f0f38ee15c 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -104,10 +104,12 @@ EXPORTS umfPoolFree umfPoolGetIPCHandleSize umfPoolGetLastAllocationError + umfPoolGetTag umfPoolGetMemoryProvider umfPoolMalloc umfPoolMallocUsableSize umfPoolRealloc + umfPoolSetTag umfProxyPoolOps umfPutIPCHandle umfScalablePoolOps diff --git a/src/libumf.map b/src/libumf.map index 41467bad59..fd1d48d348 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -99,9 +99,11 @@ UMF_1.0 { umfPoolGetIPCHandleSize; umfPoolGetLastAllocationError; umfPoolGetMemoryProvider; + umfPoolGetTag; umfPoolMalloc; umfPoolMallocUsableSize; umfPoolRealloc; + umfPoolSetTag; umfProxyPoolOps; umfPutIPCHandle; umfScalablePoolOps; diff --git a/src/memory_pool.c b/src/memory_pool.c index 4a85955efa..f4289c2151 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -55,6 +55,13 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, pool->flags = flags; pool->ops = *ops; + pool->tag = NULL; + + if (NULL == utils_mutex_init(&pool->lock)) { + LOG_ERR("Failed to initialize mutex for pool"); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto err_lock_init; + } ret = ops->initialize(pool->provider, params, &pool->pool_priv); if (ret != UMF_RESULT_SUCCESS) { @@ -66,6 +73,8 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, return UMF_RESULT_SUCCESS; err_pool_init: + utils_mutex_destroy_not_free(&pool->lock); +err_lock_init: if (!(flags & UMF_POOL_CREATE_FLAG_DISABLE_TRACKING)) { umfMemoryProviderDestroy(pool->provider); } @@ -90,6 +99,8 @@ void umfPoolDestroy(umf_memory_pool_handle_t hPool) { umfMemoryProviderDestroy(hUpstreamProvider); } + utils_mutex_destroy_not_free(&hPool->lock); + LOG_INFO("Memory pool destroyed: %p", (void *)hPool); // TODO: this free keeps memory in base allocator, so it can lead to OOM in some scenarios (it should be optimized) @@ -175,3 +186,24 @@ umf_result_t umfPoolGetLastAllocationError(umf_memory_pool_handle_t hPool) { UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); return hPool->ops.get_last_allocation_error(hPool->pool_priv); } + +umf_result_t umfPoolSetTag(umf_memory_pool_handle_t hPool, void *tag, + void **oldTag) { + UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + utils_mutex_lock(&hPool->lock); + if (oldTag) { + *oldTag = hPool->tag; + } + hPool->tag = tag; + utils_mutex_unlock(&hPool->lock); + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfPoolGetTag(umf_memory_pool_handle_t hPool, void **tag) { + UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((tag != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + utils_mutex_lock(&hPool->lock); + *tag = hPool->tag; + utils_mutex_unlock(&hPool->lock); + return UMF_RESULT_SUCCESS; +} diff --git a/src/memory_pool_internal.h b/src/memory_pool_internal.h index 90f2f16298..e556ace214 100644 --- a/src/memory_pool_internal.h +++ b/src/memory_pool_internal.h @@ -22,6 +22,7 @@ extern "C" { #endif #include "base_alloc.h" +#include "utils_concurrency.h" typedef struct umf_memory_pool_t { void *pool_priv; @@ -30,6 +31,9 @@ typedef struct umf_memory_pool_t { // Memory provider used by the pool. umf_memory_provider_handle_t provider; + + utils_mutex_t lock; + void *tag; } umf_memory_pool_t; #ifdef __cplusplus diff --git a/test/memoryPoolAPI.cpp b/test/memoryPoolAPI.cpp index 1c6d83f2af..ec137b5493 100644 --- a/test/memoryPoolAPI.cpp +++ b/test/memoryPoolAPI.cpp @@ -178,6 +178,121 @@ TEST_F(test, BasicPoolByPtrTest) { ASSERT_EQ(ret, UMF_RESULT_SUCCESS); } +struct tagTest : umf_test::test { + void SetUp() override { + test::SetUp(); + provider = umf_test::wrapProviderUnique(nullProviderCreate()); + pool = umf_test::wrapPoolUnique( + createPoolChecked(umfProxyPoolOps(), provider.get(), nullptr)); + } + + umf::provider_unique_handle_t provider; + umf::pool_unique_handle_t pool; +}; + +TEST_F(tagTest, SetAndGet) { + umf_result_t ret = umfPoolSetTag(pool.get(), (void *)0x99, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + void *tag; + ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, (void *)0x99); + + void *oldTag; + ret = umfPoolSetTag(pool.get(), (void *)0x100, &oldTag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(oldTag, (void *)0x99); + + ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, (void *)0x100); +} + +TEST_F(tagTest, SetAndGetNull) { + umf_result_t ret = umfPoolSetTag(pool.get(), nullptr, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + void *tag; + ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, nullptr); +} + +TEST_F(tagTest, NoSetAndGet) { + void *tag; + umf_result_t ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(tag, nullptr); +} + +TEST_F(tagTest, SetAndGetMt) { + static constexpr size_t NUM_THREADS = 8; + static constexpr size_t NUM_OPS_PER_THREAD = 16; + + std::vector threads; + + auto encodeTag = [](size_t thread, size_t op) -> void * { + return reinterpret_cast(thread * NUM_OPS_PER_THREAD + op); + }; + + auto decodeTag = [](void *tag) -> std::pair { + auto op = reinterpret_cast(tag) & (NUM_OPS_PER_THREAD - 1); + auto thread = reinterpret_cast(tag) / NUM_OPS_PER_THREAD; + return {thread, op}; + }; + + for (size_t i = 0; i < NUM_THREADS; i++) { + threads.emplace_back([this, i, encodeTag, decodeTag] { + for (size_t j = 0; j < NUM_OPS_PER_THREAD; j++) { + void *oldTag; + umf_result_t ret = + umfPoolSetTag(pool.get(), encodeTag(i, j), &oldTag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + void *queriedTag; + ret = umfPoolGetTag(pool.get(), &queriedTag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + auto [t1, op1] = decodeTag(oldTag); + auto [t2, op2] = decodeTag(queriedTag); + // if the tag was set by the same thread, the op part should be the same or higher + ASSERT_TRUE(t1 != t2 || op2 >= op1); + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + void *tag; + auto ret = umfPoolGetTag(pool.get(), &tag); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + auto [t, op] = decodeTag(tag); + ASSERT_TRUE(t < NUM_THREADS); + ASSERT_TRUE(op == NUM_OPS_PER_THREAD - 1); +} + +TEST_F(tagTest, SetAndGetInvalidPtr) { + umf_result_t ret = umfPoolSetTag(pool.get(), nullptr, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ret = umfPoolGetTag(pool.get(), nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(tagTest, SetAndGetInvalidPool) { + umf_result_t ret = + umfPoolSetTag(nullptr, reinterpret_cast(0x1), nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + void *tag; + ret = umfPoolGetTag(nullptr, &tag); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + INSTANTIATE_TEST_SUITE_P( mallocPoolTest, umfPoolTest, ::testing::Values(poolCreateExtParams{&MALLOC_POOL_OPS, nullptr, From 4394ed70aae99c35dabaaf201c679696783d0739 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 6 Dec 2024 08:09:32 +0100 Subject: [PATCH 29/82] Remove the disable_provider_free parameter of jemalloc pool Remove the disable_provider_free parameter of jemalloc pool and all umfJemallocPoolParams*() API. Fixes: #904 Signed-off-by: Lukasz Dorau --- README.md | 12 --- examples/dram_and_fsdax/dram_and_fsdax.c | 31 +----- include/umf/pools/pool_jemalloc.h | 27 ----- src/libumf.def | 3 - src/libumf.map | 3 - src/pool/pool_jemalloc.c | 96 +----------------- test/pools/jemalloc_pool.cpp | 119 ----------------------- 7 files changed, 4 insertions(+), 287 deletions(-) diff --git a/README.md b/README.md index 81a82bfab1..0c569c1b3c 100644 --- a/README.md +++ b/README.md @@ -209,12 +209,6 @@ Additionally, required for tests: A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y). It can be used when large memory mappings are needed. -The DevDax memory provider does not support the free operation -(`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), -so it should be used with a pool manager that will take over -the managing of the provided memory - for example the jemalloc pool -with the `disable_provider_free` parameter set to true. - ##### Requirements 1) Linux OS @@ -224,12 +218,6 @@ with the `disable_provider_free` parameter set to true. A memory provider that provides memory by mapping a regular, extendable file. -The file memory provider does not support the free operation -(`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), -so it should be used with a pool manager that will take over -the managing of the provided memory - for example the jemalloc pool -with the `disable_provider_free` parameter set to true. - IPC API requires the `UMF_MEM_MAP_SHARED` memory `visibility` mode (`UMF_RESULT_ERROR_INVALID_ARGUMENT` is returned otherwise). diff --git a/examples/dram_and_fsdax/dram_and_fsdax.c b/examples/dram_and_fsdax/dram_and_fsdax.c index 26f4517281..970242e109 100644 --- a/examples/dram_and_fsdax/dram_and_fsdax.c +++ b/examples/dram_and_fsdax/dram_and_fsdax.c @@ -78,41 +78,14 @@ static umf_memory_pool_handle_t create_fsdax_pool(const char *path) { } // Create an FSDAX memory pool - // - // The file memory provider does not support the free operation - // (`umfMemoryProviderFree()` always returns `UMF_RESULT_ERROR_NOT_SUPPORTED`), - // so it should be used with a pool manager that will take over - // the managing of the provided memory - for example the jemalloc pool - // with the `disable_provider_free` parameter set to true. - umf_jemalloc_pool_params_handle_t pool_params; - umf_result = umfJemallocPoolParamsCreate(&pool_params); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, "Failed to create jemalloc params!\n"); - umfMemoryProviderDestroy(provider_fsdax); - return NULL; - } - umf_result = umfJemallocPoolParamsSetKeepAllMemory(pool_params, true); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, "Failed to set KeepAllMemory!\n"); - umfMemoryProviderDestroy(provider_fsdax); - return NULL; - } - - // Create an FSDAX memory pool - umf_result = - umfPoolCreate(umfJemallocPoolOps(), provider_fsdax, pool_params, - UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool_fsdax); + umf_result = umfPoolCreate(umfJemallocPoolOps(), provider_fsdax, NULL, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool_fsdax); if (umf_result != UMF_RESULT_SUCCESS) { fprintf(stderr, "Failed to create an FSDAX memory pool!\n"); umfMemoryProviderDestroy(provider_fsdax); return NULL; } - umf_result = umfJemallocPoolParamsDestroy(pool_params); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, "Failed to destroy jemalloc params!\n"); - } - return pool_fsdax; } diff --git a/include/umf/pools/pool_jemalloc.h b/include/umf/pools/pool_jemalloc.h index 0cbecd38f7..c30df65092 100644 --- a/include/umf/pools/pool_jemalloc.h +++ b/include/umf/pools/pool_jemalloc.h @@ -14,35 +14,8 @@ extern "C" { #endif -#include #include -struct umf_jemalloc_pool_params_t; - -/// @brief handle to the parameters of the jemalloc pool. -typedef struct umf_jemalloc_pool_params_t *umf_jemalloc_pool_params_handle_t; - -/// @brief Create a struct to store parameters of jemalloc pool. -/// @param hParams [out] handle to the newly created parameters struct. -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. -umf_result_t -umfJemallocPoolParamsCreate(umf_jemalloc_pool_params_handle_t *hParams); - -/// @brief Destroy parameters struct. -/// @param hParams handle to the parameters of the jemalloc pool. -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. -umf_result_t -umfJemallocPoolParamsDestroy(umf_jemalloc_pool_params_handle_t hParams); - -/// @brief Set if \p umfMemoryProviderFree() should never be called. -/// @param hParams handle to the parameters of the jemalloc pool. -/// @param keepAllMemory \p true if the jemalloc pool should not call -/// \p umfMemoryProviderFree, \p false otherwise. -/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. -umf_result_t -umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, - bool keepAllMemory); - umf_memory_pool_ops_t *umfJemallocPoolOps(void); #ifdef __cplusplus diff --git a/src/libumf.def b/src/libumf.def index f2b24be6c2..3c0b63ce52 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -37,9 +37,6 @@ EXPORTS umfGetIPCHandle umfGetLastFailedMemoryProvider umfJemallocPoolOps - umfJemallocPoolParamsCreate - umfJemallocPoolParamsDestroy - umfJemallocPoolParamsSetKeepAllMemory umfLevelZeroMemoryProviderOps umfLevelZeroMemoryProviderParamsCreate umfLevelZeroMemoryProviderParamsDestroy diff --git a/src/libumf.map b/src/libumf.map index 067ec88386..85a9042203 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -31,9 +31,6 @@ UMF_1.0 { umfGetIPCHandle; umfGetLastFailedMemoryProvider; umfJemallocPoolOps; - umfJemallocPoolParamsCreate; - umfJemallocPoolParamsDestroy; - umfJemallocPoolParamsSetKeepAllMemory; umfLevelZeroMemoryProviderOps; umfLevelZeroMemoryProviderParamsCreate; umfLevelZeroMemoryProviderParamsDestroy; diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index 47bc6497fc..f88d5ce9d7 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -24,26 +24,6 @@ umf_memory_pool_ops_t *umfJemallocPoolOps(void) { return NULL; } -umf_result_t -umfJemallocPoolParamsCreate(umf_jemalloc_pool_params_handle_t *hParams) { - (void)hParams; // unused - return UMF_RESULT_ERROR_NOT_SUPPORTED; -} - -umf_result_t -umfJemallocPoolParamsDestroy(umf_jemalloc_pool_params_handle_t hParams) { - (void)hParams; // unused - return UMF_RESULT_ERROR_NOT_SUPPORTED; -} - -umf_result_t -umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, - bool keepAllMemory) { - (void)hParams; // unused - (void)keepAllMemory; // unused - return UMF_RESULT_ERROR_NOT_SUPPORTED; -} - #else #include @@ -53,16 +33,8 @@ umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, typedef struct jemalloc_memory_pool_t { umf_memory_provider_handle_t provider; unsigned int arena_index; // index of jemalloc arena - // set to true if umfMemoryProviderFree() should never be called - bool disable_provider_free; } jemalloc_memory_pool_t; -// Configuration of Jemalloc Pool -typedef struct umf_jemalloc_pool_params_t { - /// Set to true if umfMemoryProviderFree() should never be called. - bool disable_provider_free; -} umf_jemalloc_pool_params_t; - static __TLS umf_result_t TLS_last_allocation_error; static jemalloc_memory_pool_t *pool_by_arena_index[MALLCTL_ARENAS_ALL]; @@ -75,52 +47,6 @@ static jemalloc_memory_pool_t *get_pool_by_arena_index(unsigned arena_ind) { return pool_by_arena_index[arena_ind]; } -umf_result_t -umfJemallocPoolParamsCreate(umf_jemalloc_pool_params_handle_t *hParams) { - if (!hParams) { - LOG_ERR("jemalloc pool params handle is NULL"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - umf_jemalloc_pool_params_t *params_data = - umf_ba_global_alloc(sizeof(*params_data)); - if (!params_data) { - LOG_ERR("cannot allocate memory for jemalloc poolparams"); - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - params_data->disable_provider_free = false; - - *hParams = (umf_jemalloc_pool_params_handle_t)params_data; - - return UMF_RESULT_SUCCESS; -} - -umf_result_t -umfJemallocPoolParamsDestroy(umf_jemalloc_pool_params_handle_t hParams) { - if (!hParams) { - LOG_ERR("jemalloc pool params handle is NULL"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - umf_ba_global_free(hParams); - - return UMF_RESULT_SUCCESS; -} - -umf_result_t -umfJemallocPoolParamsSetKeepAllMemory(umf_jemalloc_pool_params_handle_t hParams, - bool keepAllMemory) { - if (!hParams) { - LOG_ERR("jemalloc pool params handle is NULL"); - return UMF_RESULT_ERROR_INVALID_ARGUMENT; - } - - hParams->disable_provider_free = keepAllMemory; - - return UMF_RESULT_SUCCESS; -} - // arena_extent_alloc - an extent allocation function conforms to the extent_alloc_t type and upon // success returns a pointer to size bytes of mapped memory on behalf of arena arena_ind such that // the extent's base address is a multiple of alignment, as well as setting *zero to indicate @@ -150,9 +76,7 @@ static void *arena_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, } if (new_addr != NULL && ptr != new_addr) { - if (!pool->disable_provider_free) { - umfMemoryProviderFree(pool->provider, ptr, size); - } + umfMemoryProviderFree(pool->provider, ptr, size); return NULL; } @@ -186,10 +110,6 @@ static void arena_extent_destroy(extent_hooks_t *extent_hooks, void *addr, jemalloc_memory_pool_t *pool = get_pool_by_arena_index(arena_ind); - if (pool->disable_provider_free) { - return; - } - umf_result_t ret; ret = umfMemoryProviderFree(pool->provider, addr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -212,10 +132,6 @@ static bool arena_extent_dalloc(extent_hooks_t *extent_hooks, void *addr, jemalloc_memory_pool_t *pool = get_pool_by_arena_index(arena_ind); - if (pool->disable_provider_free) { - return true; // opt-out from deallocation - } - umf_result_t ret; ret = umfMemoryProviderFree(pool->provider, addr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -466,12 +382,10 @@ static void *op_aligned_alloc(void *pool, size_t size, size_t alignment) { static umf_result_t op_initialize(umf_memory_provider_handle_t provider, void *params, void **out_pool) { + (void)params; // unused assert(provider); assert(out_pool); - umf_jemalloc_pool_params_handle_t je_params = - (umf_jemalloc_pool_params_handle_t)params; - extent_hooks_t *pHooks = &arena_extent_hooks; size_t unsigned_size = sizeof(unsigned); int err; @@ -484,12 +398,6 @@ static umf_result_t op_initialize(umf_memory_provider_handle_t provider, pool->provider = provider; - if (je_params) { - pool->disable_provider_free = je_params->disable_provider_free; - } else { - pool->disable_provider_free = false; - } - unsigned arena_index; err = je_mallctl("arenas.create", (void *)&arena_index, &unsigned_size, NULL, 0); diff --git a/test/pools/jemalloc_pool.cpp b/test/pools/jemalloc_pool.cpp index 4dddbcd32b..96c3868959 100644 --- a/test/pools/jemalloc_pool.cpp +++ b/test/pools/jemalloc_pool.cpp @@ -62,122 +62,3 @@ TEST_F(test, metadataNotAllocatedUsingProvider) { [pool = pool.get()](void *ptr) { umfPoolFree(pool, ptr); }); } } - -using jemallocPoolParams = bool; -struct umfJemallocPoolParamsTest - : umf_test::test, - ::testing::WithParamInterface { - - struct validation_params_t { - bool keep_all_memory; - }; - - struct provider_validator : public umf_test::provider_ba_global { - using base_provider = umf_test::provider_ba_global; - - umf_result_t initialize(validation_params_t *params) { - EXPECT_NE(params, nullptr); - expected_params = params; - return UMF_RESULT_SUCCESS; - } - umf_result_t free(void *ptr, size_t size) { - EXPECT_EQ(expected_params->keep_all_memory, false); - return base_provider::free(ptr, size); - } - - validation_params_t *expected_params; - }; - - static constexpr umf_memory_provider_ops_t VALIDATOR_PROVIDER_OPS = - umf::providerMakeCOps(); - - umfJemallocPoolParamsTest() : expected_params{false}, params(nullptr) {} - void SetUp() override { - test::SetUp(); - expected_params.keep_all_memory = this->GetParam(); - umf_result_t ret = umfJemallocPoolParamsCreate(¶ms); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - ret = umfJemallocPoolParamsSetKeepAllMemory( - params, expected_params.keep_all_memory); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - } - - void TearDown() override { - umfJemallocPoolParamsDestroy(params); - test::TearDown(); - } - - umf::pool_unique_handle_t makePool() { - umf_memory_provider_handle_t hProvider = nullptr; - umf_memory_pool_handle_t hPool = nullptr; - - auto ret = umfMemoryProviderCreate(&VALIDATOR_PROVIDER_OPS, - &expected_params, &hProvider); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - - ret = umfPoolCreate(umfJemallocPoolOps(), hProvider, params, - UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &hPool); - EXPECT_EQ(ret, UMF_RESULT_SUCCESS); - - return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); - } - - void allocFreeFlow() { - static const size_t ALLOC_SIZE = 128; - static const size_t NUM_ALLOCATIONS = 100; - std::vector ptrs; - - auto pool = makePool(); - ASSERT_NE(pool, nullptr); - - for (size_t i = 0; i < NUM_ALLOCATIONS; ++i) { - auto *ptr = umfPoolMalloc(pool.get(), ALLOC_SIZE); - ASSERT_NE(ptr, nullptr); - ptrs.push_back(ptr); - } - - for (size_t i = 0; i < NUM_ALLOCATIONS; ++i) { - auto ret = umfPoolFree(pool.get(), ptrs[i]); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - } - - // Now pool can call free during pool destruction - expected_params.keep_all_memory = false; - } - - validation_params_t expected_params; - umf_jemalloc_pool_params_handle_t params; -}; - -TEST_P(umfJemallocPoolParamsTest, allocFree) { allocFreeFlow(); } - -TEST_P(umfJemallocPoolParamsTest, updateParams) { - expected_params.keep_all_memory = !expected_params.keep_all_memory; - umf_result_t ret = umfJemallocPoolParamsSetKeepAllMemory( - params, expected_params.keep_all_memory); - ASSERT_EQ(ret, UMF_RESULT_SUCCESS); - - allocFreeFlow(); -} - -TEST_P(umfJemallocPoolParamsTest, invalidParams) { - umf_result_t ret = umfJemallocPoolParamsCreate(nullptr); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - ret = umfJemallocPoolParamsSetKeepAllMemory(nullptr, true); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - ret = umfJemallocPoolParamsSetKeepAllMemory(nullptr, false); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); - - ret = umfJemallocPoolParamsDestroy(nullptr); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); -} - -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfJemallocPoolParamsTest); - -/* TODO: enable this test after the issue #903 is fixed. -(https://github.com/oneapi-src/unified-memory-framework/issues/903) -INSTANTIATE_TEST_SUITE_P(jemallocPoolTest, umfJemallocPoolParamsTest, - testing::Values(false, true)); -*/ From 7d1f1d03043f008dbde7ce99d5f690571c66ff30 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 6 Dec 2024 08:26:41 +0100 Subject: [PATCH 30/82] Ignore return value of `je_mallctl()` in an error handling path Ignore return value of `je_mallctl()` in an error handling path, because we cannot do nothing more with this error. It fixes two Coverity issues. Signed-off-by: Lukasz Dorau --- src/pool/pool_jemalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index 47bc6497fc..673352b185 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -504,7 +504,7 @@ static umf_result_t op_initialize(umf_memory_provider_handle_t provider, err = je_mallctl(cmd, NULL, NULL, (void *)&pHooks, sizeof(void *)); if (err) { snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_index); - je_mallctl(cmd, NULL, 0, NULL, 0); + (void)je_mallctl(cmd, NULL, 0, NULL, 0); LOG_ERR("Could not setup extent_hooks for newly created arena."); goto err_free_pool; } @@ -528,7 +528,7 @@ static void op_finalize(void *pool) { jemalloc_memory_pool_t *je_pool = (jemalloc_memory_pool_t *)pool; char cmd[64]; snprintf(cmd, sizeof(cmd), "arena.%u.destroy", je_pool->arena_index); - je_mallctl(cmd, NULL, 0, NULL, 0); + (void)je_mallctl(cmd, NULL, 0, NULL, 0); pool_by_arena_index[je_pool->arena_index] = NULL; umf_ba_global_free(je_pool); From 4dbe19c1f400b9539a670e05b2d14693032bf99f Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 6 Dec 2024 08:49:09 +0100 Subject: [PATCH 31/82] Fix building the custom jemalloc The "make install" step is executed always and all UMF tests are re-linked also always now because of an incorrect dependence. This patch fixes that. Signed-off-by: Lukasz Dorau --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fcfcbb952..82381f5b5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,13 +164,13 @@ else() add_custom_command( COMMAND make WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} - OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la + OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.a DEPENDS ${jemalloc_targ_SOURCE_DIR}/Makefile) add_custom_command( COMMAND make install WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} OUTPUT ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a - DEPENDS ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la) + DEPENDS ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.a) add_custom_target(jemalloc_prod DEPENDS ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a) From 2fca364ae3d116395d813380c23157837d3e57f6 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 6 Dec 2024 17:11:14 +0100 Subject: [PATCH 32/82] Fix: remove incorrect assert in utils_align_ptr_up_size_down() Remove incorrect assert in utils_align_ptr_up_size_down(). A pointer is aligned, but a size is only adjusted to the new pointer. The size does not have to be aligned. Signed-off-by: Lukasz Dorau --- src/utils/utils_common.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/utils_common.c b/src/utils/utils_common.c index bffc9f3559..eaf5420fc6 100644 --- a/src/utils/utils_common.c +++ b/src/utils/utils_common.c @@ -25,7 +25,6 @@ void utils_align_ptr_up_size_down(void **ptr, size_t *size, size_t alignment) { } ASSERT(IS_ALIGNED(p, alignment)); - ASSERT(IS_ALIGNED(s, alignment)); *ptr = (void *)p; *size = s; From 6f274ec8510b6c153ad87cad94adfc8805d0f0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Mon, 9 Dec 2024 13:56:43 +0100 Subject: [PATCH 33/82] set fixed iteration count for benchmarks --- benchmark/benchmark.hpp | 5 ++++- benchmark/benchmark_interfaces.hpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index ead6b39e75..ad9ab7cc83 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -95,7 +95,8 @@ struct alloc_data { ->ArgNames( \ BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::argsName()) \ ->Name(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::name()) \ - ->MinWarmUpTime(1) + ->Iterations( \ + BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::iterations()) class fixed_alloc_size : public alloc_size_interface { public: @@ -238,6 +239,7 @@ class alloc_benchmark : public benchmark_interface { return res; } static std::string name() { return base::name() + "/alloc"; } + static int64_t iterations() { return 200000; } protected: using base = benchmark_interface; @@ -324,6 +326,7 @@ class multiple_malloc_free_benchmark : public alloc_benchmark { res.insert(res.end(), n.begin(), n.end()); return res; } + static int64_t iterations() { return 2000; } std::default_random_engine generator; distribution dist; }; diff --git a/benchmark/benchmark_interfaces.hpp b/benchmark/benchmark_interfaces.hpp index 8681160626..e25c977710 100644 --- a/benchmark/benchmark_interfaces.hpp +++ b/benchmark/benchmark_interfaces.hpp @@ -55,7 +55,7 @@ struct benchmark_interface : public benchmark::Fixture { return res; } static std::string name() { return Allocator::name(); } - + static int64_t iterations() { return 10000; } Size alloc_size; Allocator allocator; }; From 4b09af0ba6b9b55f3b8bcc9fd10d2626825424f4 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 9 Dec 2024 11:25:24 +0100 Subject: [PATCH 34/82] Make coarse_alloc() return aligned address when alignment==0 Make coarse_alloc() always return address aligned to coarse->page_size when alignment==0. Signed-off-by: Lukasz Dorau --- src/coarse/coarse.c | 36 ++++++++++++++++++++++--------- test/coarse_lib.cpp | 52 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/coarse/coarse.c b/src/coarse/coarse.c index 7294801544..0ce4ded3d3 100644 --- a/src/coarse/coarse.c +++ b/src/coarse/coarse.c @@ -744,12 +744,18 @@ static block_t *find_free_block(struct ravl *free_blocks, size_t size, size_t alignment, coarse_strategy_t allocation_strategy) { block_t *block; + size_t new_size = size + alignment; switch (allocation_strategy) { case UMF_COARSE_MEMORY_STRATEGY_FASTEST: // Always allocate a free block of the (size + alignment) size // and later cut out the properly aligned part leaving two remaining parts. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, + if (new_size < size) { + LOG_ERR("arithmetic overflow (size + alignment)"); + return NULL; + } + + return free_blocks_rm_ge(free_blocks, new_size, 0, CHECK_ONLY_THE_FIRST_BLOCK); case UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE: @@ -760,8 +766,13 @@ static block_t *find_free_block(struct ravl *free_blocks, size_t size, return block; } + if (new_size < size) { + LOG_ERR("arithmetic overflow (size + alignment)"); + return NULL; + } + // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, + return free_blocks_rm_ge(free_blocks, new_size, 0, CHECK_ONLY_THE_FIRST_BLOCK); case UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE: @@ -773,9 +784,14 @@ static block_t *find_free_block(struct ravl *free_blocks, size_t size, return block; } + if (new_size < size) { + LOG_ERR("arithmetic overflow (size + alignment)"); + return NULL; + } + // If none of them had the correct alignment, // use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - return free_blocks_rm_ge(free_blocks, size + alignment, 0, + return free_blocks_rm_ge(free_blocks, new_size, 0, CHECK_ONLY_THE_FIRST_BLOCK); } @@ -1017,17 +1033,17 @@ umf_result_t coarse_alloc(coarse_t *coarse, size_t size, size_t alignment, } // alignment must be a power of two and a multiple or a divider of the page size - if (alignment && - ((alignment & (alignment - 1)) || ((alignment % coarse->page_size) && - (coarse->page_size % alignment)))) { + if (alignment == 0) { + alignment = coarse->page_size; + } else if ((alignment & (alignment - 1)) || + ((alignment % coarse->page_size) && + (coarse->page_size % alignment))) { LOG_ERR("wrong alignment: %zu (not a power of 2 or a multiple or a " "divider of the page size (%zu))", alignment, coarse->page_size); return UMF_RESULT_ERROR_INVALID_ALIGNMENT; - } - - if (IS_NOT_ALIGNED(alignment, coarse->page_size)) { - alignment = ALIGN_UP(alignment, coarse->page_size); + } else if (IS_NOT_ALIGNED(alignment, coarse->page_size)) { + alignment = ALIGN_UP_SAFE(alignment, coarse->page_size); } if (utils_mutex_lock(&coarse->lock) != 0) { diff --git a/test/coarse_lib.cpp b/test/coarse_lib.cpp index 6a3d9637ec..c5e30ee8fc 100644 --- a/test/coarse_lib.cpp +++ b/test/coarse_lib.cpp @@ -166,9 +166,10 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_provider) { TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_fixed_memory) { // preallocate some memory and initialize the vector with zeros - const size_t buff_size = 20 * MB; + const size_t buff_size = 20 * MB + coarse_params.page_size; std::vector buffer(buff_size, 0); - void *buf = (void *)buffer.data(); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); ASSERT_NE(buf, nullptr); coarse_params.cb.alloc = NULL; @@ -206,9 +207,10 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_fixed_memory) { TEST_P(CoarseWithMemoryStrategyTest, coarseTest_fixed_memory_various) { // preallocate some memory and initialize the vector with zeros - const size_t buff_size = 20 * MB; + const size_t buff_size = 20 * MB + coarse_params.page_size; std::vector buffer(buff_size, 0); - void *buf = (void *)buffer.data(); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); ASSERT_NE(buf, nullptr); coarse_params.cb.alloc = NULL; @@ -627,6 +629,15 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic_free_cb_fails) { } TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_cb_fails) { + if (coarse_params.allocation_strategy == + UMF_COARSE_MEMORY_STRATEGY_FASTEST) { + // This test is designed for the UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE + // and UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE strategies, + // because the UMF_COARSE_MEMORY_STRATEGY_FASTEST strategy + // looks always for a block of size greater by the page size. + return; + } + umf_memory_provider_handle_t malloc_memory_provider; umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, &malloc_memory_provider); @@ -702,9 +713,10 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_split_cb_fails) { TEST_P(CoarseWithMemoryStrategyTest, coarseTest_merge_cb_fails) { // preallocate some memory and initialize the vector with zeros - const size_t buff_size = 10 * MB; + const size_t buff_size = 10 * MB + coarse_params.page_size; std::vector buffer(buff_size, 0); - void *buf = (void *)buffer.data(); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); ASSERT_NE(buf, nullptr); coarse_params.cb.alloc = NULL; @@ -901,6 +913,15 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_provider_alloc_not_set) { } TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic) { + if (coarse_params.allocation_strategy == + UMF_COARSE_MEMORY_STRATEGY_FASTEST) { + // This test is designed for the UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE + // and UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE strategies, + // because the UMF_COARSE_MEMORY_STRATEGY_FASTEST strategy + // looks always for a block of size greater by the page size. + return; + } + umf_memory_provider_handle_t malloc_memory_provider; umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, &malloc_memory_provider); @@ -1065,6 +1086,15 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_basic) { } TEST_P(CoarseWithMemoryStrategyTest, coarseTest_simple1) { + if (coarse_params.allocation_strategy == + UMF_COARSE_MEMORY_STRATEGY_FASTEST) { + // This test is designed for the UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE + // and UMF_COARSE_MEMORY_STRATEGY_CHECK_ALL_SIZE strategies, + // because the UMF_COARSE_MEMORY_STRATEGY_FASTEST strategy + // looks always for a block of size greater by the page size. + return; + } + umf_memory_provider_handle_t malloc_memory_provider; umf_result = umfMemoryProviderCreate(&UMF_MALLOC_MEMORY_PROVIDER_OPS, NULL, &malloc_memory_provider); @@ -1106,8 +1136,9 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_simple1) { ASSERT_NE(t[i], nullptr); } - if (max_alloc_size == 0) { - max_alloc_size = coarse_get_stats(ch).alloc_size; + size_t alloc_size = coarse_get_stats(ch).alloc_size; + if (alloc_size > max_alloc_size) { + max_alloc_size = alloc_size; } for (int i = 0; i < nptrs; i++) { @@ -1253,9 +1284,10 @@ TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alignment_provider) { TEST_P(CoarseWithMemoryStrategyTest, coarseTest_alignment_fixed_memory) { // preallocate some memory and initialize the vector with zeros - const size_t alloc_size = 40 * MB; + const size_t alloc_size = 40 * MB + coarse_params.page_size; std::vector buffer(alloc_size, 0); - void *buf = (void *)buffer.data(); + void *buf = (void *)ALIGN_UP_SAFE((uintptr_t)buffer.data(), + coarse_params.page_size); ASSERT_NE(buf, nullptr); coarse_params.cb.alloc = NULL; From d2ecbe9c44da0939a9f97edeabb82e0ff1a31c27 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 9 Dec 2024 12:39:15 +0100 Subject: [PATCH 35/82] Make UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE the default strategy Make UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE the default strategy, because the alignment never equals 0 now. Signed-off-by: Lukasz Dorau --- src/coarse/coarse.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coarse/coarse.h b/src/coarse/coarse.h index cd151ca27a..93ec990027 100644 --- a/src/coarse/coarse.h +++ b/src/coarse/coarse.h @@ -34,16 +34,16 @@ typedef struct coarse_callbacks_t { // coarse library allocation strategy typedef enum coarse_strategy_t { + // Check if the first free block of the 'size' size has the correct alignment. + // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. + UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE = 0, + // Always allocate a free block of the (size + alignment) size // and cut out the properly aligned part leaving two remaining parts. // It is the fastest strategy but causes memory fragmentation // when alignment is greater than 0. // It is the best strategy when alignment always equals 0. - UMF_COARSE_MEMORY_STRATEGY_FASTEST = 0, - - // Check if the first free block of the 'size' size has the correct alignment. - // If not, use the `UMF_COARSE_MEMORY_STRATEGY_FASTEST` strategy. - UMF_COARSE_MEMORY_STRATEGY_FASTEST_BUT_ONE, + UMF_COARSE_MEMORY_STRATEGY_FASTEST, // Look through all free blocks of the 'size' size // and choose the first one with the correct alignment. From 62496ed59ffe9321b1cd46db3b9c81412739607c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Thu, 5 Dec 2024 18:27:01 +0100 Subject: [PATCH 36/82] add extra test for dax provider --- test/provider_devdax_memory.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index afff1de4f7..1feaaaaa63 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -233,6 +233,13 @@ TEST_P(umfProviderTest, purge_force) { test_alloc_free_success(provider.get(), page_size, 0, PURGE_FORCE); } +TEST_P(umfProviderTest, purge_force_unalligned_alloc) { + void *ptr; + auto ret = umfMemoryProviderAlloc(provider.get(), page_plus_64, 0, &ptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + test_alloc_free_success(provider.get(), page_size, 0, PURGE_FORCE); + umfMemoryProviderFree(provider.get(), ptr, page_plus_64); +} // negative tests using test_alloc_failure TEST_P(umfProviderTest, alloc_page64_align_page_minus_1_WRONG_ALIGNMENT_1) { From 3e12e1838bfa02369c69a53036a7ab7a8d9e1cde Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Tue, 10 Dec 2024 10:44:42 +0100 Subject: [PATCH 37/82] Update description of UMF_LINK_HWLOC_STATICALLY It is supported on Linux, MacOS, and Windows with the exception that the proxy library is disabled when this flag is used on Windows with Debug config. --- CMakeLists.txt | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82381f5b5b..1e8ad72682 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ option( OFF) option( UMF_LINK_HWLOC_STATICALLY - "Link UMF with HWLOC library statically (supported for Linux, MacOS and Release build on Windows)" + "Link UMF with HWLOC library statically (proxy library will be disabled on Windows+Debug build)" OFF) option(UMF_FORMAT_CODE_STYLE "Add clang, cmake, and black -format-check and -format-apply targets" diff --git a/README.md b/README.md index 0c569c1b3c..854b25878f 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ List of options provided by CMake: | UMF_USE_MSAN | Enable MemorySanitizer checks | ON/OFF | OFF | | UMF_USE_VALGRIND | Enable Valgrind instrumentation | ON/OFF | OFF | | UMF_USE_COVERAGE | Build with coverage enabled (Linux only) | ON/OFF | OFF | -| UMF_LINK_HWLOC_STATICALLY | Link UMF with HWLOC library statically (Windows+Release only) | ON/OFF | OFF | +| UMF_LINK_HWLOC_STATICALLY | Link UMF with HWLOC library statically (proxy library will be disabled on Windows+Debug build) | ON/OFF | OFF | | UMF_DISABLE_HWLOC | Disable features that requires hwloc (OS provider, memory targets, topology discovery) | ON/OFF | OFF | ## Architecture: memory pools and providers From 36d134c145c88cb633bfe9434b27f863d4f9ec39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Tue, 10 Dec 2024 13:23:01 +0100 Subject: [PATCH 38/82] use unique_ptr instead of constructor/destructor in benchmarks On Windows static builds, destructors are invoked after the UMF destructor, causing parameter structures to be unable to be destroyed. By switching to std::unique_ptr, we ensure that parameters are properly cleaned up and the destruction order issue is resolved. --- benchmark/benchmark.cpp | 111 +++++++++++++---------------- benchmark/benchmark.hpp | 8 +++ benchmark/benchmark_interfaces.hpp | 27 +++++-- 3 files changed, 79 insertions(+), 67 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index c10bbda877..655545d1e2 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -37,20 +37,27 @@ struct glibc_malloc : public allocator_interface { }; struct os_provider : public provider_interface { - umf_os_memory_provider_params_handle_t params = NULL; - os_provider() { - umfOsMemoryProviderParamsCreate(¶ms); - return; - } - - ~os_provider() { - if (params != NULL) { - umfOsMemoryProviderParamsDestroy(params); + provider_interface::params_ptr + getParams(::benchmark::State &state) override { + umf_os_memory_provider_params_handle_t raw_params = nullptr; + umfOsMemoryProviderParamsCreate(&raw_params); + if (!raw_params) { + state.SkipWithError("Failed to create os provider params"); + return {nullptr, [](void *) {}}; } + + // Use a lambda as the custom deleter + auto deleter = [](void *p) { + auto handle = + static_cast(p); + umfOsMemoryProviderParamsDestroy(handle); + }; + + return {static_cast(raw_params), deleter}; } - void *getParams() override { return params; } - umf_memory_provider_ops_t *getOps() override { + umf_memory_provider_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { return umfOsMemoryProviderOps(); } static std::string name() { return "os_provider"; } @@ -62,73 +69,60 @@ struct proxy_pool : public pool_interface { getOps([[maybe_unused]] ::benchmark::State &state) override { return umfProxyPoolOps(); } - void *getParams([[maybe_unused]] ::benchmark::State &state) override { - return nullptr; - } + static std::string name() { return "proxy_pool<" + Provider::name() + ">"; } }; #ifdef UMF_POOL_DISJOINT_ENABLED template struct disjoint_pool : public pool_interface { - umf_disjoint_pool_params_handle_t disjoint_memory_pool_params; + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfDisjointPoolOps(); + } - disjoint_pool() { - disjoint_memory_pool_params = NULL; - auto ret = umfDisjointPoolParamsCreate(&disjoint_memory_pool_params); + typename pool_interface::params_ptr + getParams(::benchmark::State &state) override { + umf_disjoint_pool_params_handle_t raw_params = nullptr; + auto ret = umfDisjointPoolParamsCreate(&raw_params); if (ret != UMF_RESULT_SUCCESS) { - return; + state.SkipWithError("Failed to create disjoint pool params"); + return {nullptr, [](void *) {}}; } - // those function should never fail, so error handling is minimal. - ret = umfDisjointPoolParamsSetSlabMinSize(disjoint_memory_pool_params, - 4096); - if (ret != UMF_RESULT_SUCCESS) { - goto err; - } + typename pool_interface::params_ptr params( + raw_params, [](void *p) { + umfDisjointPoolParamsDestroy( + static_cast(p)); + }); - ret = umfDisjointPoolParamsSetCapacity(disjoint_memory_pool_params, 4); + ret = umfDisjointPoolParamsSetSlabMinSize(raw_params, 4096); if (ret != UMF_RESULT_SUCCESS) { - goto err; + state.SkipWithError("Failed to set slab min size"); + return {nullptr, [](void *) {}}; } - ret = umfDisjointPoolParamsSetMinBucketSize(disjoint_memory_pool_params, - 4096); + ret = umfDisjointPoolParamsSetCapacity(raw_params, 4); if (ret != UMF_RESULT_SUCCESS) { - goto err; + state.SkipWithError("Failed to set capacity"); + return {nullptr, [](void *) {}}; } - ret = umfDisjointPoolParamsSetMaxPoolableSize( - disjoint_memory_pool_params, 4096 * 16); - + ret = umfDisjointPoolParamsSetMinBucketSize(raw_params, 4096); if (ret != UMF_RESULT_SUCCESS) { - goto err; + state.SkipWithError("Failed to set min bucket size"); + return {nullptr, [](void *) {}}; } - return; - err: - umfDisjointPoolParamsDestroy(disjoint_memory_pool_params); - disjoint_memory_pool_params = NULL; - } - - ~disjoint_pool() { - if (disjoint_memory_pool_params != NULL) { - umfDisjointPoolParamsDestroy(disjoint_memory_pool_params); + ret = umfDisjointPoolParamsSetMaxPoolableSize(raw_params, 4096 * 16); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set max poolable size"); + return {nullptr, [](void *) {}}; } - } - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfDisjointPoolOps(); + return params; } - void *getParams([[maybe_unused]] ::benchmark::State &state) override { - if (disjoint_memory_pool_params == NULL) { - state.SkipWithError("Failed to create disjoint pool params"); - } - - return disjoint_memory_pool_params; - } static std::string name() { return "disjoint_pool<" + Provider::name() + ">"; } @@ -142,9 +136,7 @@ struct jemalloc_pool : public pool_interface { getOps([[maybe_unused]] ::benchmark::State &state) override { return umfJemallocPoolOps(); } - void *getParams([[maybe_unused]] ::benchmark::State &state) override { - return NULL; - } + static std::string name() { return "jemalloc_pool<" + Provider::name() + ">"; } @@ -158,10 +150,7 @@ struct scalable_pool : public pool_interface { getOps([[maybe_unused]] ::benchmark::State &state) override { return umfScalablePoolOps(); } - virtual void * - getParams([[maybe_unused]] ::benchmark::State &state) override { - return NULL; - } + static std::string name() { return "scalable_pool<" + Provider::name() + ">"; } diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index ad9ab7cc83..6ac7a4dfa5 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -232,12 +232,14 @@ class alloc_benchmark : public benchmark_interface { state.ResumeTiming(); } } + static std::vector argsName() { auto n = benchmark_interface::argsName(); std::vector res = {"max_allocs", "pre_allocs"}; res.insert(res.end(), n.begin(), n.end()); return res; } + static std::string name() { return base::name() + "/alloc"; } static int64_t iterations() { return 200000; } @@ -320,13 +322,16 @@ class multiple_malloc_free_benchmark : public alloc_benchmark { static std::string name() { return base::base::name() + "/multiple_malloc_free"; } + static std::vector argsName() { auto n = benchmark_interface::argsName(); std::vector res = {"max_allocs"}; res.insert(res.end(), n.begin(), n.end()); return res; } + static int64_t iterations() { return 2000; } + std::default_random_engine generator; distribution dist; }; @@ -352,9 +357,11 @@ class provider_allocator : public allocator_interface { } return ptr; } + void benchFree(void *ptr, size_t size) override { umfMemoryProviderFree(provider.provider, ptr, size); } + static std::string name() { return Provider::name(); } private: @@ -374,6 +381,7 @@ template class pool_allocator : public allocator_interface { virtual void *benchAlloc(size_t size) override { return umfPoolMalloc(pool.pool, size); } + virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) override { umfPoolFree(pool.pool, ptr); } diff --git a/benchmark/benchmark_interfaces.hpp b/benchmark/benchmark_interfaces.hpp index e25c977710..516a20b697 100644 --- a/benchmark/benchmark_interfaces.hpp +++ b/benchmark/benchmark_interfaces.hpp @@ -6,6 +6,7 @@ * */ +#include #include #include #include @@ -39,6 +40,7 @@ struct benchmark_interface : public benchmark::Fixture { int argPos = alloc_size.SetUp(state, 0); allocator.SetUp(state, argPos); } + void TearDown(::benchmark::State &state) { alloc_size.TearDown(state); allocator.TearDown(state); @@ -54,6 +56,7 @@ struct benchmark_interface : public benchmark::Fixture { res.insert(res.end(), a.begin(), a.end()); return res; } + static std::string name() { return Allocator::name(); } static int64_t iterations() { return 10000; } Size alloc_size; @@ -61,13 +64,16 @@ struct benchmark_interface : public benchmark::Fixture { }; struct provider_interface { + using params_ptr = std::unique_ptr; + umf_memory_provider_handle_t provider = NULL; virtual void SetUp(::benchmark::State &state) { if (state.thread_index() != 0) { return; } + auto params = getParams(state); auto umf_result = - umfMemoryProviderCreate(getOps(), getParams(), &provider); + umfMemoryProviderCreate(getOps(state), params.get(), &provider); if (umf_result != UMF_RESULT_SUCCESS) { state.SkipWithError("umfMemoryProviderCreate() failed"); } @@ -83,21 +89,30 @@ struct provider_interface { } } - virtual umf_memory_provider_ops_t *getOps() { return nullptr; } - virtual void *getParams() { return nullptr; } + virtual umf_memory_provider_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) { + return nullptr; + } + + virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { + return {nullptr, [](void *) {}}; + } }; template ::value>> struct pool_interface { + using params_ptr = std::unique_ptr; + virtual void SetUp(::benchmark::State &state) { provider.SetUp(state); if (state.thread_index() != 0) { return; } + auto params = getParams(state); auto umf_result = umfPoolCreate(getOps(state), provider.provider, - getParams(state), 0, &pool); + params.get(), 0, &pool); if (umf_result != UMF_RESULT_SUCCESS) { state.SkipWithError("umfPoolCreate() failed"); } @@ -121,8 +136,8 @@ struct pool_interface { getOps([[maybe_unused]] ::benchmark::State &state) { return nullptr; } - virtual void *getParams([[maybe_unused]] ::benchmark::State &state) { - return nullptr; + virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { + return {nullptr, [](void *) {}}; } T provider; umf_memory_pool_handle_t pool; From 397e5c4f55e05b253d05d960c2be199f67942a14 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 12 Dec 2024 11:36:27 +0100 Subject: [PATCH 39/82] Build custom jemalloc with -j$(nproc) Signed-off-by: Lukasz Dorau --- CMakeLists.txt | 7 ++++++- scripts/qemu/run-build.sh | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d0890831d..0eea5faf42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,8 +161,13 @@ else() WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} OUTPUT ${jemalloc_targ_SOURCE_DIR}/Makefile DEPENDS ${jemalloc_targ_SOURCE_DIR}/configure) + + if(NOT UMF_QEMU_BUILD) + set(MAKE_ARGUMENTS "-j$(nproc)") + endif() + add_custom_command( - COMMAND make + COMMAND make ${MAKE_ARGUMENTS} WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR} OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.a DEPENDS ${jemalloc_targ_SOURCE_DIR}/Makefile) diff --git a/scripts/qemu/run-build.sh b/scripts/qemu/run-build.sh index b0f4bee1e5..c6314153c6 100755 --- a/scripts/qemu/run-build.sh +++ b/scripts/qemu/run-build.sh @@ -21,6 +21,7 @@ cd build cmake .. \ -DCMAKE_BUILD_TYPE=Debug \ + -DUMF_QEMU_BUILD=1 \ -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON \ -DUMF_BUILD_CUDA_PROVIDER=ON \ -DUMF_FORMAT_CODE_STYLE=OFF \ From 0c75171f88af8cd0768c1e251bc93ec4cf51c61b Mon Sep 17 00:00:00 2001 From: kluszcze Date: Thu, 12 Dec 2024 13:10:32 +0100 Subject: [PATCH 40/82] fix codespell errors in CODE_OF_CONDUCT and reusable_valgrind.yml Signed-off-by: kluszcze --- .github/workflows/reusable_valgrind.yml | 2 +- CODE_OF_CONDUCT.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_valgrind.yml b/.github/workflows/reusable_valgrind.yml index 3e0af273a5..aba0e32605 100644 --- a/.github/workflows/reusable_valgrind.yml +++ b/.github/workflows/reusable_valgrind.yml @@ -1,4 +1,4 @@ -# Run tests with valgrind intstrumentation tools: memcheck, drd, helgrind +# Run tests with valgrind instrumentation tools: memcheck, drd, helgrind name: Valgrind on: workflow_call diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 58dba18db6..2e7fbf7d6c 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -5,7 +5,7 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, +identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. From b07c1a2d6f7d2c17b61827d273a85e6da81a0118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Thu, 5 Dec 2024 15:43:16 +0100 Subject: [PATCH 41/82] add fixed provider --- README.md | 7 +- include/umf/providers/provider_fixed_memory.h | 64 +++ scripts/docs_config/api.rst | 11 +- src/CMakeLists.txt | 1 + src/libumf.def | 5 +- src/libumf.map | 11 +- src/provider/provider_fixed_memory.c | 337 +++++++++++++++ test/CMakeLists.txt | 4 + test/provider_fixed_memory.cpp | 393 ++++++++++++++++++ 9 files changed, 815 insertions(+), 18 deletions(-) create mode 100644 include/umf/providers/provider_fixed_memory.h create mode 100644 src/provider/provider_fixed_memory.c create mode 100644 test/provider_fixed_memory.cpp diff --git a/README.md b/README.md index 854b25878f..df90b6852e 100644 --- a/README.md +++ b/README.md @@ -132,12 +132,9 @@ More detailed documentation is available here: https://oneapi-src.github.io/unif ### Memory providers -#### Coarse Provider +#### Fixed memory provider -A memory provider that can provide memory from: -1) a given pre-allocated buffer (the fixed-size memory provider option) or -2) from an additional upstream provider (e.g. provider that does not support the free() operation - like the File memory provider or the DevDax memory provider - see below). +A memory provider that can provide memory from a given pre-allocated buffer. #### OS memory provider diff --git a/include/umf/providers/provider_fixed_memory.h b/include/umf/providers/provider_fixed_memory.h new file mode 100644 index 0000000000..2351faf312 --- /dev/null +++ b/include/umf/providers/provider_fixed_memory.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_FIXED_MEMORY_PROVIDER_H +#define UMF_FIXED_MEMORY_PROVIDER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// @cond +#define UMF_FIXED_RESULTS_START_FROM 4000 +/// @endcond + +struct umf_fixed_memory_provider_params_t; + +typedef struct umf_fixed_memory_provider_params_t + *umf_fixed_memory_provider_params_handle_t; + +/// @brief Create a struct to store parameters of the Fixed Memory Provider. +/// @param hParams [out] handle to the newly created parameters struct. +/// @param ptr [in] pointer to the memory region. +/// @param size [in] size of the memory region in bytes. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsCreate( + umf_fixed_memory_provider_params_handle_t *hParams, void *ptr, size_t size); + +/// @brief Set the memory region in params struct. Overwrites the previous value. +/// It provides an ability to use the same instance of params to create multiple +/// instances of the provider for different memory regions. +/// @param hParams [in] handle to the parameters of the Fixed Memory Provider. +/// @param ptr [in] pointer to the memory region. +/// @param size [in] size of the memory region in bytes. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsSetMemory( + umf_fixed_memory_provider_params_handle_t hParams, void *ptr, size_t size); + +/// @brief Destroy parameters struct. +/// @param hParams [in] handle to the parameters of the Fixed Memory Provider. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfFixedMemoryProviderParamsDestroy( + umf_fixed_memory_provider_params_handle_t hParams); + +/// @brief Retrieve the operations structure for the Fixed Memory Provider. +/// @return Pointer to the umf_memory_provider_ops_t structure. +umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void); + +/// @brief Fixed Memory Provider operation results +typedef enum umf_fixed_memory_provider_native_error { + UMF_FIXED_RESULT_SUCCESS = UMF_FIXED_RESULTS_START_FROM, ///< Success + UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED, ///< Force purging failed +} umf_fixed_memory_provider_native_error_t; + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_FIXED_MEMORY_PROVIDER_H */ diff --git a/scripts/docs_config/api.rst b/scripts/docs_config/api.rst index 7f734cad24..c0448f1178 100644 --- a/scripts/docs_config/api.rst +++ b/scripts/docs_config/api.rst @@ -80,17 +80,12 @@ and operate on the provider. .. doxygenfile:: memory_provider.h :sections: define enum typedef func var -Coarse Provider +Fixed Memory Provider ------------------------------------------ -A memory provider that can provide memory from: +A memory provider that can provide memory from a given pre-allocated buffer. -1) A given pre-allocated buffer (the fixed-size memory provider option) or -2) From an additional upstream provider (e.g. provider that does not support - the free() operation like the File memory provider or the DevDax memory - provider - see below). - -.. doxygenfile:: provider_coarse.h +.. doxygenfile:: provider_fixed_memory.h :sections: define enum typedef func var OS Memory Provider diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fb32b6d2eb..2a27dce464 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,6 +62,7 @@ set(UMF_SOURCES provider/provider_cuda.c provider/provider_devdax_memory.c provider/provider_file_memory.c + provider/provider_fixed_memory.c provider/provider_level_zero.c provider/provider_os_memory.c provider/provider_tracking.c diff --git a/src/libumf.def b/src/libumf.def index 7666c146be..5d1c5047f0 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -25,13 +25,16 @@ EXPORTS umfDevDaxMemoryProviderParamsDestroy umfDevDaxMemoryProviderParamsSetDeviceDax umfDevDaxMemoryProviderParamsSetProtection - umfFree umfFileMemoryProviderOps umfFileMemoryProviderParamsCreate umfFileMemoryProviderParamsDestroy umfFileMemoryProviderParamsSetPath umfFileMemoryProviderParamsSetProtection umfFileMemoryProviderParamsSetVisibility + umfFixedMemoryProviderOps + umfFixedMemoryProviderParamsCreate + umfFixedMemoryProviderParamsDestroy + umfFree umfGetIPCHandle umfGetLastFailedMemoryProvider umfJemallocPoolOps diff --git a/src/libumf.map b/src/libumf.map index 5d1ca3b775..d604dd64e6 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -19,13 +19,16 @@ UMF_1.0 { umfDevDaxMemoryProviderParamsDestroy; umfDevDaxMemoryProviderParamsSetDeviceDax; umfDevDaxMemoryProviderParamsSetProtection; - umfFree; umfFileMemoryProviderOps; umfFileMemoryProviderParamsCreate; umfFileMemoryProviderParamsDestroy; umfFileMemoryProviderParamsSetPath; umfFileMemoryProviderParamsSetProtection; umfFileMemoryProviderParamsSetVisibility; + umfFixedMemoryProviderOps; + umfFixedMemoryProviderParamsCreate; + umfFixedMemoryProviderParamsDestroy; + umfFree; umfGetIPCHandle; umfGetLastFailedMemoryProvider; umfJemallocPoolOps; @@ -81,13 +84,13 @@ UMF_1.0 { umfOsMemoryProviderOps; umfOsMemoryProviderParamsCreate; umfOsMemoryProviderParamsDestroy; - umfOsMemoryProviderParamsSetProtection; - umfOsMemoryProviderParamsSetVisibility; - umfOsMemoryProviderParamsSetShmName; umfOsMemoryProviderParamsSetNumaList; umfOsMemoryProviderParamsSetNumaMode; umfOsMemoryProviderParamsSetPartSize; umfOsMemoryProviderParamsSetPartitions; + umfOsMemoryProviderParamsSetProtection; + umfOsMemoryProviderParamsSetShmName; + umfOsMemoryProviderParamsSetVisibility; umfPoolAlignedMalloc; umfPoolByPtr; umfPoolCalloc; diff --git a/src/provider/provider_fixed_memory.c b/src/provider/provider_fixed_memory.c new file mode 100644 index 0000000000..6392b39d39 --- /dev/null +++ b/src/provider/provider_fixed_memory.c @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "base_alloc_global.h" +#include "coarse.h" +#include "libumf.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" + +#define TLS_MSG_BUF_LEN 1024 + +typedef struct fixed_memory_provider_t { + void *base; // base address of memory + size_t size; // size of the memory region + coarse_t *coarse; // coarse library handle +} fixed_memory_provider_t; + +// Fixed Memory provider settings struct +typedef struct umf_fixed_memory_provider_params_t { + void *ptr; + size_t size; +} umf_fixed_memory_provider_params_t; + +typedef struct fixed_last_native_error_t { + int32_t native_error; + int errno_value; + char msg_buff[TLS_MSG_BUF_LEN]; +} fixed_last_native_error_t; + +static __TLS fixed_last_native_error_t TLS_last_native_error; + +// helper values used only in the Native_error_str array +#define _UMF_FIXED_RESULT_SUCCESS \ + (UMF_FIXED_RESULT_SUCCESS - UMF_FIXED_RESULT_SUCCESS) +#define _UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED \ + (UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED - UMF_FIXED_RESULT_SUCCESS) + +static const char *Native_error_str[] = { + [_UMF_FIXED_RESULT_SUCCESS] = "success", + [_UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED] = "force purging failed"}; + +static void fixed_store_last_native_error(int32_t native_error, + int errno_value) { + TLS_last_native_error.native_error = native_error; + TLS_last_native_error.errno_value = errno_value; +} + +static umf_result_t fixed_allocation_split_cb(void *provider, void *ptr, + size_t totalSize, + size_t firstSize) { + (void)provider; + (void)ptr; + (void)totalSize; + (void)firstSize; + return UMF_RESULT_SUCCESS; +} + +static umf_result_t fixed_allocation_merge_cb(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { + (void)provider; + (void)lowPtr; + (void)highPtr; + (void)totalSize; + return UMF_RESULT_SUCCESS; +} + +static umf_result_t fixed_initialize(void *params, void **provider) { + umf_result_t ret; + + if (params == NULL) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_fixed_memory_provider_params_t *in_params = + (umf_fixed_memory_provider_params_t *)params; + + fixed_memory_provider_t *fixed_provider = + umf_ba_global_alloc(sizeof(*fixed_provider)); + if (!fixed_provider) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + memset(fixed_provider, 0, sizeof(*fixed_provider)); + + coarse_params_t coarse_params = {0}; + coarse_params.provider = fixed_provider; + coarse_params.page_size = utils_get_page_size(); + // The alloc callback is not available in case of the fixed provider + // because it is a fixed-size memory provider + // and the entire memory is added as a single block + // to the coarse library. + coarse_params.cb.alloc = NULL; + coarse_params.cb.free = NULL; // not available for the fixed provider + coarse_params.cb.split = fixed_allocation_split_cb; + coarse_params.cb.merge = fixed_allocation_merge_cb; + + coarse_t *coarse = NULL; + ret = coarse_new(&coarse_params, &coarse); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("coarse_new() failed"); + goto err_free_fixed_provider; + } + + fixed_provider->coarse = coarse; + + fixed_provider->base = in_params->ptr; + fixed_provider->size = in_params->size; + + // add the entire memory as a single block + ret = coarse_add_memory_fixed(coarse, fixed_provider->base, + fixed_provider->size); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("adding memory block failed"); + goto err_coarse_delete; + } + + *provider = fixed_provider; + + return UMF_RESULT_SUCCESS; + +err_coarse_delete: + coarse_delete(fixed_provider->coarse); +err_free_fixed_provider: + umf_ba_global_free(fixed_provider); + return ret; +} + +static void fixed_finalize(void *provider) { + fixed_memory_provider_t *fixed_provider = provider; + coarse_delete(fixed_provider->coarse); + umf_ba_global_free(fixed_provider); +} + +static umf_result_t fixed_alloc(void *provider, size_t size, size_t alignment, + void **resultPtr) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + + return coarse_alloc(fixed_provider->coarse, size, alignment, resultPtr); +} + +static void fixed_get_last_native_error(void *provider, const char **ppMessage, + int32_t *pError) { + (void)provider; // unused + + if (ppMessage == NULL || pError == NULL) { + assert(0); + return; + } + + *pError = TLS_last_native_error.native_error; + if (TLS_last_native_error.errno_value == 0) { + *ppMessage = Native_error_str[*pError - UMF_FIXED_RESULT_SUCCESS]; + return; + } + + const char *msg; + size_t len; + size_t pos = 0; + + msg = Native_error_str[*pError - UMF_FIXED_RESULT_SUCCESS]; + len = strlen(msg); + memcpy(TLS_last_native_error.msg_buff + pos, msg, len + 1); + pos += len; + + msg = ": "; + len = strlen(msg); + memcpy(TLS_last_native_error.msg_buff + pos, msg, len + 1); + pos += len; + + utils_strerror(TLS_last_native_error.errno_value, + TLS_last_native_error.msg_buff + pos, TLS_MSG_BUF_LEN - pos); + + *ppMessage = TLS_last_native_error.msg_buff; +} + +static umf_result_t fixed_get_recommended_page_size(void *provider, size_t size, + size_t *page_size) { + (void)provider; // unused + (void)size; // unused + + *page_size = utils_get_page_size(); + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t fixed_get_min_page_size(void *provider, void *ptr, + size_t *page_size) { + (void)ptr; // unused + + return fixed_get_recommended_page_size(provider, 0, page_size); +} + +static umf_result_t fixed_purge_lazy(void *provider, void *ptr, size_t size) { + (void)provider; // unused + (void)ptr; // unused + (void)size; // unused + // purge_lazy is unsupported in case of the fixed memory provider + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +static umf_result_t fixed_purge_force(void *provider, void *ptr, size_t size) { + (void)provider; // unused + errno = 0; + if (utils_purge(ptr, size, UMF_PURGE_FORCE)) { + fixed_store_last_native_error(UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED, + errno); + LOG_PERR("force purging failed"); + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + return UMF_RESULT_SUCCESS; +} + +static const char *fixed_get_name(void *provider) { + (void)provider; // unused + return "FIXED"; +} + +static umf_result_t fixed_allocation_split(void *provider, void *ptr, + size_t totalSize, size_t firstSize) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + return coarse_split(fixed_provider->coarse, ptr, totalSize, firstSize); +} + +static umf_result_t fixed_allocation_merge(void *provider, void *lowPtr, + void *highPtr, size_t totalSize) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + return coarse_merge(fixed_provider->coarse, lowPtr, highPtr, totalSize); +} + +static umf_result_t fixed_free(void *provider, void *ptr, size_t size) { + fixed_memory_provider_t *fixed_provider = + (fixed_memory_provider_t *)provider; + return coarse_free(fixed_provider->coarse, ptr, size); +} + +static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = { + .version = UMF_VERSION_CURRENT, + .initialize = fixed_initialize, + .finalize = fixed_finalize, + .alloc = fixed_alloc, + .free = fixed_free, + .get_last_native_error = fixed_get_last_native_error, + .get_recommended_page_size = fixed_get_recommended_page_size, + .get_min_page_size = fixed_get_min_page_size, + .get_name = fixed_get_name, + .ext.purge_lazy = fixed_purge_lazy, + .ext.purge_force = fixed_purge_force, + .ext.allocation_merge = fixed_allocation_merge, + .ext.allocation_split = fixed_allocation_split, + .ipc.get_ipc_handle_size = NULL, + .ipc.get_ipc_handle = NULL, + .ipc.put_ipc_handle = NULL, + .ipc.open_ipc_handle = NULL, + .ipc.close_ipc_handle = NULL}; + +umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void) { + return &UMF_FIXED_MEMORY_PROVIDER_OPS; +} + +umf_result_t umfFixedMemoryProviderParamsCreate( + umf_fixed_memory_provider_params_handle_t *hParams, void *ptr, + size_t size) { + libumfInit(); + if (hParams == NULL) { + LOG_ERR("Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_fixed_memory_provider_params_handle_t params = + umf_ba_global_alloc(sizeof(*params)); + if (params == NULL) { + LOG_ERR("Allocating memory for the Memory Provider params failed"); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + umf_result_t ret = umfFixedMemoryProviderParamsSetMemory(params, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { + umf_ba_global_free(params); + return ret; + } + + *hParams = params; + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfFixedMemoryProviderParamsDestroy( + umf_fixed_memory_provider_params_handle_t hParams) { + if (hParams != NULL) { + umf_ba_global_free(hParams); + } + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfFixedMemoryProviderParamsSetMemory( + umf_fixed_memory_provider_params_handle_t hParams, void *ptr, size_t size) { + + if (hParams == NULL) { + LOG_ERR("Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (ptr == NULL) { + LOG_ERR("Memory pointer is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (size == 0) { + LOG_ERR("Size must be greater than 0"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->ptr = ptr; + hParams->size = size; + return UMF_RESULT_SUCCESS; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d5a07bfbbf..bb353a889a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -317,6 +317,10 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_file_memory_ipc SRCS provider_file_memory_ipc.cpp ${BA_SOURCES_FOR_TEST} LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_fixed_memory + SRCS provider_fixed_memory.cpp + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp new file mode 100644 index 0000000000..7f976a1f5d --- /dev/null +++ b/test/provider_fixed_memory.cpp @@ -0,0 +1,393 @@ +// Copyright (C) 2024 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "base.hpp" + +#include "cpp_helpers.hpp" +#include "test_helpers.h" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include +#include + +using umf_test::test; + +#define INVALID_PTR ((void *)0x01) + +typedef enum purge_t { + PURGE_NONE = 0, + PURGE_LAZY = 1, + PURGE_FORCE = 2, +} purge_t; + +static const char *Native_error_str[] = { + "success", // UMF_FIXED_RESULT_SUCCESS + "force purging failed", // UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED +}; + +// Test helpers + +static int compare_native_error_str(const char *message, int error) { + const char *error_str = Native_error_str[error - UMF_FIXED_RESULT_SUCCESS]; + size_t len = strlen(error_str); + return strncmp(message, error_str, len); +} + +using providerCreateExtParams = std::tuple; + +static void providerCreateExt(providerCreateExtParams params, + umf::provider_unique_handle_t *handle) { + umf_memory_provider_handle_t hProvider = nullptr; + auto [provider_ops, provider_params] = params; + + auto ret = + umfMemoryProviderCreate(provider_ops, provider_params, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + + *handle = + umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); +} + +struct FixedProviderTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + // Allocate a memory buffer to use with the fixed memory provider + memory_size = utils_get_page_size() * 10; // Allocate 10 pages + memory_buffer = malloc(memory_size); + ASSERT_NE(memory_buffer, nullptr); + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer, memory_size); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + providerCreateExt(std::make_tuple(umfFixedMemoryProviderOps(), params), + &provider); + + umfFixedMemoryProviderParamsDestroy(params); + umf_result_t umf_result = + umfMemoryProviderGetMinPageSize(provider.get(), NULL, &page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + page_plus_64 = page_size + 64; + } + + void TearDown() override { + if (memory_buffer) { + free(memory_buffer); + memory_buffer = nullptr; + } + test::TearDown(); + } + + void test_alloc_free_success(size_t size, size_t alignment, purge_t purge) { + void *ptr = nullptr; + auto provider = this->provider.get(); + + umf_result_t umf_result = + umfMemoryProviderAlloc(provider, size, alignment, &ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size); + + if (purge == PURGE_LAZY) { + umf_result = umfMemoryProviderPurgeLazy(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); + } else if (purge == PURGE_FORCE) { + umf_result = umfMemoryProviderPurgeForce(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + umf_result = umfMemoryProviderFree(provider, ptr, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + void verify_last_native_error(int32_t err) { + const char *message; + int32_t error; + auto provider = this->provider.get(); + umfMemoryProviderGetLastNativeError(provider, &message, &error); + ASSERT_EQ(error, err); + ASSERT_EQ(compare_native_error_str(message, error), 0); + } + + void test_alloc_failure(size_t size, size_t alignment, umf_result_t result, + int32_t err) { + void *ptr = nullptr; + auto provider = this->provider.get(); + + umf_result_t umf_result = + umfMemoryProviderAlloc(provider, size, alignment, &ptr); + ASSERT_EQ(umf_result, result); + ASSERT_EQ(ptr, nullptr); + + if (umf_result == UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC) { + verify_last_native_error(err); + } + } + + umf::provider_unique_handle_t provider; + size_t page_size; + size_t page_plus_64; + void *memory_buffer = nullptr; + size_t memory_size = 0; +}; + +// TESTS + +// Positive tests using test_alloc_free_success + +INSTANTIATE_TEST_SUITE_P(fixedProviderTest, FixedProviderTest, + ::testing::Values(providerCreateExtParams{ + umfFixedMemoryProviderOps(), nullptr})); + +TEST_P(FixedProviderTest, create_destroy) { + // Creation and destruction are handled in SetUp and TearDown +} + +TEST_F(test, create_no_params) { + umf_memory_provider_handle_t provider = nullptr; + auto result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), nullptr, + &provider); + ASSERT_EQ(result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(provider, nullptr); +} + +TEST_P(FixedProviderTest, two_allocations) { + umf_result_t umf_result; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + size_t size = page_plus_64; + size_t alignment = page_size; + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr2, nullptr); + + ASSERT_NE(ptr1, ptr2); + if ((uintptr_t)ptr1 > (uintptr_t)ptr2) { + ASSERT_GT((uintptr_t)ptr1 - (uintptr_t)ptr2, size); + } else { + ASSERT_GT((uintptr_t)ptr2 - (uintptr_t)ptr1, size); + } + + memset(ptr1, 0x11, size); + memset(ptr2, 0x22, size); + + umf_result = umfMemoryProviderFree(provider.get(), ptr1, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(FixedProviderTest, alloc_page64_align_0) { + test_alloc_free_success(page_plus_64, 0, PURGE_NONE); +} + +TEST_P(FixedProviderTest, alloc_page64_align_page_div_2) { + test_alloc_free_success(page_plus_64, page_size / 2, PURGE_NONE); +} + +TEST_P(FixedProviderTest, purge_lazy) { + test_alloc_free_success(page_size, 0, PURGE_LAZY); +} + +TEST_P(FixedProviderTest, purge_force) { + test_alloc_free_success(page_size, 0, PURGE_FORCE); +} + +// Negative tests using test_alloc_failure + +TEST_P(FixedProviderTest, alloc_WRONG_SIZE) { + test_alloc_failure((size_t)-1, 0, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, 0); +} + +TEST_P(FixedProviderTest, alloc_page64_WRONG_ALIGNMENT_3_pages) { + test_alloc_failure(page_plus_64, 3 * page_size, + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +TEST_P(FixedProviderTest, alloc_3pages_WRONG_ALIGNMENT_3pages) { + test_alloc_failure(3 * page_size, 3 * page_size, + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +TEST_P(FixedProviderTest, alloc_page64_align_page_plus_1_WRONG_ALIGNMENT_1) { + test_alloc_failure(page_plus_64, page_size + 1, + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +TEST_P(FixedProviderTest, alloc_page64_align_one_half_pages_WRONG_ALIGNMENT_2) { + test_alloc_failure(page_plus_64, page_size + (page_size / 2), + UMF_RESULT_ERROR_INVALID_ALIGNMENT, 0); +} + +// Other positive tests + +TEST_P(FixedProviderTest, get_min_page_size) { + size_t min_page_size; + umf_result_t umf_result = umfMemoryProviderGetMinPageSize( + provider.get(), nullptr, &min_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_LE(min_page_size, page_size); +} + +TEST_P(FixedProviderTest, get_recommended_page_size) { + size_t min_page_size; + umf_result_t umf_result = umfMemoryProviderGetMinPageSize( + provider.get(), nullptr, &min_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_LE(min_page_size, page_size); + + size_t recommended_page_size; + umf_result = umfMemoryProviderGetRecommendedPageSize( + provider.get(), 0, &recommended_page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(recommended_page_size, min_page_size); +} + +TEST_P(FixedProviderTest, get_name) { + const char *name = umfMemoryProviderGetName(provider.get()); + ASSERT_STREQ(name, "FIXED"); +} + +TEST_P(FixedProviderTest, free_size_0_ptr_not_null) { + umf_result_t umf_result = + umfMemoryProviderFree(provider.get(), INVALID_PTR, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(FixedProviderTest, free_NULL) { + umf_result_t umf_result = umfMemoryProviderFree(provider.get(), nullptr, 0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +// Other negative tests + +TEST_P(FixedProviderTest, free_INVALID_POINTER_SIZE_GT_0) { + umf_result_t umf_result = + umfMemoryProviderFree(provider.get(), INVALID_PTR, page_plus_64); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_P(FixedProviderTest, purge_lazy_INVALID_POINTER) { + umf_result_t umf_result = + umfMemoryProviderPurgeLazy(provider.get(), INVALID_PTR, 1); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_NOT_SUPPORTED); +} + +TEST_P(FixedProviderTest, purge_force_INVALID_POINTER) { + umf_result_t umf_result = + umfMemoryProviderPurgeForce(provider.get(), INVALID_PTR, 1); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC); + + verify_last_native_error(UMF_FIXED_RESULT_ERROR_PURGE_FORCE_FAILED); +} + +// Params tests + +TEST_F(test, params_null_handle) { + constexpr size_t memory_size = 100; + char memory_buffer[memory_size]; + umf_result_t umf_result = + umfFixedMemoryProviderParamsCreate(nullptr, memory_buffer, memory_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_result = umfFixedMemoryProviderParamsDestroy(nullptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_F(test, create_with_null_ptr) { + constexpr size_t memory_size = 100; + umf_fixed_memory_provider_params_handle_t wrong_params = nullptr; + umf_result_t umf_result = + umfFixedMemoryProviderParamsCreate(&wrong_params, nullptr, memory_size); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(wrong_params, nullptr); +} + +TEST_F(test, create_with_zero_size) { + constexpr size_t memory_size = 100; + char memory_buffer[memory_size]; + umf_fixed_memory_provider_params_handle_t wrong_params = nullptr; + umf_result_t umf_result = + umfFixedMemoryProviderParamsCreate(&wrong_params, memory_buffer, 0); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(wrong_params, nullptr); +} + +TEST_P(FixedProviderTest, alloc_size_exceeds_buffer) { + size_t size = memory_size + page_size; + test_alloc_failure(size, 0, UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, 0); +} + +TEST_P(FixedProviderTest, merge) { + umf_result_t umf_result; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + size_t size = page_size; + size_t alignment = page_size; + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfMemoryProviderAlloc(provider.get(), size, alignment, &ptr2); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr2, nullptr); + + ASSERT_EQ((uintptr_t)ptr2 - (uintptr_t)ptr1, size); + + memset(ptr1, 0x11, size); + memset(ptr2, 0x22, size); + + size_t merged_size = size * 2; + umf_result = umfMemoryProviderAllocationMerge(provider.get(), ptr1, ptr2, + merged_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfMemoryProviderFree(provider.get(), ptr1, merged_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(FixedProviderTest, split) { + umf_result_t umf_result; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + size_t size = page_size; + size_t alignment = page_size; + + umf_result = + umfMemoryProviderAlloc(provider.get(), size * 2, alignment, &ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr1, nullptr); + + umf_result = + umfMemoryProviderAllocationSplit(provider.get(), ptr1, size * 2, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr2 = (void *)((uintptr_t)ptr1 + size); + memset(ptr1, 0x11, size); + + umf_result = umfMemoryProviderFree(provider.get(), ptr1, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + memset(ptr2, 0x22, size); + umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} From 202984eb90da6f18b34bc96d72539167886c7885 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 17 Dec 2024 09:26:47 +0100 Subject: [PATCH 42/82] Print more info when open() fails in the custom_file_provider example Signed-off-by: Lukasz Dorau --- examples/custom_file_provider/custom_file_provider.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/custom_file_provider/custom_file_provider.c b/examples/custom_file_provider/custom_file_provider.c index ad897fe5ea..b17fdc0f08 100644 --- a/examples/custom_file_provider/custom_file_provider.c +++ b/examples/custom_file_provider/custom_file_provider.c @@ -62,7 +62,8 @@ static umf_result_t file_init(void *params, void **provider) { // Open the file file_provider->fd = open(file_params->filename, O_RDWR | O_CREAT, 0666); if (file_provider->fd < 0) { - perror("Failed to open file"); + perror("open()"); + fprintf(stderr, "Failed to open the file: %s\n", file_params->filename); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto cleanup_malloc; } From e24048963b3d755002695d1e41477c3173d0fe63 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 16 Dec 2024 13:27:29 +0100 Subject: [PATCH 43/82] Fix: add missing umf*MemoryProviderParamsDestroy() Signed-off-by: Lukasz Dorau --- test/provider_devdax_memory.cpp | 1 + test/provider_file_memory.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/test/provider_devdax_memory.cpp b/test/provider_devdax_memory.cpp index 1feaaaaa63..7765dd08da 100644 --- a/test/provider_devdax_memory.cpp +++ b/test/provider_devdax_memory.cpp @@ -165,6 +165,7 @@ TEST_F(test, test_if_mapped_with_MAP_SYNC) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); + umfDevDaxMemoryProviderParamsDestroy(params); // fail test if the "sf" flag was not found ASSERT_EQ(flag_found, true); diff --git a/test/provider_file_memory.cpp b/test/provider_file_memory.cpp index 0d54c287cb..cfa37be31a 100644 --- a/test/provider_file_memory.cpp +++ b/test/provider_file_memory.cpp @@ -162,6 +162,7 @@ TEST_F(test, test_if_mapped_with_MAP_SYNC) { ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); + umfFileMemoryProviderParamsDestroy(params); // fail test if the "sf" flag was not found ASSERT_EQ(flag_found, true); From a714e4542d600241bf3dc87434c58ea6e9ca0721 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 16 Dec 2024 15:14:50 +0100 Subject: [PATCH 44/82] Fix memory leak in the testNumaSplit:checkModeSplit test Signed-off-by: Lukasz Dorau --- test/provider_os_memory_multiple_numa_nodes.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/provider_os_memory_multiple_numa_nodes.cpp b/test/provider_os_memory_multiple_numa_nodes.cpp index e493a427ca..cfc58f2f06 100644 --- a/test/provider_os_memory_multiple_numa_nodes.cpp +++ b/test/provider_os_memory_multiple_numa_nodes.cpp @@ -674,17 +674,17 @@ TEST_P(testNumaSplit, checkModeSplit) { auto [required_numa_nodes, pages, in, out] = param; umf_result_t umf_result; - umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; - - umf_result = umfOsMemoryProviderParamsCreate(&os_memory_provider_params); - ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); - std::vector numa_nodes = get_available_numa_nodes(); if (numa_nodes.size() < required_numa_nodes) { GTEST_SKIP_("Not enough numa nodes"); } + umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; + + umf_result = umfOsMemoryProviderParamsCreate(&os_memory_provider_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(out.size(), pages) << "Wrong test input - out array size doesn't match page count"; From c549441ebdb442c2559b19aa7bf15a1073b3753b Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 18 Dec 2024 09:48:19 +0100 Subject: [PATCH 45/82] Assert in umf_ba_destroy() in DEBUG and UMF_DEVELOPER_MODE if (NDEBUG is not defined) and (UMF_DEVELOPER_MODE is defined) assert in umf_ba_destroy() if there are any memory leaks in the base allocator. Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_proxy_lib.yml | 2 +- CMakeLists.txt | 5 +++++ src/CMakeLists.txt | 3 ++- src/base_alloc/base_alloc.c | 8 +++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index e73dabe29f..27a66267d1 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -49,7 +49,7 @@ jobs: -DUMF_BUILD_BENCHMARKS=OFF -DUMF_BUILD_TESTS=ON -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=OFF + -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON -DUMF_TESTS_FAIL_ON_SKIP=ON diff --git a/CMakeLists.txt b/CMakeLists.txt index 0eea5faf42..0b88f95b5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,11 @@ else() message(FATAL_ERROR "Unknown OS type") endif() +if(UMF_DEVELOPER_MODE) + set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + UMF_DEVELOPER_MODE=1) +endif() + if(NOT UMF_BUILD_LIBUMF_POOL_JEMALLOC) set(UMF_POOL_JEMALLOC_ENABLED FALSE) set(JEMALLOC_FOUND FALSE) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2a27dce464..2d83faacf0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,7 +14,8 @@ set(UMF_CUDA_INCLUDE_DIR # Compile definitions for UMF library. # # TODO: Cleanup the compile definitions across all the CMake files -set(UMF_COMMON_COMPILE_DEFINITIONS UMF_VERSION=${UMF_VERSION}) +set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + UMF_VERSION=${UMF_VERSION}) set(BA_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc.c diff --git a/src/base_alloc/base_alloc.c b/src/base_alloc/base_alloc.c index 209ace7fe3..6f975307dc 100644 --- a/src/base_alloc/base_alloc.c +++ b/src/base_alloc/base_alloc.c @@ -303,7 +303,13 @@ void umf_ba_destroy(umf_ba_pool_t *pool) { #ifndef NDEBUG ba_debug_checks(pool); if (pool->metadata.n_allocs) { - LOG_ERR("pool->metadata.n_allocs = %zu", pool->metadata.n_allocs); + LOG_ERR("number of base allocator memory leaks: %zu", + pool->metadata.n_allocs); + +#ifdef UMF_DEVELOPER_MODE + assert(pool->metadata.n_allocs == 0 && + "memory leaks in base allocator occurred"); +#endif } #endif /* NDEBUG */ From d56b62caaae697b6653149af608c830249659b26 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 18 Dec 2024 10:04:10 +0100 Subject: [PATCH 46/82] Run also examples under valgrind Signed-off-by: Lukasz Dorau --- test/test_valgrind.sh | 55 +++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index 9f84cf0d32..be5f817dc3 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -10,8 +10,8 @@ BUILD_DIR=$2 TOOL=$3 function print_usage() { - echo "$(basename $0) - run all UMF tests under a valgrind tool (memcheck, drd or helgrind)" - echo "This script looks for './test/umf_test-*' test executables in the UMF build directory." + echo "$(basename $0) - run all UMF tests and examples under a valgrind tool (memcheck, drd or helgrind)" + echo "This script looks for './test/umf_test-*' and './examples/umf_example_*' executables in the UMF build directory." echo "Usage: $(basename $0) " } @@ -58,7 +58,7 @@ esac WORKSPACE=$(realpath $WORKSPACE) BUILD_DIR=$(realpath $BUILD_DIR) -cd ${BUILD_DIR}/test/ +cd ${BUILD_DIR} mkdir -p cpuid echo "Gathering data for hwloc so it can be run under valgrind:" @@ -71,20 +71,21 @@ echo "Running: \"valgrind $OPTION\" for the following tests:" ANY_TEST_FAILED=0 rm -f umf_test-*.log umf_test-*.err -for test in $(ls -1 umf_test-*); do +for test in $(ls -1 ./test/umf_test-* ./examples/umf_example_*); do [ ! -x $test ] && continue echo "$test - starting ..." echo -n "$test " LOG=${test}.log ERR=${test}.err - SUP="${WORKSPACE}/test/supp/${TOOL}-${test}.supp" + NAME=$(basename $test) + SUP="${WORKSPACE}/test/supp/${TOOL}-${NAME}.supp" OPT_SUP="" - [ -f ${SUP} ] && OPT_SUP="--suppressions=${SUP}" && echo -n "(${TOOL}-${test}.supp) " + [ -f ${SUP} ] && OPT_SUP="--suppressions=${SUP}" && echo -n "($(basename ${SUP})) " # skip tests incompatible with valgrind FILTER="" case $test in - umf_test-disjointPool) + ./test/umf_test-disjointPool) if [ "$TOOL" = "helgrind" ]; then # skip because of the assert in helgrind: # Helgrind: hg_main.c:308 (lockN_acquire_reader): Assertion 'lk->kind == LK_rdwr' failed. @@ -92,53 +93,61 @@ for test in $(ls -1 umf_test-*); do continue; fi ;; - umf_test-ipc_os_prov_*) + ./test/umf_test-ipc_os_prov_*) echo "- SKIPPED" continue; # skip testing helper binaries used by the ipc_os_prov_* tests ;; - umf_test-ipc_devdax_prov_*) + ./test/umf_test-ipc_devdax_prov_*) echo "- SKIPPED" continue; # skip testing helper binaries used by the ipc_devdax_prov_* tests ;; - umf_test-ipc_file_prov_*) + ./test/umf_test-ipc_file_prov_*) echo "- SKIPPED" continue; # skip testing helper binaries used by the ipc_file_prov_* tests ;; - umf_test-memspace_host_all) + ./test/umf_test-memspace_host_all) FILTER='--gtest_filter="-*allocsSpreadAcrossAllNumaNodes"' ;; - umf_test-provider_os_memory) + ./test/umf_test-provider_os_memory) FILTER='--gtest_filter="-osProviderTest/umfIpcTest*"' ;; - umf_test-provider_os_memory_config) + ./test/umf_test-provider_os_memory_config) FILTER='--gtest_filter="-*protection_flag_none:*protection_flag_read:*providerConfigTestNumaMode*"' ;; - umf_test-memspace_highest_capacity) + ./test/umf_test-memspace_highest_capacity) FILTER='--gtest_filter="-*highestCapacityVerify*"' ;; - umf_test-provider_os_memory_multiple_numa_nodes) + ./test/umf_test-provider_os_memory_multiple_numa_nodes) FILTER='--gtest_filter="-testNuma.checkModeInterleave*:testNumaNodesAllocations/testNumaOnEachNode.checkNumaNodesAllocations*:testNumaNodesAllocations/testNumaOnEachNode.checkModePreferred*:testNumaNodesAllocations/testNumaOnEachNode.checkModeInterleaveSingleNode*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModePreferredEmptyNodeset*:testNumaNodesAllocationsAllCpus/testNumaOnEachCpu.checkModeLocal*"' ;; - umf_test-memspace_highest_bandwidth) + ./test/umf_test-memspace_highest_bandwidth) FILTER='--gtest_filter="-*allocLocalMt*"' ;; - umf_test-memspace_lowest_latency) + ./test/umf_test-memspace_lowest_latency) FILTER='--gtest_filter="-*allocLocalMt*"' ;; - umf_test-memoryPool) + ./test/umf_test-memoryPool) FILTER='--gtest_filter="-*allocMaxSize*"' ;; + ./examples/umf_example_ipc_ipcapi_*) + echo "- SKIPPED" + continue; # skip testing helper binaries used by the umf_example_ipc_ipcapi_* examples + ;; esac [ "$FILTER" != "" ] && echo -n "($FILTER) " LAST_TEST_FAILED=0 - - if ! HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all ./$test $FILTER >$LOG 2>&1; then + set +e + HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all $test $FILTER >$LOG 2>&1 + RET=$? + set -e + # 125 is the return code when the test is skipped + if [ $RET -ne 0 -a $RET -ne 125 ]; then LAST_TEST_FAILED=1 ANY_TEST_FAILED=1 - echo "(valgrind FAILED) " - echo "Command: HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all ./$test $FILTER >$LOG 2>&1" + echo "(valgrind FAILED RV=$RET) " + echo "Command: HWLOC_CPUID_PATH=./cpuid valgrind $OPTION $OPT_SUP --gen-suppressions=all $test $FILTER >$LOG 2>&1" echo "Output:" cat $LOG echo "=====================" @@ -147,7 +156,7 @@ for test in $(ls -1 umf_test-*); do # grep for "ERROR SUMMARY" with errors (there can be many lines with "ERROR SUMMARY") grep -e "ERROR SUMMARY:" $LOG | grep -v -e "ERROR SUMMARY: 0 errors from 0 contexts" > $ERR || true if [ $LAST_TEST_FAILED -eq 0 -a $(cat $ERR | wc -l) -eq 0 ]; then - echo "- OK" + [ $RET -eq 0 ] && echo "- OK" || echo "- SKIPPED" rm -f $LOG $ERR else echo "- FAILED!" From 7f005d9b66e26e23e5243de4a39dc53d75891447 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 12 Dec 2024 17:45:31 +0100 Subject: [PATCH 47/82] Move PMDK CTL sources This commit introduces sources of the CTL. The CTL sources are copied from: https://github.com/pmem/pmdk/tree/master/src/common --- src/ctl/ctl.c | 563 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/ctl/ctl.h | 206 ++++++++++++++++++ 2 files changed, 769 insertions(+) create mode 100644 src/ctl/ctl.c create mode 100644 src/ctl/ctl.h diff --git a/src/ctl/ctl.c b/src/ctl/ctl.c new file mode 100644 index 0000000000..330a5fa472 --- /dev/null +++ b/src/ctl/ctl.c @@ -0,0 +1,563 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2016-2020, Intel Corporation */ + +/* + * ctl.c -- implementation of the interface for examination and modification of + * the library's internal state + */ +#include "ctl.h" +#include "alloc.h" +#include "os.h" + +#define CTL_MAX_ENTRIES 100 + +#define MAX_CONFIG_FILE_LEN (1 << 20) /* 1 megabyte */ + +#define CTL_STRING_QUERY_SEPARATOR ";" +#define CTL_NAME_VALUE_SEPARATOR "=" +#define CTL_QUERY_NODE_SEPARATOR "." +#define CTL_VALUE_ARG_SEPARATOR "," + +static int ctl_global_first_free = 0; +static struct ctl_node CTL_NODE(global)[CTL_MAX_ENTRIES]; + +/* + * This is the top level node of the ctl tree structure. Each node can contain + * children and leaf nodes. + * + * Internal nodes simply create a new path in the tree whereas child nodes are + * the ones providing the read/write functionality by the means of callbacks. + * + * Each tree node must be NULL-terminated, CTL_NODE_END macro is provided for + * convenience. + */ +struct ctl { + struct ctl_node root[CTL_MAX_ENTRIES]; + int first_free; +}; + +/* + * ctl_find_node -- (internal) searches for a matching entry point in the + * provided nodes + * + * The caller is responsible for freeing all of the allocated indexes, + * regardless of the return value. + */ +static const struct ctl_node *ctl_find_node(const struct ctl_node *nodes, + const char *name, + struct ctl_indexes *indexes) { + LOG(3, "nodes %p name %s indexes %p", nodes, name, indexes); + + const struct ctl_node *n = NULL; + char *sptr = NULL; + char *parse_str = Strdup(name); + if (parse_str == NULL) { + return NULL; + } + + char *node_name = strtok_r(parse_str, CTL_QUERY_NODE_SEPARATOR, &sptr); + + /* + * Go through the string and separate tokens that correspond to nodes + * in the main ctl tree. + */ + while (node_name != NULL) { + char *endptr; + /* + * Ignore errno from strtol: FreeBSD returns EINVAL if no + * conversion is performed. Linux does not, but endptr + * check is valid in both cases. + */ + int tmp_errno = errno; + long index_value = strtol(node_name, &endptr, 0); + errno = tmp_errno; + struct ctl_index *index_entry = NULL; + if (endptr != node_name) { /* a valid index */ + index_entry = Malloc(sizeof(*index_entry)); + if (index_entry == NULL) { + goto error; + } + index_entry->value = index_value; + PMDK_SLIST_INSERT_HEAD(indexes, index_entry, entry); + } + + for (n = &nodes[0]; n->name != NULL; ++n) { + if (index_entry && n->type == CTL_NODE_INDEXED) { + break; + } else if (strcmp(n->name, node_name) == 0) { + break; + } + } + if (n->name == NULL) { + goto error; + } + + if (index_entry) { + index_entry->name = n->name; + } + + nodes = n->children; + node_name = strtok_r(NULL, CTL_QUERY_NODE_SEPARATOR, &sptr); + } + + Free(parse_str); + return n; + +error: + Free(parse_str); + return NULL; +} + +/* + * ctl_delete_indexes -- + * (internal) removes and frees all entries on the index list + */ +static void ctl_delete_indexes(struct ctl_indexes *indexes) { + while (!PMDK_SLIST_EMPTY(indexes)) { + struct ctl_index *index = PMDK_SLIST_FIRST(indexes); + PMDK_SLIST_REMOVE_HEAD(indexes, entry); + Free(index); + } +} + +/* + * ctl_parse_args -- (internal) parses a string argument based on the node + * structure + */ +static void *ctl_parse_args(const struct ctl_argument *arg_proto, char *arg) { + ASSERTne(arg, NULL); + + char *dest_arg = Malloc(arg_proto->dest_size); + if (dest_arg == NULL) { + ERR("!Malloc"); + return NULL; + } + + char *sptr = NULL; + char *arg_sep = strtok_r(arg, CTL_VALUE_ARG_SEPARATOR, &sptr); + for (const struct ctl_argument_parser *p = arg_proto->parsers; + p->parser != NULL; ++p) { + ASSERT(p->dest_offset + p->dest_size <= arg_proto->dest_size); + if (arg_sep == NULL) { + ERR("!strtok_r"); + goto error_parsing; + } + + if (p->parser(arg_sep, dest_arg + p->dest_offset, p->dest_size) != 0) { + goto error_parsing; + } + + arg_sep = strtok_r(NULL, CTL_VALUE_ARG_SEPARATOR, &sptr); + } + + return dest_arg; + +error_parsing: + Free(dest_arg); + return NULL; +} + +/* + * ctl_query_get_real_args -- (internal) returns a pointer with actual argument + * structure as required by the node callback + */ +static void *ctl_query_get_real_args(const struct ctl_node *n, void *write_arg, + enum ctl_query_source source) { + void *real_arg = NULL; + switch (source) { + case CTL_QUERY_CONFIG_INPUT: + real_arg = ctl_parse_args(n->arg, write_arg); + break; + case CTL_QUERY_PROGRAMMATIC: + real_arg = write_arg; + break; + default: + ASSERT(0); + break; + } + + return real_arg; +} + +/* + * ctl_query_cleanup_real_args -- (internal) cleanups relevant argument + * structures allocated as a result of the get_real_args call + */ +static void ctl_query_cleanup_real_args(const struct ctl_node *n, + void *real_arg, + enum ctl_query_source source) { + switch (source) { + case CTL_QUERY_CONFIG_INPUT: + Free(real_arg); + break; + case CTL_QUERY_PROGRAMMATIC: + break; + default: + ASSERT(0); + break; + } +} + +/* + * ctl_exec_query_read -- (internal) calls the read callback of a node + */ +static int ctl_exec_query_read(void *ctx, const struct ctl_node *n, + enum ctl_query_source source, void *arg, + struct ctl_indexes *indexes) { + if (arg == NULL) { + ERR("read queries require non-NULL argument"); + errno = EINVAL; + return -1; + } + + return n->cb[CTL_QUERY_READ](ctx, source, arg, indexes); +} + +/* + * ctl_exec_query_write -- (internal) calls the write callback of a node + */ +static int ctl_exec_query_write(void *ctx, const struct ctl_node *n, + enum ctl_query_source source, void *arg, + struct ctl_indexes *indexes) { + if (arg == NULL) { + ERR("write queries require non-NULL argument"); + errno = EINVAL; + return -1; + } + + void *real_arg = ctl_query_get_real_args(n, arg, source); + if (real_arg == NULL) { + LOG(1, "Invalid arguments"); + return -1; + } + + int ret = n->cb[CTL_QUERY_WRITE](ctx, source, real_arg, indexes); + ctl_query_cleanup_real_args(n, real_arg, source); + + return ret; +} + +/* + * ctl_exec_query_runnable -- (internal) calls the run callback of a node + */ +static int ctl_exec_query_runnable(void *ctx, const struct ctl_node *n, + enum ctl_query_source source, void *arg, + struct ctl_indexes *indexes) { + return n->cb[CTL_QUERY_RUNNABLE](ctx, source, arg, indexes); +} + +static int (*ctl_exec_query[MAX_CTL_QUERY_TYPE])( + void *ctx, const struct ctl_node *n, enum ctl_query_source source, + void *arg, struct ctl_indexes *indexes) = { + ctl_exec_query_read, + ctl_exec_query_write, + ctl_exec_query_runnable, +}; + +/* + * ctl_query -- (internal) parses the name and calls the appropriate methods + * from the ctl tree + */ +int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, + const char *name, enum ctl_query_type type, void *arg) { + LOG(3, "ctl %p ctx %p source %d name %s type %d arg %p", ctl, ctx, source, + name, type, arg); + + if (name == NULL) { + ERR("invalid query"); + errno = EINVAL; + return -1; + } + + /* + * All of the indexes are put on this list so that the handlers can + * easily retrieve the index values. The list is cleared once the ctl + * query has been handled. + */ + struct ctl_indexes indexes; + PMDK_SLIST_INIT(&indexes); + + int ret = -1; + + const struct ctl_node *n = ctl_find_node(CTL_NODE(global), name, &indexes); + + if (n == NULL && ctl) { + ctl_delete_indexes(&indexes); + n = ctl_find_node(ctl->root, name, &indexes); + } + + if (n == NULL || n->type != CTL_NODE_LEAF || n->cb[type] == NULL) { + ERR("invalid query entry point %s", name); + errno = EINVAL; + goto out; + } + + ret = ctl_exec_query[type](ctx, n, source, arg, &indexes); + +out: + ctl_delete_indexes(&indexes); + + return ret; +} + +/* + * ctl_register_module_node -- adds a new node to the CTL tree root. + */ +void ctl_register_module_node(struct ctl *c, const char *name, + struct ctl_node *n) { + struct ctl_node *nnode = c == NULL + ? &CTL_NODE(global)[ctl_global_first_free++] + : &c->root[c->first_free++]; + + nnode->children = n; + nnode->type = CTL_NODE_NAMED; + nnode->name = name; +} + +/* + * ctl_parse_query -- (internal) splits an entire query string + * into name and value + */ +static int ctl_parse_query(char *qbuf, char **name, char **value) { + if (qbuf == NULL) { + return -1; + } + + char *sptr; + *name = strtok_r(qbuf, CTL_NAME_VALUE_SEPARATOR, &sptr); + if (*name == NULL) { + return -1; + } + + *value = strtok_r(NULL, CTL_NAME_VALUE_SEPARATOR, &sptr); + if (*value == NULL) { + return -1; + } + + /* the value itself mustn't include CTL_NAME_VALUE_SEPARATOR */ + char *extra = strtok_r(NULL, CTL_NAME_VALUE_SEPARATOR, &sptr); + if (extra != NULL) { + return -1; + } + + return 0; +} + +/* + * ctl_load_config -- executes the entire query collection from a provider + */ +static int ctl_load_config(struct ctl *ctl, void *ctx, char *buf) { + int r = 0; + char *sptr = NULL; /* for internal use of strtok */ + char *name; + char *value; + + ASSERTne(buf, NULL); + + char *qbuf = strtok_r(buf, CTL_STRING_QUERY_SEPARATOR, &sptr); + while (qbuf != NULL) { + r = ctl_parse_query(qbuf, &name, &value); + if (r != 0) { + ERR("failed to parse query %s", qbuf); + return -1; + } + + r = ctl_query(ctl, ctx, CTL_QUERY_CONFIG_INPUT, name, CTL_QUERY_WRITE, + value); + + if (r < 0 && ctx != NULL) { + return -1; + } + + qbuf = strtok_r(NULL, CTL_STRING_QUERY_SEPARATOR, &sptr); + } + + return 0; +} + +/* + * ctl_load_config_from_string -- loads obj configuration from string + */ +int ctl_load_config_from_string(struct ctl *ctl, void *ctx, + const char *cfg_string) { + LOG(3, "ctl %p ctx %p cfg_string \"%s\"", ctl, ctx, cfg_string); + + char *buf = Strdup(cfg_string); + if (buf == NULL) { + ERR("!Strdup"); + return -1; + } + + int ret = ctl_load_config(ctl, ctx, buf); + + Free(buf); + return ret; +} + +/* + * ctl_load_config_from_file -- loads obj configuration from file + * + * This function opens up the config file, allocates a buffer of size equal to + * the size of the file, reads its content and sanitizes it for ctl_load_config. + */ +int ctl_load_config_from_file(struct ctl *ctl, void *ctx, + const char *cfg_file) { + LOG(3, "ctl %p ctx %p cfg_file \"%s\"", ctl, ctx, cfg_file); + + int ret = -1; + + FILE *fp = os_fopen(cfg_file, "r"); + if (fp == NULL) { + return ret; + } + + int err; + if ((err = fseek(fp, 0, SEEK_END)) != 0) { + goto error_file_parse; + } + + long fsize = ftell(fp); + if (fsize == -1) { + goto error_file_parse; + } + + if (fsize > MAX_CONFIG_FILE_LEN) { + ERR("Config file too large"); + goto error_file_parse; + } + + if ((err = fseek(fp, 0, SEEK_SET)) != 0) { + goto error_file_parse; + } + + char *buf = Zalloc((size_t)fsize + 1); /* +1 for NULL-termination */ + if (buf == NULL) { + ERR("!Zalloc"); + goto error_file_parse; + } + + size_t bufpos = 0; + + int c; + int is_comment_section = 0; + while ((c = fgetc(fp)) != EOF) { + if (c == '#') { + is_comment_section = 1; + } else if (c == '\n') { + is_comment_section = 0; + } else if (!is_comment_section && !isspace(c)) { + buf[bufpos++] = (char)c; + } + } + + ret = ctl_load_config(ctl, ctx, buf); + + Free(buf); + +error_file_parse: + (void)fclose(fp); + return ret; +} + +/* + * ctl_new -- allocates and initializes ctl data structures + */ +struct ctl *ctl_new(void) { + struct ctl *c = Zalloc(sizeof(struct ctl)); + if (c == NULL) { + ERR("!Zalloc"); + return NULL; + } + + c->first_free = 0; + return c; +} + +/* + * ctl_delete -- deletes ctl + */ +void ctl_delete(struct ctl *c) { Free(c); } + +/* + * ctl_parse_ll -- (internal) parses and returns a long long signed integer + */ +static long long ctl_parse_ll(const char *str) { + char *endptr; + int olderrno = errno; + errno = 0; + long long val = strtoll(str, &endptr, 0); + if (endptr == str || errno != 0) { + return LLONG_MIN; + } + errno = olderrno; + + return val; +} + +/* + * ctl_arg_boolean -- checks whether the provided argument contains + * either a 1 or y or Y. + */ +int ctl_arg_boolean(const void *arg, void *dest, size_t dest_size) { + int *intp = dest; + char in = ((char *)arg)[0]; + + if (tolower(in) == 'y' || in == '1') { + *intp = 1; + return 0; + } else if (tolower(in) == 'n' || in == '0') { + *intp = 0; + return 0; + } + + return -1; +} + +/* + * ctl_arg_integer -- parses signed integer argument + */ +int ctl_arg_integer(const void *arg, void *dest, size_t dest_size) { + long long val = ctl_parse_ll(arg); + if (val == LLONG_MIN) { + return -1; + } + + switch (dest_size) { + case sizeof(int): + if (val > INT_MAX || val < INT_MIN) { + return -1; + } + *(int *)dest = (int)val; + break; + case sizeof(long long): + *(long long *)dest = val; + break; + case sizeof(uint8_t): + if (val > UINT8_MAX || val < 0) { + return -1; + } + *(uint8_t *)dest = (uint8_t)val; + break; + default: + ERR("invalid destination size %zu", dest_size); + errno = EINVAL; + return -1; + } + + return 0; +} + +/* + * ctl_arg_string -- verifies length and copies a string argument into a zeroed + * buffer + */ +int ctl_arg_string(const void *arg, void *dest, size_t dest_size) { + /* check if the incoming string is longer or equal to dest_size */ + if (strnlen(arg, dest_size) == dest_size) { + return -1; + } + + strncpy(dest, arg, dest_size); + + return 0; +} diff --git a/src/ctl/ctl.h b/src/ctl/ctl.h new file mode 100644 index 0000000000..12b9d18fd6 --- /dev/null +++ b/src/ctl/ctl.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright 2016-2020, Intel Corporation */ + +/* + * ctl.h -- internal declaration of statistics and control related structures + */ + +#ifndef PMDK_CTL_H +#define PMDK_CTL_H 1 + +#include "errno.h" +#include "out.h" +#include "queue.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct ctl; + +struct ctl_index { + const char *name; + long value; + PMDK_SLIST_ENTRY(ctl_index) entry; +}; + +PMDK_SLIST_HEAD(ctl_indexes, ctl_index); + +enum ctl_query_source { + CTL_UNKNOWN_QUERY_SOURCE, + /* query executed directly from the program */ + CTL_QUERY_PROGRAMMATIC, + /* query executed from the config file */ + CTL_QUERY_CONFIG_INPUT, + + MAX_CTL_QUERY_SOURCE +}; + +enum ctl_query_type { + CTL_QUERY_READ, + CTL_QUERY_WRITE, + CTL_QUERY_RUNNABLE, + + MAX_CTL_QUERY_TYPE +}; + +typedef int (*node_callback)(void *ctx, enum ctl_query_source type, void *arg, + struct ctl_indexes *indexes); + +enum ctl_node_type { + CTL_NODE_UNKNOWN, + CTL_NODE_NAMED, + CTL_NODE_LEAF, + CTL_NODE_INDEXED, + + MAX_CTL_NODE +}; + +typedef int (*ctl_arg_parser)(const void *arg, void *dest, size_t dest_size); + +struct ctl_argument_parser { + size_t dest_offset; /* offset of the field inside of the argument */ + size_t dest_size; /* size of the field inside of the argument */ + ctl_arg_parser parser; +}; + +struct ctl_argument { + size_t dest_size; /* sizeof the entire argument */ + struct ctl_argument_parser parsers[]; /* array of 'fields' in arg */ +}; + +#define sizeof_member(t, m) sizeof(((t *)0)->m) + +#define CTL_ARG_PARSER(t, p) \ + { 0, sizeof(t), p } + +#define CTL_ARG_PARSER_STRUCT(t, m, p) \ + { offsetof(t, m), sizeof_member(t, m), p } + +#define CTL_ARG_PARSER_END \ + { 0, 0, NULL } + +/* + * CTL Tree node structure, do not use directly. All the necessary functionality + * is provided by the included macros. + */ +struct ctl_node { + const char *name; + enum ctl_node_type type; + + node_callback cb[MAX_CTL_QUERY_TYPE]; + const struct ctl_argument *arg; + + const struct ctl_node *children; +}; + +struct ctl *ctl_new(void); +void ctl_delete(struct ctl *stats); + +int ctl_load_config_from_string(struct ctl *ctl, void *ctx, + const char *cfg_string); +int ctl_load_config_from_file(struct ctl *ctl, void *ctx, const char *cfg_file); + +/* Use through CTL_REGISTER_MODULE, never directly */ +void ctl_register_module_node(struct ctl *c, const char *name, + struct ctl_node *n); + +int ctl_arg_boolean(const void *arg, void *dest, size_t dest_size); +#define CTL_ARG_BOOLEAN \ + {sizeof(int), {{0, sizeof(int), ctl_arg_boolean}, CTL_ARG_PARSER_END}}; + +int ctl_arg_integer(const void *arg, void *dest, size_t dest_size); +#define CTL_ARG_INT \ + {sizeof(int), {{0, sizeof(int), ctl_arg_integer}, CTL_ARG_PARSER_END}}; + +#define CTL_ARG_LONG_LONG \ + {sizeof(long long), \ + {{0, sizeof(long long), ctl_arg_integer}, CTL_ARG_PARSER_END}}; + +int ctl_arg_string(const void *arg, void *dest, size_t dest_size); +#define CTL_ARG_STRING(len) \ + {len, {{0, len, ctl_arg_string}, CTL_ARG_PARSER_END}}; + +#define CTL_STR(name) #name + +#define CTL_NODE_END \ + { NULL, CTL_NODE_UNKNOWN, {NULL, NULL, NULL}, NULL, NULL } + +#define CTL_NODE(name, ...) ctl_node_##__VA_ARGS__##_##name + +int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, + const char *name, enum ctl_query_type type, void *arg); + +/* Declaration of a new child node */ +#define CTL_CHILD(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_NAMED, {NULL, NULL, NULL}, NULL, \ + (struct ctl_node *)CTL_NODE(name, __VA_ARGS__) \ + } + +/* Declaration of a new indexed node */ +#define CTL_INDEXED(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_INDEXED, {NULL, NULL, NULL}, NULL, \ + (struct ctl_node *)CTL_NODE(name, __VA_ARGS__) \ + } + +#define CTL_READ_HANDLER(name, ...) ctl_##__VA_ARGS__##_##name##_read + +#define CTL_WRITE_HANDLER(name, ...) ctl_##__VA_ARGS__##_##name##_write + +#define CTL_RUNNABLE_HANDLER(name, ...) ctl_##__VA_ARGS__##_##name##_runnable + +#define CTL_ARG(name) ctl_arg_##name + +/* + * Declaration of a new read-only leaf. If used the corresponding read function + * must be declared by CTL_READ_HANDLER macro. + */ +#define CTL_LEAF_RO(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {CTL_READ_HANDLER(name, __VA_ARGS__), NULL, NULL}, NULL, NULL \ + } + +/* + * Declaration of a new write-only leaf. If used the corresponding write + * function must be declared by CTL_WRITE_HANDLER macro. + */ +#define CTL_LEAF_WO(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {NULL, CTL_WRITE_HANDLER(name, __VA_ARGS__), NULL}, \ + &CTL_ARG(name), NULL \ + } + +/* + * Declaration of a new runnable leaf. If used the corresponding run + * function must be declared by CTL_RUNNABLE_HANDLER macro. + */ +#define CTL_LEAF_RUNNABLE(name, ...) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {NULL, NULL, CTL_RUNNABLE_HANDLER(name, __VA_ARGS__)}, NULL, NULL \ + } + +/* + * Declaration of a new read-write leaf. If used both read and write function + * must be declared by CTL_READ_HANDLER and CTL_WRITE_HANDLER macros. + */ +#define CTL_LEAF_RW(name) \ + { \ + CTL_STR(name), CTL_NODE_LEAF, \ + {CTL_READ_HANDLER(name), CTL_WRITE_HANDLER(name), NULL}, \ + &CTL_ARG(name), NULL \ + } + +#define CTL_REGISTER_MODULE(_ctl, name) \ + ctl_register_module_node((_ctl), CTL_STR(name), \ + (struct ctl_node *)CTL_NODE(name)) + +#ifdef __cplusplus +} +#endif + +#endif From 580420707e4ccc43533e22eb44b04fe01d8d42ba Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 12 Dec 2024 17:52:14 +0100 Subject: [PATCH 48/82] [CMake] Disable pedantic mode and disable cast qualifier warning --- cmake/helpers.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 0a165bc3af..ddcd5f03db 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -232,9 +232,8 @@ function(add_umf_target_compile_options name) PRIVATE -fPIC -Wall -Wextra - -Wpedantic -Wformat-security - -Wcast-qual + -Wno-cast-qual $<$:-fdiagnostics-color=auto>) if(CMAKE_BUILD_TYPE STREQUAL "Release") target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2) From abb24e7b583c118cd39e14daf168d8fb6bad63d2 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 12 Dec 2024 17:46:32 +0100 Subject: [PATCH 49/82] CTL: Add a CTL functionality to the UMF Signed-off-by: Krzysztof Filipek --- cmake/helpers.cmake | 3 +- src/CMakeLists.txt | 3 + src/ctl/ctl.c | 233 ++++++++++++++++++++++++------------------- src/ctl/ctl.h | 40 +++++--- test/CMakeLists.txt | 5 + test/ctl/config.txt | 1 + test/ctl/ctl_debug.c | 128 ++++++++++++++++++++++++ test/ctl/ctl_debug.h | 32 ++++++ test/ctl/test.cpp | 93 +++++++++++++++++ 9 files changed, 422 insertions(+), 116 deletions(-) create mode 100644 test/ctl/config.txt create mode 100644 test/ctl/ctl_debug.c create mode 100644 test/ctl/ctl_debug.h create mode 100644 test/ctl/test.cpp diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index ddcd5f03db..56692ff6ec 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -233,7 +233,8 @@ function(add_umf_target_compile_options name) -Wall -Wextra -Wformat-security - -Wno-cast-qual + -Wno-cast-qual # TODO: remove this when const qualifier drop + # will be solved in CTL $<$:-fdiagnostics-color=auto>) if(CMAKE_BUILD_TYPE STREQUAL "Release") target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ffd928f7cc..4edaa59571 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,6 +26,8 @@ add_subdirectory(coarse) set(UMF_LIBS $ $) +set(CTL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ctl/ctl.c) + if(LINUX) set(BA_SOURCES ${BA_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/base_alloc/base_alloc_linux.c) @@ -45,6 +47,7 @@ set(HWLOC_DEPENDENT_SOURCES topology.c) set(UMF_SOURCES ${BA_SOURCES} + ${CTL_SOURCES} libumf.c ipc.c ipc_cache.c diff --git a/src/ctl/ctl.c b/src/ctl/ctl.c index 330a5fa472..124d56f6cc 100644 --- a/src/ctl/ctl.c +++ b/src/ctl/ctl.c @@ -1,13 +1,38 @@ +/* + * + * Copyright (C) 2016-2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +// This file was originally under following license: // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2016-2020, Intel Corporation */ +/* Copyright 2016-2024, Intel Corporation */ /* * ctl.c -- implementation of the interface for examination and modification of - * the library's internal state + * the library's internal state */ + #include "ctl.h" -#include "alloc.h" -#include "os.h" + +#include +#include +#include +#include +#include + +#include "base_alloc/base_alloc_global.h" +#include "utils/utils_common.h" +#include "utlist.h" + +#ifdef _WIN32 +#define strtok_r strtok_s +#else +#include +#endif #define CTL_MAX_ENTRIES 100 @@ -19,7 +44,7 @@ #define CTL_VALUE_ARG_SEPARATOR "," static int ctl_global_first_free = 0; -static struct ctl_node CTL_NODE(global)[CTL_MAX_ENTRIES]; +static struct ctl_node CTL_NODE(global, )[CTL_MAX_ENTRIES]; /* * This is the top level node of the ctl tree structure. Each node can contain @@ -36,18 +61,33 @@ struct ctl { int first_free; }; +void *Zalloc(size_t sz) { + void *ptr = umf_ba_global_alloc(sz); + if (ptr) { + memset(ptr, 0, sz); + } + return ptr; +} + +char *Strdup(const char *s) { + size_t len = strlen(s) + 1; + char *p = umf_ba_global_alloc(len); + if (p) { + memcpy(p, s, len); + } + return p; +} + /* * ctl_find_node -- (internal) searches for a matching entry point in the - * provided nodes + * provided nodes * * The caller is responsible for freeing all of the allocated indexes, * regardless of the return value. */ static const struct ctl_node *ctl_find_node(const struct ctl_node *nodes, const char *name, - struct ctl_indexes *indexes) { - LOG(3, "nodes %p name %s indexes %p", nodes, name, indexes); - + struct ctl_index_utlist *indexes) { const struct ctl_node *n = NULL; char *sptr = NULL; char *parse_str = Strdup(name); @@ -58,27 +98,27 @@ static const struct ctl_node *ctl_find_node(const struct ctl_node *nodes, char *node_name = strtok_r(parse_str, CTL_QUERY_NODE_SEPARATOR, &sptr); /* - * Go through the string and separate tokens that correspond to nodes - * in the main ctl tree. - */ + * Go through the string and separate tokens that correspond to nodes + * in the main ctl tree. + */ while (node_name != NULL) { char *endptr; /* - * Ignore errno from strtol: FreeBSD returns EINVAL if no - * conversion is performed. Linux does not, but endptr - * check is valid in both cases. - */ + * Ignore errno from strtol: FreeBSD returns EINVAL if no + * conversion is performed. Linux does not, but endptr + * check is valid in both cases. + */ int tmp_errno = errno; long index_value = strtol(node_name, &endptr, 0); errno = tmp_errno; - struct ctl_index *index_entry = NULL; + struct ctl_index_utlist *index_entry = NULL; if (endptr != node_name) { /* a valid index */ - index_entry = Malloc(sizeof(*index_entry)); + index_entry = umf_ba_global_alloc(sizeof(*index_entry)); if (index_entry == NULL) { goto error; } index_entry->value = index_value; - PMDK_SLIST_INSERT_HEAD(indexes, index_entry, entry); + LL_PREPEND(indexes, index_entry); } for (n = &nodes[0]; n->name != NULL; ++n) { @@ -100,36 +140,38 @@ static const struct ctl_node *ctl_find_node(const struct ctl_node *nodes, node_name = strtok_r(NULL, CTL_QUERY_NODE_SEPARATOR, &sptr); } - Free(parse_str); + umf_ba_global_free(parse_str); return n; error: - Free(parse_str); + umf_ba_global_free(parse_str); return NULL; } /* * ctl_delete_indexes -- - * (internal) removes and frees all entries on the index list + * (internal) removes and frees all entries on the index list */ -static void ctl_delete_indexes(struct ctl_indexes *indexes) { - while (!PMDK_SLIST_EMPTY(indexes)) { - struct ctl_index *index = PMDK_SLIST_FIRST(indexes); - PMDK_SLIST_REMOVE_HEAD(indexes, entry); - Free(index); +static void ctl_delete_indexes(struct ctl_index_utlist *indexes) { + if (!indexes) { + return; + } + struct ctl_index_utlist *elem, *tmp; + LL_FOREACH_SAFE(indexes, elem, tmp) { + LL_DELETE(indexes, elem); + if (elem) { + umf_ba_global_free(elem); + } } } /* * ctl_parse_args -- (internal) parses a string argument based on the node - * structure + * structure */ static void *ctl_parse_args(const struct ctl_argument *arg_proto, char *arg) { - ASSERTne(arg, NULL); - - char *dest_arg = Malloc(arg_proto->dest_size); + char *dest_arg = umf_ba_global_alloc(arg_proto->dest_size); if (dest_arg == NULL) { - ERR("!Malloc"); return NULL; } @@ -137,9 +179,7 @@ static void *ctl_parse_args(const struct ctl_argument *arg_proto, char *arg) { char *arg_sep = strtok_r(arg, CTL_VALUE_ARG_SEPARATOR, &sptr); for (const struct ctl_argument_parser *p = arg_proto->parsers; p->parser != NULL; ++p) { - ASSERT(p->dest_offset + p->dest_size <= arg_proto->dest_size); if (arg_sep == NULL) { - ERR("!strtok_r"); goto error_parsing; } @@ -153,13 +193,13 @@ static void *ctl_parse_args(const struct ctl_argument *arg_proto, char *arg) { return dest_arg; error_parsing: - Free(dest_arg); + umf_ba_global_free(dest_arg); return NULL; } /* * ctl_query_get_real_args -- (internal) returns a pointer with actual argument - * structure as required by the node callback + * structure as required by the node callback */ static void *ctl_query_get_real_args(const struct ctl_node *n, void *write_arg, enum ctl_query_source source) { @@ -172,7 +212,6 @@ static void *ctl_query_get_real_args(const struct ctl_node *n, void *write_arg, real_arg = write_arg; break; default: - ASSERT(0); break; } @@ -181,19 +220,21 @@ static void *ctl_query_get_real_args(const struct ctl_node *n, void *write_arg, /* * ctl_query_cleanup_real_args -- (internal) cleanups relevant argument - * structures allocated as a result of the get_real_args call + * structures allocated as a result of the get_real_args call */ static void ctl_query_cleanup_real_args(const struct ctl_node *n, void *real_arg, enum ctl_query_source source) { + /* suppress unused-parameter errors */ + (void)n; + switch (source) { case CTL_QUERY_CONFIG_INPUT: - Free(real_arg); + umf_ba_global_free(real_arg); break; case CTL_QUERY_PROGRAMMATIC: break; default: - ASSERT(0); break; } } @@ -203,9 +244,8 @@ static void ctl_query_cleanup_real_args(const struct ctl_node *n, */ static int ctl_exec_query_read(void *ctx, const struct ctl_node *n, enum ctl_query_source source, void *arg, - struct ctl_indexes *indexes) { + struct ctl_index_utlist *indexes) { if (arg == NULL) { - ERR("read queries require non-NULL argument"); errno = EINVAL; return -1; } @@ -218,16 +258,14 @@ static int ctl_exec_query_read(void *ctx, const struct ctl_node *n, */ static int ctl_exec_query_write(void *ctx, const struct ctl_node *n, enum ctl_query_source source, void *arg, - struct ctl_indexes *indexes) { + struct ctl_index_utlist *indexes) { if (arg == NULL) { - ERR("write queries require non-NULL argument"); errno = EINVAL; return -1; } void *real_arg = ctl_query_get_real_args(n, arg, source); if (real_arg == NULL) { - LOG(1, "Invalid arguments"); return -1; } @@ -242,13 +280,13 @@ static int ctl_exec_query_write(void *ctx, const struct ctl_node *n, */ static int ctl_exec_query_runnable(void *ctx, const struct ctl_node *n, enum ctl_query_source source, void *arg, - struct ctl_indexes *indexes) { + struct ctl_index_utlist *indexes) { return n->cb[CTL_QUERY_RUNNABLE](ctx, source, arg, indexes); } static int (*ctl_exec_query[MAX_CTL_QUERY_TYPE])( void *ctx, const struct ctl_node *n, enum ctl_query_source source, - void *arg, struct ctl_indexes *indexes) = { + void *arg, struct ctl_index_utlist *indexes) = { ctl_exec_query_read, ctl_exec_query_write, ctl_exec_query_runnable, @@ -256,46 +294,45 @@ static int (*ctl_exec_query[MAX_CTL_QUERY_TYPE])( /* * ctl_query -- (internal) parses the name and calls the appropriate methods - * from the ctl tree + * from the ctl tree */ int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, const char *name, enum ctl_query_type type, void *arg) { - LOG(3, "ctl %p ctx %p source %d name %s type %d arg %p", ctl, ctx, source, - name, type, arg); - if (name == NULL) { - ERR("invalid query"); errno = EINVAL; return -1; } /* - * All of the indexes are put on this list so that the handlers can - * easily retrieve the index values. The list is cleared once the ctl - * query has been handled. - */ - struct ctl_indexes indexes; - PMDK_SLIST_INIT(&indexes); + * All of the indexes are put on this list so that the handlers can + * easily retrieve the index values. The list is cleared once the ctl + * query has been handled. + */ + struct ctl_index_utlist *indexes = NULL; + indexes = Zalloc(sizeof(*indexes)); + if (!indexes) { + return -1; + } int ret = -1; - const struct ctl_node *n = ctl_find_node(CTL_NODE(global), name, &indexes); + const struct ctl_node *n = ctl_find_node(CTL_NODE(global, ), name, indexes); if (n == NULL && ctl) { - ctl_delete_indexes(&indexes); - n = ctl_find_node(ctl->root, name, &indexes); + ctl_delete_indexes(indexes); + indexes = NULL; + n = ctl_find_node(ctl->root, name, indexes); } if (n == NULL || n->type != CTL_NODE_LEAF || n->cb[type] == NULL) { - ERR("invalid query entry point %s", name); errno = EINVAL; goto out; } - ret = ctl_exec_query[type](ctx, n, source, arg, &indexes); + ret = ctl_exec_query[type](ctx, n, source, arg, indexes); out: - ctl_delete_indexes(&indexes); + ctl_delete_indexes(indexes); return ret; } @@ -306,7 +343,7 @@ int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, void ctl_register_module_node(struct ctl *c, const char *name, struct ctl_node *n) { struct ctl_node *nnode = c == NULL - ? &CTL_NODE(global)[ctl_global_first_free++] + ? &CTL_NODE(global, )[ctl_global_first_free++] : &c->root[c->first_free++]; nnode->children = n; @@ -316,14 +353,14 @@ void ctl_register_module_node(struct ctl *c, const char *name, /* * ctl_parse_query -- (internal) splits an entire query string - * into name and value + * into name and value */ static int ctl_parse_query(char *qbuf, char **name, char **value) { if (qbuf == NULL) { return -1; } - char *sptr; + char *sptr = NULL; *name = strtok_r(qbuf, CTL_NAME_VALUE_SEPARATOR, &sptr); if (*name == NULL) { return -1; @@ -351,14 +388,11 @@ static int ctl_load_config(struct ctl *ctl, void *ctx, char *buf) { char *sptr = NULL; /* for internal use of strtok */ char *name; char *value; - - ASSERTne(buf, NULL); - char *qbuf = strtok_r(buf, CTL_STRING_QUERY_SEPARATOR, &sptr); + while (qbuf != NULL) { r = ctl_parse_query(qbuf, &name, &value); if (r != 0) { - ERR("failed to parse query %s", qbuf); return -1; } @@ -380,17 +414,14 @@ static int ctl_load_config(struct ctl *ctl, void *ctx, char *buf) { */ int ctl_load_config_from_string(struct ctl *ctl, void *ctx, const char *cfg_string) { - LOG(3, "ctl %p ctx %p cfg_string \"%s\"", ctl, ctx, cfg_string); - char *buf = Strdup(cfg_string); if (buf == NULL) { - ERR("!Strdup"); return -1; } int ret = ctl_load_config(ctl, ctx, buf); - Free(buf); + umf_ba_global_free(buf); return ret; } @@ -400,13 +431,14 @@ int ctl_load_config_from_string(struct ctl *ctl, void *ctx, * This function opens up the config file, allocates a buffer of size equal to * the size of the file, reads its content and sanitizes it for ctl_load_config. */ +#ifndef _WIN32 // TODO: implement for Windows int ctl_load_config_from_file(struct ctl *ctl, void *ctx, const char *cfg_file) { - LOG(3, "ctl %p ctx %p cfg_file \"%s\"", ctl, ctx, cfg_file); - int ret = -1; + long fsize = 0; + char *buf = NULL; - FILE *fp = os_fopen(cfg_file, "r"); + FILE *fp = fopen(cfg_file, "r"); if (fp == NULL) { return ret; } @@ -416,13 +448,12 @@ int ctl_load_config_from_file(struct ctl *ctl, void *ctx, goto error_file_parse; } - long fsize = ftell(fp); + fsize = ftell(fp); if (fsize == -1) { goto error_file_parse; } if (fsize > MAX_CONFIG_FILE_LEN) { - ERR("Config file too large"); goto error_file_parse; } @@ -430,34 +461,35 @@ int ctl_load_config_from_file(struct ctl *ctl, void *ctx, goto error_file_parse; } - char *buf = Zalloc((size_t)fsize + 1); /* +1 for NULL-termination */ + buf = Zalloc((size_t)fsize + 1); /* +1 for NULL-termination */ if (buf == NULL) { - ERR("!Zalloc"); goto error_file_parse; } - size_t bufpos = 0; - - int c; - int is_comment_section = 0; - while ((c = fgetc(fp)) != EOF) { - if (c == '#') { - is_comment_section = 1; - } else if (c == '\n') { - is_comment_section = 0; - } else if (!is_comment_section && !isspace(c)) { - buf[bufpos++] = (char)c; + { + size_t bufpos = 0; + int c; + int is_comment_section = 0; + while ((c = fgetc(fp)) != EOF) { + if (c == '#') { + is_comment_section = 1; + } else if (c == '\n') { + is_comment_section = 0; + } else if (!is_comment_section && !isspace(c)) { + buf[bufpos++] = (char)c; + } } } ret = ctl_load_config(ctl, ctx, buf); - Free(buf); + umf_ba_global_free(buf); error_file_parse: (void)fclose(fp); return ret; } +#endif /* * ctl_new -- allocates and initializes ctl data structures @@ -465,7 +497,6 @@ int ctl_load_config_from_file(struct ctl *ctl, void *ctx, struct ctl *ctl_new(void) { struct ctl *c = Zalloc(sizeof(struct ctl)); if (c == NULL) { - ERR("!Zalloc"); return NULL; } @@ -476,7 +507,7 @@ struct ctl *ctl_new(void) { /* * ctl_delete -- deletes ctl */ -void ctl_delete(struct ctl *c) { Free(c); } +void ctl_delete(struct ctl *c) { umf_ba_global_free(c); } /* * ctl_parse_ll -- (internal) parses and returns a long long signed integer @@ -496,11 +527,14 @@ static long long ctl_parse_ll(const char *str) { /* * ctl_arg_boolean -- checks whether the provided argument contains - * either a 1 or y or Y. + * either a 1 or y or Y. */ int ctl_arg_boolean(const void *arg, void *dest, size_t dest_size) { + /* suppress unused-parameter errors */ + (void)dest_size; + int *intp = dest; - char in = ((char *)arg)[0]; + char in = ((const char *)arg)[0]; if (tolower(in) == 'y' || in == '1') { *intp = 1; @@ -539,7 +573,6 @@ int ctl_arg_integer(const void *arg, void *dest, size_t dest_size) { *(uint8_t *)dest = (uint8_t)val; break; default: - ERR("invalid destination size %zu", dest_size); errno = EINVAL; return -1; } @@ -549,7 +582,7 @@ int ctl_arg_integer(const void *arg, void *dest, size_t dest_size) { /* * ctl_arg_string -- verifies length and copies a string argument into a zeroed - * buffer + * buffer */ int ctl_arg_string(const void *arg, void *dest, size_t dest_size) { /* check if the incoming string is longer or equal to dest_size */ diff --git a/src/ctl/ctl.h b/src/ctl/ctl.h index 12b9d18fd6..f183abaf3b 100644 --- a/src/ctl/ctl.h +++ b/src/ctl/ctl.h @@ -1,3 +1,13 @@ +/* + * + * Copyright (C) 2016-2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +// This file was originally under following license: /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright 2016-2020, Intel Corporation */ @@ -5,12 +15,11 @@ * ctl.h -- internal declaration of statistics and control related structures */ -#ifndef PMDK_CTL_H -#define PMDK_CTL_H 1 +#ifndef UMF_CTL_H +#define UMF_CTL_H 1 -#include "errno.h" -#include "out.h" -#include "queue.h" +#include +#include #ifdef __cplusplus extern "C" { @@ -18,14 +27,12 @@ extern "C" { struct ctl; -struct ctl_index { +struct ctl_index_utlist { const char *name; long value; - PMDK_SLIST_ENTRY(ctl_index) entry; + struct ctl_index_utlist *next; }; -PMDK_SLIST_HEAD(ctl_indexes, ctl_index); - enum ctl_query_source { CTL_UNKNOWN_QUERY_SOURCE, /* query executed directly from the program */ @@ -45,7 +52,7 @@ enum ctl_query_type { }; typedef int (*node_callback)(void *ctx, enum ctl_query_source type, void *arg, - struct ctl_indexes *indexes); + struct ctl_index_utlist *indexes); enum ctl_node_type { CTL_NODE_UNKNOWN, @@ -65,7 +72,7 @@ struct ctl_argument_parser { }; struct ctl_argument { - size_t dest_size; /* sizeof the entire argument */ + size_t dest_size; /* size of the entire argument */ struct ctl_argument_parser parsers[]; /* array of 'fields' in arg */ }; @@ -114,8 +121,11 @@ int ctl_arg_integer(const void *arg, void *dest, size_t dest_size); {sizeof(int), {{0, sizeof(int), ctl_arg_integer}, CTL_ARG_PARSER_END}}; #define CTL_ARG_LONG_LONG \ - {sizeof(long long), \ - {{0, sizeof(long long), ctl_arg_integer}, CTL_ARG_PARSER_END}}; + { \ + sizeof(long long), { \ + {0, sizeof(long long), ctl_arg_integer}, CTL_ARG_PARSER_END \ + } \ + } int ctl_arg_string(const void *arg, void *dest, size_t dest_size); #define CTL_ARG_STRING(len) \ @@ -191,13 +201,13 @@ int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, #define CTL_LEAF_RW(name) \ { \ CTL_STR(name), CTL_NODE_LEAF, \ - {CTL_READ_HANDLER(name), CTL_WRITE_HANDLER(name), NULL}, \ + {CTL_READ_HANDLER(name, ), CTL_WRITE_HANDLER(name, ), NULL}, \ &CTL_ARG(name), NULL \ } #define CTL_REGISTER_MODULE(_ctl, name) \ ctl_register_module_node((_ctl), CTL_STR(name), \ - (struct ctl_node *)CTL_NODE(name)) + (struct ctl_node *)CTL_NODE(name, )) #ifdef __cplusplus } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b237428669..4b50a88020 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -192,6 +192,11 @@ add_umf_test( SRCS utils/utils_log.cpp ${UMF_UTILS_SOURCES} LIBS ${UMF_LOGGER_LIBS}) +add_umf_test( + NAME ctl + SRCS ctl/test.cpp ctl/ctl_debug.c ../src/ctl/ctl.c ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( NAME utils_common SRCS utils/utils.cpp diff --git a/test/ctl/config.txt b/test/ctl/config.txt new file mode 100644 index 0000000000..5d4f9c62bd --- /dev/null +++ b/test/ctl/config.txt @@ -0,0 +1 @@ +debug.heap.alloc_pattern=321 \ No newline at end of file diff --git a/test/ctl/ctl_debug.c b/test/ctl/ctl_debug.c new file mode 100644 index 0000000000..d523b3f801 --- /dev/null +++ b/test/ctl/ctl_debug.c @@ -0,0 +1,128 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +/* + * ctl_debug.c -- implementation of the debug CTL namespace + */ + +#include "ctl_debug.h" + +static struct ctl *ctl_debug; + +static int alloc_pattern = 0; +static int enable_logging = 0; +static int log_level = 0; + +struct ctl *get_debug_ctl(void) { return ctl_debug; } + +/* + * CTL_WRITE_HANDLER(alloc_pattern) -- sets the alloc_pattern field in heap + */ +static int +CTL_WRITE_HANDLER(alloc_pattern, )(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int arg_in = *(int *)arg; + alloc_pattern = arg_in; + return 0; +} + +/* + * CTL_READ_HANDLER(alloc_pattern) -- returns alloc_pattern heap field + */ +static int CTL_READ_HANDLER(alloc_pattern, )(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int *arg_out = arg; + *arg_out = alloc_pattern; + return 0; +} + +static int +CTL_WRITE_HANDLER(enable_logging, )(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int arg_in = *(int *)arg; + enable_logging = arg_in; + return 0; +} + +static int +CTL_READ_HANDLER(enable_logging, )(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int *arg_out = arg; + *arg_out = enable_logging; + return 0; +} + +static int CTL_WRITE_HANDLER(log_level, )(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int arg_in = *(int *)arg; + log_level = arg_in; + return 0; +} + +static int CTL_READ_HANDLER(log_level, )(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { + /* suppress unused-parameter errors */ + (void)source, (void)indexes, (void)ctx; + + int *arg_out = arg; + *arg_out = log_level; + return 0; +} + +static const struct ctl_argument CTL_ARG(alloc_pattern) = CTL_ARG_LONG_LONG; + +static const struct ctl_argument CTL_ARG(enable_logging) = CTL_ARG_BOOLEAN; + +static const struct ctl_argument CTL_ARG(log_level) = CTL_ARG_INT; + +static const struct ctl_node CTL_NODE(heap, )[] = {CTL_LEAF_RW(alloc_pattern), + CTL_LEAF_RW(enable_logging), + CTL_LEAF_RW(log_level), + + CTL_NODE_END}; + +static const struct ctl_node CTL_NODE(debug, )[] = {CTL_CHILD(heap, ), + + CTL_NODE_END}; + +/* + * debug_ctl_register -- registers ctl nodes for "debug" module + */ +void debug_ctl_register(struct ctl *ctl) { CTL_REGISTER_MODULE(ctl, debug); } + +void initialize_debug_ctl(void) { + ctl_debug = ctl_new(); + debug_ctl_register(ctl_debug); +} + +void deinitialize_debug_ctl(void) { ctl_delete(ctl_debug); } diff --git a/test/ctl/ctl_debug.h b/test/ctl/ctl_debug.h new file mode 100644 index 0000000000..9dd8bade5b --- /dev/null +++ b/test/ctl/ctl_debug.h @@ -0,0 +1,32 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +/* + * ctl_debug.h -- definitions for CTL test + */ + +#ifndef UMF_CTL_DEBUG_H +#define UMF_CTL_DEBUG_H 1 + +#include "../src/ctl/ctl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void debug_ctl_register(struct ctl *ctl); +struct ctl *get_debug_ctl(void); +void initialize_debug_ctl(void); +void deinitialize_debug_ctl(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/ctl/test.cpp b/test/ctl/test.cpp new file mode 100644 index 0000000000..c35759c673 --- /dev/null +++ b/test/ctl/test.cpp @@ -0,0 +1,93 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include "../common/base.hpp" +#include "ctl/ctl.h" +#include "ctl/ctl_debug.h" + +using namespace umf_test; + +TEST_F(test, ctl_debug_read_from_string) { + initialize_debug_ctl(); + auto ctl_handler = get_debug_ctl(); + ctl_load_config_from_string(ctl_handler, NULL, + "debug.heap.alloc_pattern=1"); + + int value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 1); + + // Test setting alloc_pattern to 2 + ctl_load_config_from_string(ctl_handler, NULL, + "debug.heap.alloc_pattern=2"); + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 2); + + // Test setting alloc_pattern to 0 + ctl_load_config_from_string(ctl_handler, NULL, + "debug.heap.alloc_pattern=0"); + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 0); + + // Negative test: non-existent configuration + ASSERT_NE(ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.non_existent", CTL_QUERY_READ, &value), + 0); + + // Negative test: invalid path + ASSERT_NE(ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "invalid.path.alloc_pattern", CTL_QUERY_READ, &value), + 0); + + debug_ctl_register(ctl_handler); + deinitialize_debug_ctl(); +} + +int ctl_config_write_to_file(const char *filename, const char *data) { + FILE *file = fopen(filename == NULL ? "config.txt" : filename, "w+"); + if (file == NULL) { + return -1; + } + fputs(data, file); + fclose(file); + return 0; +} + +TEST_F(test, ctl_debug_read_from_file) { +#ifndef _WIN32 + ASSERT_EQ(ctl_config_write_to_file( + "config.txt", "debug.heap.alloc_pattern=321;\ndebug.heap." + "enable_logging=1;\ndebug.heap.log_level=5;\n"), + 0); + initialize_debug_ctl(); + auto ctl_handler = get_debug_ctl(); + ASSERT_EQ(ctl_load_config_from_file(ctl_handler, NULL, "config.txt"), 0); + + int value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.alloc_pattern", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 321); + + value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, "debug.heap.log_level", + CTL_QUERY_READ, &value); + ASSERT_EQ(value, 5); + + value = 0; + ctl_query(ctl_handler, NULL, CTL_QUERY_PROGRAMMATIC, + "debug.heap.enable_logging", CTL_QUERY_READ, &value); + ASSERT_EQ(value, 1); + + debug_ctl_register(ctl_handler); + deinitialize_debug_ctl(); +#endif +} From 9544998f34c836436f4f0599be83e31c4de0b63a Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 18 Dec 2024 09:55:14 +0100 Subject: [PATCH 50/82] Run only given tests/examples under valgrind Signed-off-by: Lukasz Dorau --- test/test_valgrind.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index be5f817dc3..1b1675bd13 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -8,11 +8,16 @@ set -e WORKSPACE=$1 BUILD_DIR=$2 TOOL=$3 +TESTS=$4 function print_usage() { - echo "$(basename $0) - run all UMF tests and examples under a valgrind tool (memcheck, drd or helgrind)" - echo "This script looks for './test/umf_test-*' and './examples/umf_example_*' executables in the UMF build directory." - echo "Usage: $(basename $0) " + echo "$(basename $0) - run UMF tests and examples under a valgrind tool (memcheck, drd or helgrind)" + echo "Usage: $(basename $0) [tests_examples]" + echo "Where:" + echo + echo "tests_examples - (optional) list of tests or examples to be run (paths relative to the build directory)." + echo " If it is empty, all tests (./test/umf_test-*) and examples (./examples/umf_example_*)" + echo " found in will be run." } if ! valgrind --version > /dev/null; then @@ -71,7 +76,14 @@ echo "Running: \"valgrind $OPTION\" for the following tests:" ANY_TEST_FAILED=0 rm -f umf_test-*.log umf_test-*.err -for test in $(ls -1 ./test/umf_test-* ./examples/umf_example_*); do +[ "$TESTS" = "" ] && TESTS=$(ls -1 ./test/umf_test-* ./examples/umf_example_*) + +for test in $TESTS; do + if [ ! -f $test ]; then + echo + echo "error: the $test (${BUILD_DIR}/$test) file does not exist" + exit 1 + fi [ ! -x $test ] && continue echo "$test - starting ..." echo -n "$test " From 083252af8090fae73eb6a0450fdcfe2cb16bc909 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 18 Dec 2024 10:58:26 +0100 Subject: [PATCH 51/82] Silence hwloc-gather-cpuid Signed-off-by: Lukasz Dorau --- test/test_valgrind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index 1b1675bd13..46bfe7d1cc 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -67,7 +67,7 @@ cd ${BUILD_DIR} mkdir -p cpuid echo "Gathering data for hwloc so it can be run under valgrind:" -hwloc-gather-cpuid ./cpuid +hwloc-gather-cpuid ./cpuid >/dev/null echo echo "Working directory: $(pwd)" From 9f6f8ba08f1ccd248a81c8fdcdd55b0af56201c3 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 18 Dec 2024 10:12:18 +0100 Subject: [PATCH 52/82] Run DAX tests under valgrind Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_dax.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/reusable_dax.yml b/.github/workflows/reusable_dax.yml index af15226d29..f7f4fbe508 100644 --- a/.github/workflows/reusable_dax.yml +++ b/.github/workflows/reusable_dax.yml @@ -31,6 +31,7 @@ env: INSTL_DIR : "${{github.workspace}}/../install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" COVERAGE_NAME : "exports-coverage-dax" + DAX_TESTS: "./test/umf_test-provider_file_memory ./test/umf_test-provider_devdax_memory" jobs: dax: @@ -126,6 +127,12 @@ jobs: UMF_TESTS_FSDAX_PATH_2=${{env.UMF_TESTS_FSDAX_PATH_2}} ctest -C ${{matrix.build_type}} -V -R "file|fsdax" + - name: Run DAX tests under valgrind + run: | + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} memcheck "${{env.DAX_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} drd "${{env.DAX_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} helgrind "${{env.DAX_TESTS}}" + - name: Check coverage if: ${{ matrix.build_type == 'Debug' }} working-directory: ${{env.BUILD_DIR}} From 96b00eef47eef0bd65d4df5e55a17e5794780361 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 19 Dec 2024 11:20:16 +0100 Subject: [PATCH 53/82] Run NUMA tests under valgrind Signed-off-by: Lukasz Dorau --- .github/workflows/reusable_multi_numa.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/reusable_multi_numa.yml b/.github/workflows/reusable_multi_numa.yml index f654789843..f546b05451 100644 --- a/.github/workflows/reusable_multi_numa.yml +++ b/.github/workflows/reusable_multi_numa.yml @@ -10,6 +10,7 @@ env: BUILD_DIR : "${{github.workspace}}/build" COVERAGE_DIR : "${{github.workspace}}/coverage" COVERAGE_NAME : "exports-coverage-multinuma" + NUMA_TESTS: "./test/umf_test-memspace_numa ./test/umf_test-provider_os_memory_multiple_numa_nodes" jobs: multi_numa: @@ -68,6 +69,13 @@ jobs: ./test/umf_test-provider_os_memory_multiple_numa_nodes \ --gtest_filter="-*checkModeLocal/*:*checkModePreferredEmptyNodeset/*:testNuma.checkModeInterleave" + - name: Run NUMA tests under valgrind + if: matrix.os != 'rhel-9.1' + run: | + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} memcheck "${{env.NUMA_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} drd "${{env.NUMA_TESTS}}" + ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{env.BUILD_DIR}} helgrind "${{env.NUMA_TESTS}}" + - name: Check coverage if: ${{ matrix.build_type == 'Debug' && matrix.os == 'ubuntu-22.04' }} working-directory: ${{env.BUILD_DIR}} From d3d1bb197d95c829e914fcac2031c704f6d10ee7 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 19 Dec 2024 11:48:33 +0100 Subject: [PATCH 54/82] Fix name of the proxy_lib_size_threshold test Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bb353a889a..0e12885be9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -455,10 +455,10 @@ if(UMF_PROXY_LIB_ENABLED AND UMF_BUILD_SHARED_LIBRARY) # TODO enable this test on Windows if(LINUX) add_umf_test( - NAME test_proxy_lib_size_threshold + NAME proxy_lib_size_threshold SRCS ${BA_SOURCES_FOR_TEST} test_proxy_lib_size_threshold.cpp LIBS ${UMF_UTILS_FOR_TEST} umf_proxy) - set_property(TEST umf-test_proxy_lib_size_threshold + set_property(TEST umf-proxy_lib_size_threshold PROPERTY ENVIRONMENT UMF_PROXY="size.threshold=64") endif() From f70c7ca14a435b69de7582b2e9e5ea26d15dbdec Mon Sep 17 00:00:00 2001 From: kluszcze Date: Thu, 12 Dec 2024 13:08:51 +0100 Subject: [PATCH 55/82] add python script to run codespell scan Signed-off-by: Katarzyna Luszczewska --- .github/scripts/run-codespell.py | 40 +++++++++++++++++++++++++++ .github/workflows/reusable_checks.yml | 5 +++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/run-codespell.py diff --git a/.github/scripts/run-codespell.py b/.github/scripts/run-codespell.py new file mode 100644 index 0000000000..b87bf37bd5 --- /dev/null +++ b/.github/scripts/run-codespell.py @@ -0,0 +1,40 @@ +""" + Copyright (C) 2024 Intel Corporation + + Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import subprocess # nosec B404 +import logging +import sys + +logging.basicConfig( + level=logging.INFO, format="[%(levelname)s]: [%(asctime)s] %(message)s" +) + + +def codespell_scan(): + try: + codespell_result = subprocess.run( # nosec + [ + "codespell", + "-H", + "--quiet-level=3", + "--skip=./.git,./.venv,./.github/workflows/.spellcheck-conf.toml", + ], + text=True, + stdout=subprocess.PIPE, + ) + if codespell_result.returncode != 0: + for line in codespell_result.stdout.splitlines(): + logging.error(line.strip()) + sys.exit(1) + else: + logging.info("No spelling errors found") + except subprocess.CalledProcessError as ex: + logging.error(ex) + sys.exit(1) + + +codespell_scan() diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index e3e264b0db..6298b98831 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -29,7 +29,7 @@ jobs: python3 -m venv .venv . .venv/bin/activate echo "$PATH" >> $GITHUB_PATH - python3 -m pip install bandit + python3 -m pip install bandit codespell - name: Configure CMake run: > @@ -57,6 +57,9 @@ jobs: with: config: ./.github/workflows/.spellcheck-conf.toml + - name: Run codespell + run: python3 ./.github/scripts/run-codespell.py + # Run Bandit recursively, but omit _deps directory (with 3rd party code) and python's venv - name: Run Bandit run: python3 -m bandit -r . -x '/_deps/,/.venv/' From d93b3d761e6882aa660038ad3cbbf0dabd821f3c Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 19 Dec 2024 14:19:39 +0100 Subject: [PATCH 56/82] Remove unnecessary commas due to removing pedantic option This commit reverts changes that can make code reading more difficult. --- src/ctl/ctl.c | 6 ++--- src/ctl/ctl.h | 4 +-- test/ctl/ctl_debug.c | 58 +++++++++++++++++++++----------------------- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/ctl/ctl.c b/src/ctl/ctl.c index 124d56f6cc..d54e8390ea 100644 --- a/src/ctl/ctl.c +++ b/src/ctl/ctl.c @@ -44,7 +44,7 @@ #define CTL_VALUE_ARG_SEPARATOR "," static int ctl_global_first_free = 0; -static struct ctl_node CTL_NODE(global, )[CTL_MAX_ENTRIES]; +static struct ctl_node CTL_NODE(global)[CTL_MAX_ENTRIES]; /* * This is the top level node of the ctl tree structure. Each node can contain @@ -316,7 +316,7 @@ int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, int ret = -1; - const struct ctl_node *n = ctl_find_node(CTL_NODE(global, ), name, indexes); + const struct ctl_node *n = ctl_find_node(CTL_NODE(global), name, indexes); if (n == NULL && ctl) { ctl_delete_indexes(indexes); @@ -343,7 +343,7 @@ int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, void ctl_register_module_node(struct ctl *c, const char *name, struct ctl_node *n) { struct ctl_node *nnode = c == NULL - ? &CTL_NODE(global, )[ctl_global_first_free++] + ? &CTL_NODE(global)[ctl_global_first_free++] : &c->root[c->first_free++]; nnode->children = n; diff --git a/src/ctl/ctl.h b/src/ctl/ctl.h index f183abaf3b..9327b01afe 100644 --- a/src/ctl/ctl.h +++ b/src/ctl/ctl.h @@ -201,13 +201,13 @@ int ctl_query(struct ctl *ctl, void *ctx, enum ctl_query_source source, #define CTL_LEAF_RW(name) \ { \ CTL_STR(name), CTL_NODE_LEAF, \ - {CTL_READ_HANDLER(name, ), CTL_WRITE_HANDLER(name, ), NULL}, \ + {CTL_READ_HANDLER(name), CTL_WRITE_HANDLER(name), NULL}, \ &CTL_ARG(name), NULL \ } #define CTL_REGISTER_MODULE(_ctl, name) \ ctl_register_module_node((_ctl), CTL_STR(name), \ - (struct ctl_node *)CTL_NODE(name, )) + (struct ctl_node *)CTL_NODE(name)) #ifdef __cplusplus } diff --git a/test/ctl/ctl_debug.c b/test/ctl/ctl_debug.c index d523b3f801..711cb5e179 100644 --- a/test/ctl/ctl_debug.c +++ b/test/ctl/ctl_debug.c @@ -24,10 +24,10 @@ struct ctl *get_debug_ctl(void) { return ctl_debug; } /* * CTL_WRITE_HANDLER(alloc_pattern) -- sets the alloc_pattern field in heap */ -static int -CTL_WRITE_HANDLER(alloc_pattern, )(void *ctx, enum ctl_query_source source, - void *arg, - struct ctl_index_utlist *indexes) { +static int CTL_WRITE_HANDLER(alloc_pattern)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { /* suppress unused-parameter errors */ (void)source, (void)indexes, (void)ctx; @@ -39,10 +39,10 @@ CTL_WRITE_HANDLER(alloc_pattern, )(void *ctx, enum ctl_query_source source, /* * CTL_READ_HANDLER(alloc_pattern) -- returns alloc_pattern heap field */ -static int CTL_READ_HANDLER(alloc_pattern, )(void *ctx, - enum ctl_query_source source, - void *arg, - struct ctl_index_utlist *indexes) { +static int CTL_READ_HANDLER(alloc_pattern)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { /* suppress unused-parameter errors */ (void)source, (void)indexes, (void)ctx; @@ -51,10 +51,10 @@ static int CTL_READ_HANDLER(alloc_pattern, )(void *ctx, return 0; } -static int -CTL_WRITE_HANDLER(enable_logging, )(void *ctx, enum ctl_query_source source, - void *arg, - struct ctl_index_utlist *indexes) { +static int CTL_WRITE_HANDLER(enable_logging)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { /* suppress unused-parameter errors */ (void)source, (void)indexes, (void)ctx; @@ -63,10 +63,10 @@ CTL_WRITE_HANDLER(enable_logging, )(void *ctx, enum ctl_query_source source, return 0; } -static int -CTL_READ_HANDLER(enable_logging, )(void *ctx, enum ctl_query_source source, - void *arg, - struct ctl_index_utlist *indexes) { +static int CTL_READ_HANDLER(enable_logging)(void *ctx, + enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { /* suppress unused-parameter errors */ (void)source, (void)indexes, (void)ctx; @@ -75,10 +75,9 @@ CTL_READ_HANDLER(enable_logging, )(void *ctx, enum ctl_query_source source, return 0; } -static int CTL_WRITE_HANDLER(log_level, )(void *ctx, - enum ctl_query_source source, - void *arg, - struct ctl_index_utlist *indexes) { +static int CTL_WRITE_HANDLER(log_level)(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { /* suppress unused-parameter errors */ (void)source, (void)indexes, (void)ctx; @@ -87,10 +86,9 @@ static int CTL_WRITE_HANDLER(log_level, )(void *ctx, return 0; } -static int CTL_READ_HANDLER(log_level, )(void *ctx, - enum ctl_query_source source, - void *arg, - struct ctl_index_utlist *indexes) { +static int CTL_READ_HANDLER(log_level)(void *ctx, enum ctl_query_source source, + void *arg, + struct ctl_index_utlist *indexes) { /* suppress unused-parameter errors */ (void)source, (void)indexes, (void)ctx; @@ -105,15 +103,15 @@ static const struct ctl_argument CTL_ARG(enable_logging) = CTL_ARG_BOOLEAN; static const struct ctl_argument CTL_ARG(log_level) = CTL_ARG_INT; -static const struct ctl_node CTL_NODE(heap, )[] = {CTL_LEAF_RW(alloc_pattern), - CTL_LEAF_RW(enable_logging), - CTL_LEAF_RW(log_level), +static const struct ctl_node CTL_NODE(heap)[] = {CTL_LEAF_RW(alloc_pattern), + CTL_LEAF_RW(enable_logging), + CTL_LEAF_RW(log_level), - CTL_NODE_END}; + CTL_NODE_END}; -static const struct ctl_node CTL_NODE(debug, )[] = {CTL_CHILD(heap, ), +static const struct ctl_node CTL_NODE(debug)[] = {CTL_CHILD(heap), - CTL_NODE_END}; + CTL_NODE_END}; /* * debug_ctl_register -- registers ctl nodes for "debug" module From e83ee3492730823978c474ecb5141fe1a84a4364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 16 Dec 2024 14:58:26 +0100 Subject: [PATCH 57/82] Minor cleanups and additions in docs includes: - add missing CUDA provider in the web docs, - proxy_pool is enabled by default, move req. info to proxy_lib, - add links in README. --- README.md | 17 ++++++++++++----- scripts/docs_config/api.rst | 13 ++++++++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index df90b6852e..4cd1d8ff58 100644 --- a/README.md +++ b/README.md @@ -122,11 +122,16 @@ List of options provided by CMake: ## Architecture: memory pools and providers -A UMF memory pool is a combination of a pool allocator and a memory provider. A memory provider is responsible for coarse-grained memory allocations and management of memory pages, while the pool allocator controls memory pooling and handles fine-grained memory allocations. +A UMF memory pool is a combination of a pool allocator and a memory provider. A memory provider is responsible for +coarse-grained memory allocations and management of memory pages, while the pool allocator controls memory pooling +and handles fine-grained memory allocations. Pool allocator can leverage existing allocators (e.g. jemalloc or tbbmalloc) or be written from scratch. -UMF comes with predefined pool allocators (see include/pool) and providers (see include/provider). UMF can also work with user-defined pools and providers that implement a specific interface (see include/umf/memory_pool_ops.h and include/umf/memory_provider_ops.h). +UMF comes with predefined pool allocators (see [`include/umf/pools`](include/umf/pools)) and providers +(see [`include/umf/providers`](include/umf/providers)). UMF can also work with user-defined pools and +providers that implement a specific interface (see [`include/umf/memory_pool_ops.h`](include/umf/memory_pool_ops.h) +and [`include/umf/memory_provider_ops.h`](include/umf/memory_provider_ops.h)). More detailed documentation is available here: https://oneapi-src.github.io/unified-memory-framework/ @@ -152,6 +157,7 @@ a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported Permission to duplicate another process's file descriptor is governed by a ptrace access mode `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: + ```sh $ sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" ``` @@ -183,6 +189,7 @@ a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported Permission to duplicate another process's file descriptor is governed by a ptrace access mode `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: + ```sh $ sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" ``` @@ -203,7 +210,7 @@ Additionally, required for tests: #### DevDax memory provider (Linux only) -A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y). +A memory provider that provides memory from a device DAX (a character device file like `/dev/daxX.Y`). It can be used when large memory mappings are needed. ##### Requirements @@ -249,8 +256,6 @@ This memory pool is distributed as part of libumf. It forwards all requests to t memory provider. Currently umfPoolRealloc, umfPoolCalloc and umfPoolMallocUsableSize functions are not supported by the proxy pool. -To enable this feature, the `UMF_BUILD_SHARED_LIBRARY` option needs to be turned `ON`. - #### Disjoint pool TODO: Add a description @@ -326,6 +331,8 @@ Querying the latency value requires HMAT support on the platform. Calling `umfMe UMF provides the UMF proxy library (`umf_proxy`) that makes it possible to override the default allocator in other programs in both Linux and Windows. +To enable this feature, the `UMF_BUILD_SHARED_LIBRARY` option needs to be turned `ON`. + #### Linux In case of Linux it can be done without any code changes using the `LD_PRELOAD` environment variable: diff --git a/scripts/docs_config/api.rst b/scripts/docs_config/api.rst index c0448f1178..3eedc8f1d9 100644 --- a/scripts/docs_config/api.rst +++ b/scripts/docs_config/api.rst @@ -58,6 +58,9 @@ supported by the Proxy Pool. Scalable Pool ------------------------------------------ + +A oneTBB-based memory pool manager. + .. doxygenfile:: pool_scalable.h :sections: define enum typedef func var @@ -104,10 +107,18 @@ A memory provider that provides memory from L0 device. .. doxygenfile:: provider_level_zero.h :sections: define enum typedef func var +CUDA Provider +------------------------------------------ + +A memory provider that provides memory from CUDA device. + +.. doxygenfile:: provider_cuda.h + :sections: define enum typedef func var + DevDax Memory Provider ------------------------------------------ -A memory provider that provides memory from a device DAX (a character device file /dev/daxX.Y). +A memory provider that provides memory from a device DAX (a character device file like /dev/daxX.Y). .. doxygenfile:: provider_devdax_memory.h :sections: define enum typedef func var From 718c61dbf5ac76fd72166b9409d1a97644d4b34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 16 Dec 2024 15:51:24 +0100 Subject: [PATCH 58/82] Move docs content into a separate dir and update the script to work outside of scripts dir. --- .github/workflows/docs.yml | 8 +++--- .github/workflows/reusable_docs_build.yml | 6 +++-- .gitignore | 2 +- RELEASE_STEPS.md | 2 +- docs/README.md | 8 ++++++ .../assets/images/intro_architecture.png | Bin {scripts/docs_config => docs/config}/Doxyfile | 2 +- {scripts/docs_config => docs/config}/api.rst | 0 {scripts/docs_config => docs/config}/conf.py | 4 ++- .../docs_config => docs/config}/examples.rst | 0 .../docs_config => docs/config}/glossary.rst | 0 .../docs_config => docs/config}/index.rst | 0 .../config}/introduction.rst | 0 {scripts => docs}/generate_docs.py | 24 +++++++++++++----- scripts/README.md | 5 ---- 15 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 docs/README.md rename {scripts => docs}/assets/images/intro_architecture.png (100%) rename {scripts/docs_config => docs/config}/Doxyfile (99%) rename {scripts/docs_config => docs/config}/api.rst (100%) rename {scripts/docs_config => docs/config}/conf.py (92%) rename {scripts/docs_config => docs/config}/examples.rst (100%) rename {scripts/docs_config => docs/config}/glossary.rst (100%) rename {scripts/docs_config => docs/config}/index.rst (100%) rename {scripts/docs_config => docs/config}/introduction.rst (100%) rename {scripts => docs}/generate_docs.py (71%) delete mode 100644 scripts/README.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3d9bfc29b4..87e34cc749 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,13 +41,15 @@ jobs: run: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py + run: | + mkdir build + cd build + python3 ../docs/generate_docs.py - name: Upload artifact uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 with: - path: docs/html + path: build/docs_build/generated/html deploy: name: Deploy docs to GitHub Pages diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml index 269560c674..6702f9a66d 100644 --- a/.github/workflows/reusable_docs_build.yml +++ b/.github/workflows/reusable_docs_build.yml @@ -30,5 +30,7 @@ jobs: python3 -m pip install -r third_party/requirements.txt - name: Build the documentation - working-directory: scripts - run: python3 generate_docs.py + run: | + mkdir build + cd build + python3 ../docs/generate_docs.py diff --git a/.gitignore b/.gitignore index a1a488bc14..e177e395ef 100644 --- a/.gitignore +++ b/.gitignore @@ -58,7 +58,7 @@ __pycache__/ *.py[cod] # Generated docs -docs/ +docs_build/ # Build files /build*/ diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index fb46f156b9..efdadbe9fd 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -41,7 +41,7 @@ Do changes for a release: - For major releases mention API and ABI compatibility with the previous release - Update project's version in a few places: - For major and minor releases: `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - - `release` variable in `scripts/docs_config/conf.py` (for docs) + - `release` variable in `docs/config/conf.py` (for docs) - `UMF_VERSION` variable in `.github/workflows/reusable_basic.yml` (for installation test) - For major releases update ABI version in `.map` and `.def` files - These files are defined for all public libraries (`libumf` and `proxy_lib`, at the moment) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..1124b53bd9 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,8 @@ +To generate HTML documentation run the `generate_docs.py` script from the `build` dir. +It will create extra `./docs_build` directory, where the intermediate and final files +will be created. HTML docs will be in the `./docs_build/generated/html` directory. + +The script requires: + * [Doxygen](http://www.doxygen.nl/) at least v1.9.1 + * [Python](https://www.python.org/downloads/) at least v3.8 + * and python pip requirements, as defined in `third_party/requirements.txt` diff --git a/scripts/assets/images/intro_architecture.png b/docs/assets/images/intro_architecture.png similarity index 100% rename from scripts/assets/images/intro_architecture.png rename to docs/assets/images/intro_architecture.png diff --git a/scripts/docs_config/Doxyfile b/docs/config/Doxyfile similarity index 99% rename from scripts/docs_config/Doxyfile rename to docs/config/Doxyfile index 43ff2a6037..f23117ff29 100644 --- a/scripts/docs_config/Doxyfile +++ b/docs/config/Doxyfile @@ -2058,7 +2058,7 @@ GENERATE_XML = YES # The default directory is: xml. # This tag requires that the tag GENERATE_XML is set to YES. -XML_OUTPUT = ../docs/xml +XML_OUTPUT = docs_build/doxyxml # If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program # listings (including syntax highlighting and cross-referencing information) to diff --git a/scripts/docs_config/api.rst b/docs/config/api.rst similarity index 100% rename from scripts/docs_config/api.rst rename to docs/config/api.rst diff --git a/scripts/docs_config/conf.py b/docs/config/conf.py similarity index 92% rename from scripts/docs_config/conf.py rename to docs/config/conf.py index 577bc0b484..3af2df3787 100644 --- a/scripts/docs_config/conf.py +++ b/docs/config/conf.py @@ -49,7 +49,9 @@ # -- Extension configuration ------------------------------------------------- # -- Options for breathe extension ------------------------------------------- -breathe_projects = {project: "../../docs/xml"} +# 'doxyxml' dir is generated with Doxygen; it's supposed to be in a directory +# one above the config directory. +breathe_projects = {project: "../doxyxml"} breathe_default_project = project breathe_show_include = False breathe_default_members = ("members", "undoc-members") diff --git a/scripts/docs_config/examples.rst b/docs/config/examples.rst similarity index 100% rename from scripts/docs_config/examples.rst rename to docs/config/examples.rst diff --git a/scripts/docs_config/glossary.rst b/docs/config/glossary.rst similarity index 100% rename from scripts/docs_config/glossary.rst rename to docs/config/glossary.rst diff --git a/scripts/docs_config/index.rst b/docs/config/index.rst similarity index 100% rename from scripts/docs_config/index.rst rename to docs/config/index.rst diff --git a/scripts/docs_config/introduction.rst b/docs/config/introduction.rst similarity index 100% rename from scripts/docs_config/introduction.rst rename to docs/config/introduction.rst diff --git a/scripts/generate_docs.py b/docs/generate_docs.py similarity index 71% rename from scripts/generate_docs.py rename to docs/generate_docs.py index d5b2a01282..1697eacfe6 100644 --- a/scripts/generate_docs.py +++ b/docs/generate_docs.py @@ -6,17 +6,20 @@ """ from pathlib import Path -from shutil import rmtree +from shutil import rmtree, copytree import subprocess # nosec B404 import time def _check_cwd() -> None: - script_path = Path(__file__).resolve().parent cwd = Path.cwd() - if script_path != cwd: + include_dir = Path(cwd, "../include") + # Verify if include dir is one level up (as defined in Doxyfile) + if not include_dir.exists(): print( - f"{__file__} script has to be run from the 'scripts' directory. Terminating..." + f"Include directory {include_dir.resolve()} not found! " + "Please run this script from /build!", + flush=True, ) exit(1) @@ -66,8 +69,17 @@ def _generate_html(config_path: Path, docs_path: Path) -> None: def main() -> None: _check_cwd() - config_path = Path("docs_config").resolve() - docs_path = Path("..", "docs").resolve() + + script_dir = Path(__file__).resolve().parent + docs_build_path = Path("docs_build").resolve() + + # Sphinx and breathe require access to a Doxygen generated dir ('doxyxml') + # so we copy the whole content of the 'docs' dir to the build dir. + copytree(Path(script_dir), docs_build_path, dirs_exist_ok=True) + + config_path = Path(docs_build_path, "config").resolve() + docs_path = Path(docs_build_path, "generated").resolve() + start = time.time() _prepare_docs_dir(docs_path) _generate_xml(config_path, docs_path) diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index e3a9ed533f..0000000000 --- a/scripts/README.md +++ /dev/null @@ -1,5 +0,0 @@ -The documentation HTML files are generated using the following dependencies: - * [Python](https://www.python.org/downloads/) at least v3.8 - * [Doxygen](http://www.doxygen.nl/) at least v1.9.1 - - To generate files run the `generate_docs.py` script from the `scripts` directory. Files will be generated to the `docs/html` directory relative to the main directory of this repository. From 78d27981ba94512c27c6ac774f340eff71551a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 17 Dec 2024 10:57:31 +0100 Subject: [PATCH 59/82] [CMake] Add 'docs' target --- .github/workflows/docs.yml | 5 ++--- .github/workflows/reusable_docs_build.yml | 10 +++++++--- CMakeLists.txt | 11 +++++++++++ docs/README.md | 22 +++++++++++++++++++--- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 87e34cc749..c507f7994c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,9 +42,8 @@ jobs: - name: Build the documentation run: | - mkdir build - cd build - python3 ../docs/generate_docs.py + cmake -B build -DUMF_TESTS_FAIL_ON_SKIP=ON + cmake --build build --target docs - name: Upload artifact uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml index 6702f9a66d..92dcda5558 100644 --- a/.github/workflows/reusable_docs_build.yml +++ b/.github/workflows/reusable_docs_build.yml @@ -31,6 +31,10 @@ jobs: - name: Build the documentation run: | - mkdir build - cd build - python3 ../docs/generate_docs.py + cmake -B build \ + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF \ + -DUMF_BUILD_CUDA_PROVIDER=OFF \ + -DUMF_BUILD_TESTS=OFF \ + -DUMF_BUILD_EXAMPLES=OFF \ + -DUMF_DISABLE_HWLOC=ON + cmake --build build --target docs diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b88f95b5f..4e181f246c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -760,6 +760,17 @@ if(UMF_FORMAT_CODE_STYLE) endif() endif() +find_package(Python3 3.8) +if(Python3_FOUND) + message(STATUS "Adding 'docs' target for creating a documentation.") + add_custom_target( + docs + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND ${Python3_EXECUTABLE} + ${UMF_CMAKE_SOURCE_DIR}/docs/generate_docs.py + COMMENT "Generate HTML documentation using Doxygen") +endif() + # --------------------------------------------------------------------------- # # Configure make install/uninstall and packages # --------------------------------------------------------------------------- # diff --git a/docs/README.md b/docs/README.md index 1124b53bd9..3564d86db0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,24 @@ -To generate HTML documentation run the `generate_docs.py` script from the `build` dir. -It will create extra `./docs_build` directory, where the intermediate and final files +# Documentation + +To generate HTML documentation run the `generate_docs.py` script from any sub-dir of the +repository (most likely `build`) or enable and use build target 'docs' (see details below). + +This script will create `./docs_build` sub-directory, where the intermediate and final files will be created. HTML docs will be in the `./docs_build/generated/html` directory. -The script requires: +## make docs + +To run documentation generation via build target use CMake commands below. +To enable this target, python executable (in required version) has to be found in the system. + +```bash +$ cmake -B build +$ cmake --build build --target docs +``` + +## Requirements + +Script to generate HTML docs requires: * [Doxygen](http://www.doxygen.nl/) at least v1.9.1 * [Python](https://www.python.org/downloads/) at least v3.8 * and python pip requirements, as defined in `third_party/requirements.txt` From c7fdc11978e965af6dd4a2fce52320967b20e434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 17 Dec 2024 11:04:19 +0100 Subject: [PATCH 60/82] [CI] make docs workflow reusable in deploy job --- .github/workflows/docs.yml | 43 ++++------------------- .github/workflows/reusable_docs_build.yml | 14 +++++++- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c507f7994c..165cc1754f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,45 +14,14 @@ permissions: contents: read jobs: - build: - name: Build docs - runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-latest' }} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install doxygen - run: | - sudo apt-get update - sudo apt-get install -y doxygen - - # Latest distros do not allow global pip installation - - name: Install Python requirements in venv - run: | - python3 -m venv .venv - . .venv/bin/activate - echo "$PATH" >> $GITHUB_PATH - python3 -m pip install -r third_party/requirements.txt - - - name: Setup PATH for python - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Build the documentation - run: | - cmake -B build -DUMF_TESTS_FAIL_ON_SKIP=ON - cmake --build build --target docs - - - name: Upload artifact - uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 - with: - path: build/docs_build/generated/html + DocsBuild: + uses: ./.github/workflows/reusable_docs_build.yml + with: + upload: true - deploy: + DocsDeploy: name: Deploy docs to GitHub Pages - needs: build + needs: DocsBuild permissions: pages: write diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml index 92dcda5558..e90ca87aed 100644 --- a/.github/workflows/reusable_docs_build.yml +++ b/.github/workflows/reusable_docs_build.yml @@ -1,6 +1,12 @@ name: Docs build -on: workflow_call +on: + workflow_call: + inputs: + upload: + description: Should HTML documentation be uploaded as artifact? + type: boolean + default: false permissions: contents: read @@ -38,3 +44,9 @@ jobs: -DUMF_BUILD_EXAMPLES=OFF \ -DUMF_DISABLE_HWLOC=ON cmake --build build --target docs + + - name: Upload artifact + if: ${{ inputs.upload == true }} + uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0 + with: + path: build/docs_build/generated/html From 70c59068b314de0c6ac3459566bad138d3fdbd63 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 30 Dec 2024 09:27:22 +0100 Subject: [PATCH 61/82] Fix paths of logs in test_valgrind.sh Signed-off-by: Lukasz Dorau --- test/test_valgrind.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_valgrind.sh b/test/test_valgrind.sh index 46bfe7d1cc..954a3a56bb 100755 --- a/test/test_valgrind.sh +++ b/test/test_valgrind.sh @@ -74,9 +74,12 @@ echo "Working directory: $(pwd)" echo "Running: \"valgrind $OPTION\" for the following tests:" ANY_TEST_FAILED=0 -rm -f umf_test-*.log umf_test-*.err +PATH_TESTS="./test/umf_test-*" +PATH_EXAMPLES="./examples/umf_example_*" -[ "$TESTS" = "" ] && TESTS=$(ls -1 ./test/umf_test-* ./examples/umf_example_*) +rm -f ${PATH_TESTS}.log ${PATH_TESTS}.err ${PATH_EXAMPLES}.log ${PATH_EXAMPLES}.err + +[ "$TESTS" = "" ] && TESTS=$(ls -1 ${PATH_TESTS} ${PATH_EXAMPLES}) for test in $TESTS; do if [ ! -f $test ]; then @@ -185,7 +188,7 @@ echo echo "======================================================================" echo -for log in $(ls -1 umf_test-*.log); do +for log in $(ls -1 ${PATH_TESTS}.log ${PATH_EXAMPLES}.log); do echo ">>>>>>> LOG $log" cat $log echo From 5844c5afed09b731f79e8fab5c2257bbeb0f6a2e Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Thu, 26 Dec 2024 21:01:33 +0100 Subject: [PATCH 62/82] Fix L0 provider Set device_properties.stype during init. Found by L0 validation layer. --- src/provider/provider_level_zero.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 70f0acfe54..964d91e105 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -336,6 +336,10 @@ static umf_result_t ze_memory_provider_initialize(void *params, ze_provider->device = ze_params->level_zero_device_handle; ze_provider->memory_type = (ze_memory_type_t)ze_params->memory_type; + memset(&ze_provider->device_properties, 0, + sizeof(ze_provider->device_properties)); + ze_provider->device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + if (ze_provider->device) { umf_result_t ret = ze2umf_result(g_ze_ops.zeDeviceGetProperties( ze_provider->device, &ze_provider->device_properties)); @@ -345,9 +349,6 @@ static umf_result_t ze_memory_provider_initialize(void *params, umf_ba_global_free(ze_provider); return ret; } - } else { - memset(&ze_provider->device_properties, 0, - sizeof(ze_provider->device_properties)); } if (ze_params->resident_device_count) { From 58ba8e9ff09205dc1ccf4c998485e17ae894e7d5 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Tue, 31 Dec 2024 13:20:22 +0100 Subject: [PATCH 63/82] enable building examples on win static hwloc CI --- .github/workflows/reusable_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 3b573453d8..ae67aae657 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -333,7 +333,7 @@ jobs: -B ${{env.BUILD_DIR}} -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_BUILD_EXAMPLES=OFF + -DUMF_BUILD_EXAMPLES=ON -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON @@ -376,7 +376,7 @@ jobs: -B ${{env.BUILD_DIR}} -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=OFF - -DUMF_BUILD_EXAMPLES=OFF + -DUMF_BUILD_EXAMPLES=ON -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON From 91f14d7d8039dc07f0a0595f0f2f441e7d526930 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Tue, 31 Dec 2024 13:20:41 +0100 Subject: [PATCH 64/82] fix setting LIBHWLOC_LIBRARIES on Windows --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b88f95b5f..4d00bc2c2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,8 +251,8 @@ else() set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_SOURCE_DIR}/include;${hwloc_targ_BINARY_DIR}/include) - set(LIBHWLOC_LIBRARY_DIRS - ${hwloc_targ_BINARY_DIR}/Release;${hwloc_targ_BINARY_DIR}/Debug) + set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/$) + set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/$/hwloc.lib) else() include(FetchContent) message( From 0b4cbaa79f2909222efe43021b4d30558c5d8458 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Tue, 7 Jan 2025 13:42:28 +0100 Subject: [PATCH 65/82] use the UMF version from git describe in tests, not the harcoded one --- .github/workflows/reusable_basic.yml | 18 ++++++++++++++++-- RELEASE_STEPS.md | 1 - 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index ae67aae657..5866f939e7 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -7,8 +7,6 @@ permissions: contents: read env: - # for installation testing - it should match with version set in git - UMF_VERSION: 0.11.0 BUILD_DIR : "${{github.workspace}}/build" INSTL_DIR : "${{github.workspace}}/../install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" @@ -150,6 +148,11 @@ jobs: - name: Set ptrace value for IPC test run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" + - name: Get UMF version + run: | + VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') + echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV + - name: Configure build run: > ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} @@ -266,6 +269,12 @@ jobs: run: vcpkg install shell: pwsh # Specifies PowerShell as the shell for running the script. + - name: Get UMF version + run: | + $version = (git describe --tags --abbrev=0 | Select-String -Pattern '\d+\.\d+\.\d+').Matches.Value + echo "UMF_VERSION=$version" >> $env:GITHUB_ENV + shell: pwsh + - name: Configure build run: > cmake @@ -469,6 +478,11 @@ jobs: - name: Install hwloc run: brew install hwloc tbb automake + - name: Get UMF version + run: | + VERSION=$(git describe --tags --abbrev=0 | grep -Eo '\d+\.\d+\.\d+') + echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV + - name: Configure build run: > cmake diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index efdadbe9fd..2609e36bb0 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -42,7 +42,6 @@ Do changes for a release: - Update project's version in a few places: - For major and minor releases: `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - `release` variable in `docs/config/conf.py` (for docs) - - `UMF_VERSION` variable in `.github/workflows/reusable_basic.yml` (for installation test) - For major releases update ABI version in `.map` and `.def` files - These files are defined for all public libraries (`libumf` and `proxy_lib`, at the moment) - Commit these changes and tag the release: From da8cfb84990b8031e46dbd7bde96230348b126a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Wed, 6 Nov 2024 16:02:30 +0100 Subject: [PATCH 66/82] Revert "Disable temporarily failing CI job with ICX compiler" This reverts commit e8cde28437c067e0c73a381147260206d45bdeba. --- .github/workflows/reusable_basic.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 5866f939e7..ab4c8061b5 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -74,15 +74,15 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test icx compiler - # - os: 'ubuntu-22.04' - # build_type: Release - # compiler: {c: icx, cxx: icpx} - # shared_library: 'ON' - # level_zero_provider: 'ON' - # cuda_provider: 'ON' - # install_tbb: 'ON' - # disable_hwloc: 'OFF' - # link_hwloc_statically: 'OFF' + - os: 'ubuntu-22.04' + build_type: Release + compiler: {c: icx, cxx: icpx} + shared_library: 'ON' + level_zero_provider: 'ON' + cuda_provider: 'ON' + install_tbb: 'ON' + disable_hwloc: 'OFF' + link_hwloc_statically: 'OFF' # test without installing TBB - os: 'ubuntu-22.04' build_type: Release From 7a59bd3979b0a50d3c8b15dcea6d37f0d9d7ae54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Thu, 19 Dec 2024 14:24:32 +0100 Subject: [PATCH 67/82] refactor benchmark file structure --- benchmark/benchmark.cpp | 162 ++----------------- benchmark/benchmark.hpp | 171 ++++++++------------ benchmark/benchmark_interfaces.hpp | 144 ----------------- benchmark/benchmark_size.hpp | 63 ++++++++ benchmark/benchmark_umf.hpp | 252 +++++++++++++++++++++++++++++ 5 files changed, 401 insertions(+), 391 deletions(-) delete mode 100644 benchmark/benchmark_interfaces.hpp create mode 100644 benchmark/benchmark_size.hpp create mode 100644 benchmark/benchmark_umf.hpp diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 655545d1e2..df4fe6e5d5 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -6,161 +6,29 @@ * */ -#include -#include -#ifdef UMF_POOL_SCALABLE_ENABLED -#include -#endif -#include - -#ifdef UMF_POOL_DISJOINT_ENABLED -#include -#endif - -#ifdef UMF_POOL_JEMALLOC_ENABLED -#include -#endif - #include "benchmark.hpp" -struct glibc_malloc : public allocator_interface { - unsigned SetUp([[maybe_unused]] ::benchmark::State &state, - unsigned argPos) override { - return argPos; - } - void TearDown([[maybe_unused]] ::benchmark::State &state) override{}; - void *benchAlloc(size_t size) override { return malloc(size); } - void benchFree(void *ptr, [[maybe_unused]] size_t size) override { - free(ptr); +#define UMF_BENCHMARK_TEMPLATE_DEFINE(BaseClass, Method, ...) \ + BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, __VA_ARGS__) \ + (benchmark::State & state) { \ + for (auto _ : state) { \ + bench(state); \ + } \ } - static std::string name() { return "glibc"; } -}; - -struct os_provider : public provider_interface { - provider_interface::params_ptr - getParams(::benchmark::State &state) override { - umf_os_memory_provider_params_handle_t raw_params = nullptr; - umfOsMemoryProviderParamsCreate(&raw_params); - if (!raw_params) { - state.SkipWithError("Failed to create os provider params"); - return {nullptr, [](void *) {}}; - } - - // Use a lambda as the custom deleter - auto deleter = [](void *p) { - auto handle = - static_cast(p); - umfOsMemoryProviderParamsDestroy(handle); - }; - - return {static_cast(raw_params), deleter}; - } - - umf_memory_provider_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfOsMemoryProviderOps(); - } - static std::string name() { return "os_provider"; } -}; - -template -struct proxy_pool : public pool_interface { - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfProxyPoolOps(); - } - - static std::string name() { return "proxy_pool<" + Provider::name() + ">"; } -}; -#ifdef UMF_POOL_DISJOINT_ENABLED -template -struct disjoint_pool : public pool_interface { - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfDisjointPoolOps(); - } - - typename pool_interface::params_ptr - getParams(::benchmark::State &state) override { - umf_disjoint_pool_params_handle_t raw_params = nullptr; - auto ret = umfDisjointPoolParamsCreate(&raw_params); - if (ret != UMF_RESULT_SUCCESS) { - state.SkipWithError("Failed to create disjoint pool params"); - return {nullptr, [](void *) {}}; - } - - typename pool_interface::params_ptr params( - raw_params, [](void *p) { - umfDisjointPoolParamsDestroy( - static_cast(p)); - }); - - ret = umfDisjointPoolParamsSetSlabMinSize(raw_params, 4096); - if (ret != UMF_RESULT_SUCCESS) { - state.SkipWithError("Failed to set slab min size"); - return {nullptr, [](void *) {}}; - } - - ret = umfDisjointPoolParamsSetCapacity(raw_params, 4); - if (ret != UMF_RESULT_SUCCESS) { - state.SkipWithError("Failed to set capacity"); - return {nullptr, [](void *) {}}; - } - - ret = umfDisjointPoolParamsSetMinBucketSize(raw_params, 4096); - if (ret != UMF_RESULT_SUCCESS) { - state.SkipWithError("Failed to set min bucket size"); - return {nullptr, [](void *) {}}; - } - - ret = umfDisjointPoolParamsSetMaxPoolableSize(raw_params, 4096 * 16); - if (ret != UMF_RESULT_SUCCESS) { - state.SkipWithError("Failed to set max poolable size"); - return {nullptr, [](void *) {}}; - } - - return params; - } - - static std::string name() { - return "disjoint_pool<" + Provider::name() + ">"; - } -}; -#endif - -#ifdef UMF_POOL_JEMALLOC_ENABLED -template -struct jemalloc_pool : public pool_interface { - umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfJemallocPoolOps(); - } - - static std::string name() { - return "jemalloc_pool<" + Provider::name() + ">"; - } -}; -#endif - -#ifdef UMF_POOL_SCALABLE_ENABLED -template -struct scalable_pool : public pool_interface { - virtual umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) override { - return umfScalablePoolOps(); - } - - static std::string name() { - return "scalable_pool<" + Provider::name() + ">"; - } -}; -#endif -// Benchmarks scenarios: +#define UMF_BENCHMARK_REGISTER_F(BaseClass, Method) \ + BENCHMARK_REGISTER_F(BaseClass, Method) \ + ->ArgNames( \ + BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::argsName()) \ + ->Name(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::name()) \ + ->Iterations( \ + BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::iterations()) UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_fix, fixed_alloc_size, glibc_malloc); +// Benchmarks scenarios: + // The benchmark arguments specified in Args() are, in order: // benchmark arguments, allocator arguments, size generator arguments. // The exact meaning of each argument depends on the benchmark, allocator, and size components used. diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 6ac7a4dfa5..df5d6a5927 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -75,70 +75,97 @@ #include #include -#include "benchmark_interfaces.hpp" +#include "benchmark_size.hpp" +#include "benchmark_umf.hpp" struct alloc_data { void *ptr; size_t size; }; -#define UMF_BENCHMARK_TEMPLATE_DEFINE(BaseClass, Method, ...) \ - BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, __VA_ARGS__) \ - (benchmark::State & state) { \ - for (auto _ : state) { \ - bench(state); \ - } \ +template ::value>> +class provider_allocator : public allocator_interface { + public: + unsigned SetUp(::benchmark::State &state, unsigned r) override { + provider.SetUp(state); + return r; } -#define UMF_BENCHMARK_REGISTER_F(BaseClass, Method) \ - BENCHMARK_REGISTER_F(BaseClass, Method) \ - ->ArgNames( \ - BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::argsName()) \ - ->Name(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::name()) \ - ->Iterations( \ - BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::iterations()) + void TearDown(::benchmark::State &state) override { + provider.TearDown(state); + } -class fixed_alloc_size : public alloc_size_interface { - public: - unsigned SetUp(::benchmark::State &state, unsigned argPos) override { - size = state.range(argPos); - return argPos + 1; + void *benchAlloc(size_t size) override { + void *ptr; + if (umfMemoryProviderAlloc(provider.provider, size, 0, &ptr) != + UMF_RESULT_SUCCESS) { + return NULL; + } + return ptr; + } + + void benchFree(void *ptr, size_t size) override { + umfMemoryProviderFree(provider.provider, ptr, size); } - void TearDown([[maybe_unused]] ::benchmark::State &state) override {} - size_t nextSize() override { return size; }; - static std::vector argsName() { return {"size"}; } + + static std::string name() { return Provider::name(); } private: - size_t size; + Provider provider; }; -class uniform_alloc_size : public alloc_size_interface { - using distribution = std::uniform_int_distribution; - +// TODO: assert Pool to be a pool_interface. +template class pool_allocator : public allocator_interface { public: - unsigned SetUp(::benchmark::State &state, unsigned argPos) override { - auto min = state.range(argPos++); - auto max = state.range(argPos++); - auto gran = state.range(argPos++); - if (min % gran != 0 && max % gran != 0) { - state.SkipWithError("min and max must be divisible by granularity"); - return argPos; - } + unsigned SetUp(::benchmark::State &state, unsigned r) override { + pool.SetUp(state); + return r; + } + + void TearDown(::benchmark::State &state) override { pool.TearDown(state); } - dist.param(distribution::param_type(min / gran, max / gran)); - multiplier = gran; - return argPos; + virtual void *benchAlloc(size_t size) override { + return umfPoolMalloc(pool.pool, size); } - void TearDown([[maybe_unused]] ::benchmark::State &state) override {} - size_t nextSize() override { return dist(generator) * multiplier; } - static std::vector argsName() { - return {"min size", "max size", "granularity"}; + + virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) override { + umfPoolFree(pool.pool, ptr); } + static std::string name() { return Pool::name(); } + private: - std::default_random_engine generator; - distribution dist; - size_t multiplier; + Pool pool; +}; + +template +struct benchmark_interface : public benchmark::Fixture { + void SetUp(::benchmark::State &state) { + int argPos = alloc_size.SetUp(state, 0); + allocator.SetUp(state, argPos); + } + + void TearDown(::benchmark::State &state) { + alloc_size.TearDown(state); + allocator.TearDown(state); + } + + virtual void bench(::benchmark::State &state) = 0; + + static std::vector argsName() { + auto s = Size::argsName(); + auto a = Allocator::argsName(); + std::vector res = {}; + res.insert(res.end(), s.begin(), s.end()); + res.insert(res.end(), a.begin(), a.end()); + return res; + } + + static std::string name() { return Allocator::name(); } + static int64_t iterations() { return 10000; } + Size alloc_size; + Allocator allocator; }; // This class benchmarks speed of alloc() operations. @@ -335,59 +362,3 @@ class multiple_malloc_free_benchmark : public alloc_benchmark { std::default_random_engine generator; distribution dist; }; - -template ::value>> -class provider_allocator : public allocator_interface { - public: - unsigned SetUp(::benchmark::State &state, unsigned r) override { - provider.SetUp(state); - return r; - } - - void TearDown(::benchmark::State &state) override { - provider.TearDown(state); - } - - void *benchAlloc(size_t size) override { - void *ptr; - if (umfMemoryProviderAlloc(provider.provider, size, 0, &ptr) != - UMF_RESULT_SUCCESS) { - return NULL; - } - return ptr; - } - - void benchFree(void *ptr, size_t size) override { - umfMemoryProviderFree(provider.provider, ptr, size); - } - - static std::string name() { return Provider::name(); } - - private: - Provider provider; -}; - -// TODO: assert Pool to be a pool_interface. -template class pool_allocator : public allocator_interface { - public: - unsigned SetUp(::benchmark::State &state, unsigned r) override { - pool.SetUp(state); - return r; - } - - void TearDown(::benchmark::State &state) override { pool.TearDown(state); } - - virtual void *benchAlloc(size_t size) override { - return umfPoolMalloc(pool.pool, size); - } - - virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) override { - umfPoolFree(pool.pool, ptr); - } - - static std::string name() { return Pool::name(); } - - private: - Pool pool; -}; diff --git a/benchmark/benchmark_interfaces.hpp b/benchmark/benchmark_interfaces.hpp deleted file mode 100644 index 516a20b697..0000000000 --- a/benchmark/benchmark_interfaces.hpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2024 Intel Corporation - * - * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include -#include -#include -#include - -#include -#include -#include - -class alloc_size_interface { - public: - virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, - [[maybe_unused]] unsigned argPos) = 0; - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; - virtual size_t nextSize() = 0; - static std::vector argsName() { return {""}; }; -}; - -class allocator_interface { - public: - virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, - [[maybe_unused]] unsigned argPos) = 0; - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; - virtual void *benchAlloc(size_t size) = 0; - virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) = 0; - static std::vector argsName() { return {}; } -}; - -template -struct benchmark_interface : public benchmark::Fixture { - void SetUp(::benchmark::State &state) { - int argPos = alloc_size.SetUp(state, 0); - allocator.SetUp(state, argPos); - } - - void TearDown(::benchmark::State &state) { - alloc_size.TearDown(state); - allocator.TearDown(state); - } - - virtual void bench(::benchmark::State &state) = 0; - - static std::vector argsName() { - auto s = Size::argsName(); - auto a = Allocator::argsName(); - std::vector res = {}; - res.insert(res.end(), s.begin(), s.end()); - res.insert(res.end(), a.begin(), a.end()); - return res; - } - - static std::string name() { return Allocator::name(); } - static int64_t iterations() { return 10000; } - Size alloc_size; - Allocator allocator; -}; - -struct provider_interface { - using params_ptr = std::unique_ptr; - - umf_memory_provider_handle_t provider = NULL; - virtual void SetUp(::benchmark::State &state) { - if (state.thread_index() != 0) { - return; - } - auto params = getParams(state); - auto umf_result = - umfMemoryProviderCreate(getOps(state), params.get(), &provider); - if (umf_result != UMF_RESULT_SUCCESS) { - state.SkipWithError("umfMemoryProviderCreate() failed"); - } - } - - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { - if (state.thread_index() != 0) { - return; - } - - if (provider) { - umfMemoryProviderDestroy(provider); - } - } - - virtual umf_memory_provider_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) { - return nullptr; - } - - virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { - return {nullptr, [](void *) {}}; - } -}; - -template ::value>> -struct pool_interface { - using params_ptr = std::unique_ptr; - - virtual void SetUp(::benchmark::State &state) { - provider.SetUp(state); - if (state.thread_index() != 0) { - return; - } - auto params = getParams(state); - auto umf_result = umfPoolCreate(getOps(state), provider.provider, - params.get(), 0, &pool); - if (umf_result != UMF_RESULT_SUCCESS) { - state.SkipWithError("umfPoolCreate() failed"); - } - } - virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { - if (state.thread_index() != 0) { - return; - } - // TODO: The scalable pool destruction process can race with other threads - // performing TLS (Thread-Local Storage) destruction. - // As a temporary workaround, we introduce a delay (sleep) - // to ensure the pool is destroyed only after all threads have completed. - // Issue: #933 - std::this_thread::sleep_for(std::chrono::milliseconds(500)); - if (pool) { - umfPoolDestroy(pool); - } - }; - - virtual umf_memory_pool_ops_t * - getOps([[maybe_unused]] ::benchmark::State &state) { - return nullptr; - } - virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { - return {nullptr, [](void *) {}}; - } - T provider; - umf_memory_pool_handle_t pool; -}; diff --git a/benchmark/benchmark_size.hpp b/benchmark/benchmark_size.hpp new file mode 100644 index 0000000000..d17a6b2869 --- /dev/null +++ b/benchmark/benchmark_size.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include +#include +#include + +class alloc_size_interface { + public: + virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, + [[maybe_unused]] unsigned argPos) = 0; + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; + virtual size_t nextSize() = 0; + static std::vector argsName() { return {""}; }; +}; + +class fixed_alloc_size : public alloc_size_interface { + public: + unsigned SetUp(::benchmark::State &state, unsigned argPos) override { + size = state.range(argPos); + return argPos + 1; + } + void TearDown([[maybe_unused]] ::benchmark::State &state) override {} + size_t nextSize() override { return size; }; + static std::vector argsName() { return {"size"}; } + + private: + size_t size; +}; + +class uniform_alloc_size : public alloc_size_interface { + using distribution = std::uniform_int_distribution; + + public: + unsigned SetUp(::benchmark::State &state, unsigned argPos) override { + auto min = state.range(argPos++); + auto max = state.range(argPos++); + auto gran = state.range(argPos++); + if (min % gran != 0 && max % gran != 0) { + state.SkipWithError("min and max must be divisible by granularity"); + return argPos; + } + + dist.param(distribution::param_type(min / gran, max / gran)); + multiplier = gran; + return argPos; + } + void TearDown([[maybe_unused]] ::benchmark::State &state) override {} + size_t nextSize() override { return dist(generator) * multiplier; } + static std::vector argsName() { + return {"min size", "max size", "granularity"}; + } + + private: + std::default_random_engine generator; + distribution dist; + size_t multiplier; +}; diff --git a/benchmark/benchmark_umf.hpp b/benchmark/benchmark_umf.hpp new file mode 100644 index 0000000000..389c224ed1 --- /dev/null +++ b/benchmark/benchmark_umf.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2024-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ +#include +#include + +#include +#include +#include + +#include +#include +#ifdef UMF_POOL_SCALABLE_ENABLED +#include +#endif +#include + +#ifdef UMF_POOL_DISJOINT_ENABLED +#include +#endif + +#ifdef UMF_POOL_JEMALLOC_ENABLED +#include +#endif + +struct provider_interface { + using params_ptr = std::unique_ptr; + + umf_memory_provider_handle_t provider = NULL; + virtual void SetUp(::benchmark::State &state) { + if (state.thread_index() != 0) { + return; + } + auto params = getParams(state); + auto umf_result = + umfMemoryProviderCreate(getOps(state), params.get(), &provider); + if (umf_result != UMF_RESULT_SUCCESS) { + state.SkipWithError("umfMemoryProviderCreate() failed"); + } + } + + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { + if (state.thread_index() != 0) { + return; + } + + if (provider) { + umfMemoryProviderDestroy(provider); + } + } + + virtual umf_memory_provider_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) { + return nullptr; + } + + virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { + return {nullptr, [](void *) {}}; + } +}; + +template ::value>> +struct pool_interface { + using params_ptr = std::unique_ptr; + + virtual void SetUp(::benchmark::State &state) { + provider.SetUp(state); + if (state.thread_index() != 0) { + return; + } + auto params = getParams(state); + auto umf_result = umfPoolCreate(getOps(state), provider.provider, + params.get(), 0, &pool); + if (umf_result != UMF_RESULT_SUCCESS) { + state.SkipWithError("umfPoolCreate() failed"); + } + } + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) { + if (state.thread_index() != 0) { + return; + } + // TODO: The scalable pool destruction process can race with other threads + // performing TLS (Thread-Local Storage) destruction. + // As a temporary workaround, we introduce a delay (sleep) + // to ensure the pool is destroyed only after all threads have completed. + // Issue: #933 + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + if (pool) { + umfPoolDestroy(pool); + } + }; + + virtual umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) { + return nullptr; + } + virtual params_ptr getParams([[maybe_unused]] ::benchmark::State &state) { + return {nullptr, [](void *) {}}; + } + T provider; + umf_memory_pool_handle_t pool; +}; + +class allocator_interface { + public: + virtual unsigned SetUp([[maybe_unused]] ::benchmark::State &state, + [[maybe_unused]] unsigned argPos) = 0; + virtual void TearDown([[maybe_unused]] ::benchmark::State &state) = 0; + virtual void *benchAlloc(size_t size) = 0; + virtual void benchFree(void *ptr, [[maybe_unused]] size_t size) = 0; + static std::vector argsName() { return {}; } +}; + +struct glibc_malloc : public allocator_interface { + unsigned SetUp([[maybe_unused]] ::benchmark::State &state, + unsigned argPos) override { + return argPos; + } + void TearDown([[maybe_unused]] ::benchmark::State &state) override{}; + void *benchAlloc(size_t size) override { return malloc(size); } + void benchFree(void *ptr, [[maybe_unused]] size_t size) override { + free(ptr); + } + static std::string name() { return "glibc"; } +}; + +struct os_provider : public provider_interface { + provider_interface::params_ptr + getParams(::benchmark::State &state) override { + umf_os_memory_provider_params_handle_t raw_params = nullptr; + umfOsMemoryProviderParamsCreate(&raw_params); + if (!raw_params) { + state.SkipWithError("Failed to create os provider params"); + return {nullptr, [](void *) {}}; + } + + // Use a lambda as the custom deleter + auto deleter = [](void *p) { + auto handle = + static_cast(p); + umfOsMemoryProviderParamsDestroy(handle); + }; + + return {static_cast(raw_params), deleter}; + } + + umf_memory_provider_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfOsMemoryProviderOps(); + } + static std::string name() { return "os_provider"; } +}; + +template +struct proxy_pool : public pool_interface { + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfProxyPoolOps(); + } + + static std::string name() { return "proxy_pool<" + Provider::name() + ">"; } +}; + +#ifdef UMF_POOL_DISJOINT_ENABLED +template +struct disjoint_pool : public pool_interface { + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfDisjointPoolOps(); + } + + typename pool_interface::params_ptr + getParams(::benchmark::State &state) override { + umf_disjoint_pool_params_handle_t raw_params = nullptr; + auto ret = umfDisjointPoolParamsCreate(&raw_params); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to create disjoint pool params"); + return {nullptr, [](void *) {}}; + } + + typename pool_interface::params_ptr params( + raw_params, [](void *p) { + umfDisjointPoolParamsDestroy( + static_cast(p)); + }); + + ret = umfDisjointPoolParamsSetSlabMinSize(raw_params, 4096); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set slab min size"); + return {nullptr, [](void *) {}}; + } + + ret = umfDisjointPoolParamsSetCapacity(raw_params, 4); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set capacity"); + return {nullptr, [](void *) {}}; + } + + ret = umfDisjointPoolParamsSetMinBucketSize(raw_params, 4096); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set min bucket size"); + return {nullptr, [](void *) {}}; + } + + ret = umfDisjointPoolParamsSetMaxPoolableSize(raw_params, 4096 * 16); + if (ret != UMF_RESULT_SUCCESS) { + state.SkipWithError("Failed to set max poolable size"); + return {nullptr, [](void *) {}}; + } + + return params; + } + + static std::string name() { + return "disjoint_pool<" + Provider::name() + ">"; + } +}; +#endif + +#ifdef UMF_POOL_JEMALLOC_ENABLED +template +struct jemalloc_pool : public pool_interface { + umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfJemallocPoolOps(); + } + + static std::string name() { + return "jemalloc_pool<" + Provider::name() + ">"; + } +}; +#endif + +#ifdef UMF_POOL_SCALABLE_ENABLED +template +struct scalable_pool : public pool_interface { + virtual umf_memory_pool_ops_t * + getOps([[maybe_unused]] ::benchmark::State &state) override { + return umfScalablePoolOps(); + } + + static std::string name() { + return "scalable_pool<" + Provider::name() + ">"; + } +}; +#endif From 4d9e2a5bb6b5cb9e135d13095c7448b0a3d8cdb6 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 8 Jan 2025 09:45:42 +0100 Subject: [PATCH 68/82] use the UMF version from git describe in docs, not the harcoded one --- CMakeLists.txt | 2 +- RELEASE_STEPS.md | 4 +--- docs/README.md | 22 +++++++++++++++------- docs/config/conf.py | 12 +++++++++--- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f9cfc3a073..495c70de30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -766,7 +766,7 @@ if(Python3_FOUND) add_custom_target( docs WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMAND ${Python3_EXECUTABLE} + COMMAND UMF_VERSION=${UMF_CMAKE_VERSION} ${Python3_EXECUTABLE} ${UMF_CMAKE_SOURCE_DIR}/docs/generate_docs.py COMMENT "Generate HTML documentation using Doxygen") endif() diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index 2609e36bb0..9e04dc8502 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -39,9 +39,7 @@ Do changes for a release: - For major/minor release start from the `main` branch - Add an entry to ChangeLog, remember to change the day of the week in the release date - For major releases mention API and ABI compatibility with the previous release -- Update project's version in a few places: - - For major and minor releases: `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - - `release` variable in `docs/config/conf.py` (for docs) +- For major and minor releases, update `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - For major releases update ABI version in `.map` and `.def` files - These files are defined for all public libraries (`libumf` and `proxy_lib`, at the moment) - Commit these changes and tag the release: diff --git a/docs/README.md b/docs/README.md index 3564d86db0..737bb12595 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,14 @@ # Documentation -To generate HTML documentation run the `generate_docs.py` script from any sub-dir of the -repository (most likely `build`) or enable and use build target 'docs' (see details below). +To generate HTML documentation, run the `generate_docs.py` script from any sub-directory of the repository (most likely `build`). +To display the proper version of UMF in the documentation title, set the `UMF_VERSION` variable before running the script. + +```bash +cd build +$ UMF_VERSION= python ../docs/generate_docs.py +``` + +Documentation can also be built using the build target 'docs' (see details below). This script will create `./docs_build` sub-directory, where the intermediate and final files will be created. HTML docs will be in the `./docs_build/generated/html` directory. @@ -12,13 +19,14 @@ To run documentation generation via build target use CMake commands below. To enable this target, python executable (in required version) has to be found in the system. ```bash -$ cmake -B build -$ cmake --build build --target docs +cmake -B build +cmake --build build --target docs ``` ## Requirements Script to generate HTML docs requires: - * [Doxygen](http://www.doxygen.nl/) at least v1.9.1 - * [Python](https://www.python.org/downloads/) at least v3.8 - * and python pip requirements, as defined in `third_party/requirements.txt` + +* [Doxygen](http://www.doxygen.nl/) at least v1.9.1 +* [Python](https://www.python.org/downloads/) at least v3.8 +* and python pip requirements, as defined in `third_party/requirements.txt` diff --git a/docs/config/conf.py b/docs/config/conf.py index 3af2df3787..fa4788ff42 100644 --- a/docs/config/conf.py +++ b/docs/config/conf.py @@ -1,3 +1,5 @@ +import os + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -18,12 +20,16 @@ # -- Project information ----------------------------------------------------- project = "Intel Unified Memory Framework" -copyright = "2023-2024, Intel" +copyright = "2023-2025, Intel" author = "Intel" # The full version, including alpha/beta/rc tags -release = "0.11.0" - +release = os.getenv("UMF_VERSION", "") +print( + f"UMF_VERSION used in docs: {release}" + if release != "" + else "please set UMF_VERSION environment variable before running this script" +) # -- General configuration --------------------------------------------------- From e94647d1bf0f6cc4c1388cd9c41b187cf2ef028d Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Tue, 7 Jan 2025 12:32:33 +0100 Subject: [PATCH 69/82] Change linking to static for ICX Intel libraries --- .github/workflows/reusable_basic.yml | 3 +-- .github/workflows/reusable_sanitizers.yml | 1 - CMakeLists.txt | 6 ++++++ test/CMakeLists.txt | 8 +++++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index ab4c8061b5..83542efbbe 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -182,8 +182,7 @@ jobs: - name: Run tests working-directory: ${{env.BUILD_DIR}} run: | - ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} - ctest --output-on-failure # run all tests for better coverage + LD_LIBRARY_PATH=${{env.BUILD_DIR}}/lib/ ctest --output-on-failure # run all tests for better coverage - name: Check coverage if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} diff --git a/.github/workflows/reusable_sanitizers.yml b/.github/workflows/reusable_sanitizers.yml index f9e121f889..93752ff846 100644 --- a/.github/workflows/reusable_sanitizers.yml +++ b/.github/workflows/reusable_sanitizers.yml @@ -77,7 +77,6 @@ jobs: ASAN_OPTIONS: allocator_may_return_null=1 TSAN_OPTIONS: allocator_may_return_null=1 run: | - ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} ctest --output-on-failure windows-build: diff --git a/CMakeLists.txt b/CMakeLists.txt index f9cfc3a073..6c4f4e4c6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,12 @@ include(CMakePackageConfigHelpers) include(GNUInstallDirs) find_package(PkgConfig) +if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + # Compiler dependencies needs to be in library path or to be linked + # statically + add_link_options(-static-intel) +endif() + # Build Options option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UMF_BUILD_LEVEL_ZERO_PROVIDER "Build Level Zero memory provider" ON) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 04ebfe1098..7eed07e09e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,10 +1,16 @@ -# Copyright (C) 2022-2024 Intel Corporation +# Copyright (C) 2022-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED YES) +if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + # Compiler dependencies needs to be in library path or to be linked + # statically + add_link_options(-static-intel) +endif() + include(FetchContent) FetchContent_Declare( googletest From e34952096ae5c6ea3e5368ba7f9625f740e8d52c Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Thu, 9 Jan 2025 15:22:12 +0100 Subject: [PATCH 70/82] Remove all Intel libs from main library --- CMakeLists.txt | 6 ------ cmake/helpers.cmake | 5 ++++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c4f4e4c6a..f9cfc3a073 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,12 +33,6 @@ include(CMakePackageConfigHelpers) include(GNUInstallDirs) find_package(PkgConfig) -if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") - # Compiler dependencies needs to be in library path or to be linked - # statically - add_link_options(-static-intel) -endif() - # Build Options option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UMF_BUILD_LEVEL_ZERO_PROVIDER "Build Level Zero memory provider" ON) diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 56692ff6ec..2d14e2f45f 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -378,6 +378,9 @@ function(add_umf_library) elseif(LINUX) target_link_options(${ARG_NAME} PRIVATE "-Wl,--version-script=${ARG_LINUX_MAP_FILE}") + if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + target_link_options(${ARG_NAME} PRIVATE -no-intel-lib) + endif() endif() endif() From ed94adb2dfdf4477f4ad036b3cbe3d1dac20bf00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2025 21:51:07 +0000 Subject: [PATCH 71/82] Bump pygments in /third_party in the pip-dependencies group Bumps the pip-dependencies group in /third_party with 1 update: [pygments](https://github.com/pygments/pygments). Updates `pygments` from 2.18.0 to 2.19.1 - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.18.0...2.19.1) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production update-type: version-update:semver-minor dependency-group: pip-dependencies ... Signed-off-by: dependabot[bot] --- third_party/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/requirements.txt b/third_party/requirements.txt index 6a8be6e46a..9832bf2f0f 100644 --- a/third_party/requirements.txt +++ b/third_party/requirements.txt @@ -6,7 +6,7 @@ black==24.3.0 # Tests packaging==24.2 # Generating HTML documentation -pygments==2.18.0 +pygments==2.19.1 sphinxcontrib_applehelp==2.0.0 sphinxcontrib_devhelp==2.0.0 sphinxcontrib_htmlhelp==2.1.0 From 5db7da2bedb54581e8293c878db70b58f7d3e9d8 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Wed, 8 Jan 2025 15:32:49 +0100 Subject: [PATCH 72/82] Added license checker based on https://github.com/pmem/pmemstream/tree/master/utils/check_license Changes for this repository: - update dates - file exceptions --- .github/workflows/reusable_checks.yml | 8 + include/umf.h | 2 +- include/umf/memory_pool_ops.h | 2 +- include/umf/memtarget.h | 2 +- include/umf/pools/pool_jemalloc.h | 2 +- include/umf/pools/pool_scalable.h | 2 +- scripts/check_license/check_headers.sh | 177 +++++++++++++++++++++++ scripts/check_license/file-exceptions.sh | 34 +++++ src/ctl/ctl.c | 4 +- src/libumf.def | 2 +- src/libumf.map | 2 +- src/libumf.rc.in | 4 +- src/memory_pool_internal.h | 2 +- src/memory_provider_get_last_failed.c | 2 +- src/memory_provider_internal.h | 2 +- src/memspaces/memspace_numa.c | 2 +- src/memtargets/memtarget_numa.h | 2 +- src/pool/pool_disjoint.cpp | 2 +- src/proxy_lib/proxy_lib.rc.in | 4 +- src/utils/utils_common.h | 2 +- src/utils/utils_posix_concurrency.c | 2 +- src/utils/utils_windows_concurrency.c | 2 +- src/utils/utils_windows_math.c | 2 +- test/common/pool_trace.c | 2 +- test/common/test_helpers.c | 2 +- test/malloc_compliance_tests.cpp | 2 +- test/poolFixtures.hpp | 2 +- test/pools/jemalloc_pool.cpp | 2 +- test/pools/pool_base_alloc.cpp | 2 +- test/pools/scalable_pool.cpp | 2 +- test/provider_os_memory.cpp | 2 +- 31 files changed, 250 insertions(+), 31 deletions(-) create mode 100755 scripts/check_license/check_headers.sh create mode 100755 scripts/check_license/file-exceptions.sh diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index 6298b98831..de28161a58 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -52,6 +52,14 @@ jobs: - name: Check Python formatting run: cmake --build build --target black-format-check + - name: Run check-license + run: | + ./scripts/check_license/check_headers.sh . "Apache-2.0 WITH LLVM-exception" -v + + - name: Run copyright-format + run: | + ./scripts/check_license/check_headers.sh . "Apache-2.0 WITH LLVM-exception" -d + - name: Run a spell check uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2 with: diff --git a/include/umf.h b/include/umf.h index 3e2d827991..57bebef8a9 100644 --- a/include/umf.h +++ b/include/umf.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index 67afdd1669..829f49fb76 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/memtarget.h b/include/umf/memtarget.h index d74947f14e..55ca30919a 100644 --- a/include/umf/memtarget.h +++ b/include/umf/memtarget.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/pools/pool_jemalloc.h b/include/umf/pools/pool_jemalloc.h index c30df65092..5974e6440a 100644 --- a/include/umf/pools/pool_jemalloc.h +++ b/include/umf/pools/pool_jemalloc.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/include/umf/pools/pool_scalable.h b/include/umf/pools/pool_scalable.h index 072169b68c..1915ad0b7a 100644 --- a/include/umf/pools/pool_scalable.h +++ b/include/umf/pools/pool_scalable.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/scripts/check_license/check_headers.sh b/scripts/check_license/check_headers.sh new file mode 100755 index 0000000000..d68b0891bc --- /dev/null +++ b/scripts/check_license/check_headers.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + # Copyright (C) 2016-2024 Intel Corporation + # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# check-headers.sh - check copyright and license in source files + +SELF=$0 + +function usage() { + echo "Usage: $SELF [-h|-v|-a|-d]" + echo " -h, --help this help message" + echo " -v, --verbose verbose mode" + echo " -a, --all check all files (only modified files are checked by default)" + echo " -d, --update_dates change Copyright dates in all analyzed files (rather not use with -a)" +} + +if [ "$#" -lt 2 ]; then + usage >&2 + exit 2 +fi + +SOURCE_ROOT=$1 +shift +LICENSE=$1 +shift + +PATTERN=`mktemp` +TMP=`mktemp` +TMP2=`mktemp` +TEMPFILE=`mktemp` +rm -f $PATTERN $TMP $TMP2 + +if [ "$1" == "-h" -o "$1" == "--help" ]; then + usage + exit 0 +fi + +export GIT="git -C ${SOURCE_ROOT}" +$GIT rev-parse || exit 1 + +if [ -f $SOURCE_ROOT/.git/shallow ]; then + SHALLOW_CLONE=1 + echo + echo "Warning: This is a shallow clone. Checking dates in copyright headers" + echo " will be skipped in case of files that have no history." + echo +else + SHALLOW_CLONE=0 +fi + +VERBOSE=0 +CHECK_ALL=0 +UPDATE_DATES=0 +while [ "$1" != "" ]; do + case $1 in + -v|--verbose) + VERBOSE=1 + ;; + -a|--all) + CHECK_ALL=1 + ;; + -d|--update_dates) + UPDATE_DATES=1 + ;; + esac + shift +done + +if [ $CHECK_ALL -eq 0 ]; then + CURRENT_COMMIT=$($GIT log --pretty=%H -1) + MERGE_BASE=$($GIT merge-base HEAD origin/master 2>/dev/null) + [ -z $MERGE_BASE ] && \ + MERGE_BASE=$($GIT log --pretty="%cN:%H" | grep GitHub | head -n1 | cut -d: -f2) + [ -z $MERGE_BASE -o "$CURRENT_COMMIT" = "$MERGE_BASE" ] && \ + CHECK_ALL=1 +fi + +if [ $CHECK_ALL -eq 1 ]; then + echo "INFO: Checking copyright headers of all files..." + GIT_COMMAND="ls-tree -r --name-only HEAD" +else + echo "INFO: Checking copyright headers of modified files only..." + GIT_COMMAND="diff --name-only $MERGE_BASE $CURRENT_COMMIT" +fi + +FILES=$($GIT $GIT_COMMAND | ${SOURCE_ROOT}/scripts/check_license/file-exceptions.sh) + +RV=0 +for file in $FILES ; do + if [ $VERBOSE -eq 1 ]; then + echo "Checking file: $file" + fi + # The src_path is a path which should be used in every command except git. + # git is called with -C flag so filepaths should be relative to SOURCE_ROOT + src_path="${SOURCE_ROOT}/$file" + [ ! -f $src_path ] && continue + # ensure that file is UTF-8 encoded + ENCODING=`file -b --mime-encoding $src_path` + iconv -f $ENCODING -t "UTF-8" $src_path > $TEMPFILE + + if ! grep -q "SPDX-License-Identifier: $LICENSE" $src_path; then + echo >&2 "error: no $LICENSE SPDX tag in file: $src_path" + RV=1 + fi + + if [ $SHALLOW_CLONE -eq 0 ]; then + $GIT log --no-merges --format="%ai %aE" -- $file | sort > $TMP + else + # mark the grafted commits (commits with no parents) + $GIT log --no-merges --format="%ai %aE grafted-%p-commit" -- $file | sort > $TMP + fi + + # skip checking dates for non-Intel commits + [[ ! $(tail -n1 $TMP) =~ "@intel.com" ]] && continue + + # skip checking dates for new files + [ $(cat $TMP | wc -l) -le 1 ] && continue + + # grep out the grafted commits (commits with no parents) + # and skip checking dates for non-Intel commits + grep -v -e "grafted--commit" $TMP | grep -e "@intel.com" > $TMP2 + + [ $(cat $TMP2 | wc -l) -eq 0 ] && continue + + FIRST=`head -n1 $TMP2` + LAST=` tail -n1 $TMP2` + + YEARS=$(sed ' +/.*Copyright (C) \+.*[0-9-]\+ Intel Corporation/!d +s/.*Copyright (C) \([0-9]\+\)-\([0-9]\+\).*/\1-\2/ +s/.*Copyright (C) \([0-9]\+\).*/\1/' "$src_path") + if [ -z "$YEARS" ]; then + echo >&2 "No copyright years in $src_path" + RV=1 + continue + fi + + HEADER_FIRST=`echo $YEARS | cut -d"-" -f1` + HEADER_LAST=` echo $YEARS | cut -d"-" -f2` + + COMMIT_FIRST=`echo $FIRST | cut -d"-" -f1` + COMMIT_LAST=` echo $LAST | cut -d"-" -f1` + + if [ "$COMMIT_FIRST" != "" -a "$COMMIT_LAST" != "" ]; then + if [[ -n "$COMMIT_FIRST" && -n "$COMMIT_LAST" ]]; then + if [[ $COMMIT_FIRST -eq $COMMIT_LAST ]]; then + NEW=$COMMIT_LAST + else + NEW=$COMMIT_FIRST-$COMMIT_LAST + fi + + if [[ "$YEARS" == "$NEW" ]]; then + continue + else + if [[ ${UPDATE_DATES} -eq 1 ]]; then + sed -i "s/Copyright ${YEARS}/Copyright ${NEW}/g" "${src_path}" + else + echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 + RV=1 + fi + fi + fi + else + echo "error: unknown commit dates in file: $file" >&2 + RV=1 + fi +done +rm -f $TMP $TMP2 $TEMPFILE + +# check if error found +if [ $RV -eq 0 ]; then + echo "Copyright headers are OK." +else + echo "Error(s) in copyright headers found!" >&2 +fi +exit $RV diff --git a/scripts/check_license/file-exceptions.sh b/scripts/check_license/file-exceptions.sh new file mode 100755 index 0000000000..144e4b65b4 --- /dev/null +++ b/scripts/check_license/file-exceptions.sh @@ -0,0 +1,34 @@ +#!/bin/sh -e +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# You can add an exception file +# list for license and copyright check +grep -v -E -e 'benchmark/ubench.h' \ + -e 'ChangeLog' \ + -e 'CODEOWNERS$' \ + -e 'docs/assets/.*' \ + -e 'docs/config/conf.py' \ + -e 'docs/config/Doxyfile' \ + -e 'include/umf/proxy_lib_new_delete.h' \ + -e 'LICENSE.TXT' \ + -e 'scripts/assets/images/.*' \ + -e 'src/uthash/.*' \ + -e 'src/uthash/utlist.h' \ + -e 'src/uthash/uthash.h' \ + -e 'test/supp/.*' \ + -e '.clang-format$' \ + -e '.cmake-format$' \ + -e '.cmake.in$' \ + -e '.gitignore' \ + -e '.json$' \ + -e '.mailmap' \ + -e '.md$' \ + -e '.patch$' \ + -e '.rst$' \ + -e '.spellcheck-conf.toml' \ + -e '.trivyignore' \ + -e '.txt$' \ + -e '.xml$' \ + -e '.yml$' diff --git a/src/ctl/ctl.c b/src/ctl/ctl.c index d54e8390ea..4db11ac21f 100644 --- a/src/ctl/ctl.c +++ b/src/ctl/ctl.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2016-2024 Intel Corporation + * Copyright (C) 2016-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,7 +9,7 @@ // This file was originally under following license: // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2016-2024, Intel Corporation */ +/* Copyright 2024, Intel Corporation */ /* * ctl.c -- implementation of the interface for examination and modification of diff --git a/src/libumf.def b/src/libumf.def index 5d1c5047f0..d4c8bb7776 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -1,5 +1,5 @@ ;;;; Begin Copyright Notice -; Copyright (C) 2024 Intel Corporation +; Copyright (C) 2023-2025 Intel Corporation ; Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. ; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ;;;; End Copyright Notice diff --git a/src/libumf.map b/src/libumf.map index d604dd64e6..ff6348e229 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -1,4 +1,4 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/libumf.rc.in b/src/libumf.rc.in index 7aba79e7ed..8ee85d6268 100644 --- a/src/libumf.rc.in +++ b/src/libumf.rc.in @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -51,7 +51,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Unified Memory Framework (UMF) library\0" VALUE "FileVersion", _UMF_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" + VALUE "LegalCopyright", "Copyright 2024-2025, Intel Corporation. All rights reserved.\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "umf.dll\0" VALUE "ProductName", "Unified Memory Framework (UMF)\0" diff --git a/src/memory_pool_internal.h b/src/memory_pool_internal.h index e556ace214..ab3378163d 100644 --- a/src/memory_pool_internal.h +++ b/src/memory_pool_internal.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/memory_provider_get_last_failed.c b/src/memory_provider_get_last_failed.c index 9434eea976..09bd075e10 100644 --- a/src/memory_provider_get_last_failed.c +++ b/src/memory_provider_get_last_failed.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/memory_provider_internal.h b/src/memory_provider_internal.h index 60955e0fbd..0b7f45f80d 100644 --- a/src/memory_provider_internal.h +++ b/src/memory_provider_internal.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/memspaces/memspace_numa.c b/src/memspaces/memspace_numa.c index 0028e394dc..83e65fc291 100644 --- a/src/memspaces/memspace_numa.c +++ b/src/memspaces/memspace_numa.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/memtargets/memtarget_numa.h b/src/memtargets/memtarget_numa.h index 2d3e3fd704..6659d045ef 100644 --- a/src/memtargets/memtarget_numa.h +++ b/src/memtargets/memtarget_numa.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/pool/pool_disjoint.cpp b/src/pool/pool_disjoint.cpp index e0298b43df..0390f53751 100644 --- a/src/pool/pool_disjoint.cpp +++ b/src/pool/pool_disjoint.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/proxy_lib/proxy_lib.rc.in b/src/proxy_lib/proxy_lib.rc.in index dce151ec3e..f0497fb400 100644 --- a/src/proxy_lib/proxy_lib.rc.in +++ b/src/proxy_lib/proxy_lib.rc.in @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -51,7 +51,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Unified Memory Framework (UMF) proxy library\0" VALUE "FileVersion", _UMF_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" + VALUE "LegalCopyright", "Copyright 2024-2025, Intel Corporation. All rights reserved.\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "umf_proxy.dll\0" VALUE "ProductName", "Unified Memory Framework (UMF)\0" diff --git a/src/utils/utils_common.h b/src/utils/utils_common.h index 9ef2b3cf13..6af5a08d91 100644 --- a/src/utils/utils_common.h +++ b/src/utils/utils_common.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/utils/utils_posix_concurrency.c b/src/utils/utils_posix_concurrency.c index fcf04ed952..531e09c10a 100644 --- a/src/utils/utils_posix_concurrency.c +++ b/src/utils/utils_posix_concurrency.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/utils/utils_windows_concurrency.c b/src/utils/utils_windows_concurrency.c index 696f4523bc..e2cc574a9e 100644 --- a/src/utils/utils_windows_concurrency.c +++ b/src/utils/utils_windows_concurrency.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/src/utils/utils_windows_math.c b/src/utils/utils_windows_math.c index 07c4c9978b..cd21ae696f 100644 --- a/src/utils/utils_windows_math.c +++ b/src/utils/utils_windows_math.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/common/pool_trace.c b/test/common/pool_trace.c index 29329f31c0..d8b7522ea8 100644 --- a/test/common/pool_trace.c +++ b/test/common/pool_trace.c @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/common/test_helpers.c b/test/common/test_helpers.c index 71f018d0f7..d69ca35353 100644 --- a/test/common/test_helpers.c +++ b/test/common/test_helpers.c @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains tests for UMF pool API diff --git a/test/malloc_compliance_tests.cpp b/test/malloc_compliance_tests.cpp index 06e3b5dd78..b91bde1f6d 100644 --- a/test/malloc_compliance_tests.cpp +++ b/test/malloc_compliance_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 6f54fe1142..bd97ac1fa6 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/pools/jemalloc_pool.cpp b/test/pools/jemalloc_pool.cpp index 042841fc48..86784d919f 100644 --- a/test/pools/jemalloc_pool.cpp +++ b/test/pools/jemalloc_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/pools/pool_base_alloc.cpp b/test/pools/pool_base_alloc.cpp index ec07a7c2fe..752d9f01e2 100644 --- a/test/pools/pool_base_alloc.cpp +++ b/test/pools/pool_base_alloc.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/pools/scalable_pool.cpp b/test/pools/scalable_pool.cpp index 51cc020305..ce55923d9a 100644 --- a/test/pools/scalable_pool.cpp +++ b/test/pools/scalable_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/provider_os_memory.cpp b/test/provider_os_memory.cpp index 4c81b84f9e..9544a6feda 100644 --- a/test/provider_os_memory.cpp +++ b/test/provider_os_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception From 83a6a46709ab47c18db51e8930ebdfaa55cc7e84 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Mon, 13 Jan 2025 15:04:41 +0100 Subject: [PATCH 73/82] Add GAI Tooling Notice to readme --- README.md | 49 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4cd1d8ff58..b16f35ff64 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For a quick introduction to UMF usage, please see [examples](https://oneapi-src.github.io/unified-memory-framework/examples.html) documentation, which includes the code of the [basic example](https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/basic.c). -The are also more advanced that allocates USM memory from the +The are also more advanced that allocates USM memory from the [Level Zero device](https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/level_zero_shared_memory/level_zero_shared_memory.c) using the Level Zero API and UMF Level Zero memory provider and [CUDA device](https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/cuda_shared_memory/cuda_shared_memory.c) using the CUDA API and UMF CUDA memory provider. @@ -28,19 +28,23 @@ using the CUDA API and UMF CUDA memory provider. ### Requirements Required packages: + - libhwloc-dev >= 2.3.0 (Linux) / hwloc >= 2.3.0 (Windows) - C compiler - [CMake](https://cmake.org/) >= 3.14.0 For development and contributions: + - clang-format-15.0 (can be installed with `python -m pip install clang-format==15.0.7`) - cmake-format-0.6 (can be installed with `python -m pip install cmake-format==0.6.13`) - black (can be installed with `python -m pip install black==24.3.0`) For building tests, multithreaded benchmarks and Disjoint Pool: + - C++ compiler with C++17 support For Level Zero memory provider tests: + - Level Zero headers and libraries - compatible GPU with installed driver @@ -50,8 +54,8 @@ Executable and binaries will be in **build/bin**. The `{build_config}` can be either `Debug` or `Release`. ```bash -$ cmake -B build -DCMAKE_BUILD_TYPE={build_config} -$ cmake --build build -j $(nproc) +cmake -B build -DCMAKE_BUILD_TYPE={build_config} +cmake --build build -j $(nproc) ``` ### Windows @@ -60,8 +64,8 @@ Generating Visual Studio Project. EXE and binaries will be in **build/bin/{build The `{build_config}` can be either `Debug` or `Release`. ```bash -$ cmake -B build -G "Visual Studio 15 2017 Win64" -$ cmake --build build --config {build_config} -j $Env:NUMBER_OF_PROCESSORS +cmake -B build -G "Visual Studio 15 2017 Win64" +cmake --build build --config {build_config} -j $Env:NUMBER_OF_PROCESSORS ``` ### Benchmark @@ -73,20 +77,22 @@ UMF also provides multithreaded benchmarks that can be enabled by setting both `UMF_BUILD_BENCHMARKS` and `UMF_BUILD_BENCHMARKS_MT` CMake configuration flags to `ON`. Multithreaded benchmarks require a C++ support. -The Scalable Pool requirements can be found in the relevant 'Memory Pool +The Scalable Pool requirements can be found in the relevant 'Memory Pool managers' section below. ### Sanitizers List of sanitizers available on Linux: + - AddressSanitizer - UndefinedBehaviorSanitizer - ThreadSanitizer - - Is mutually exclusive with other sanitizers. + - Is mutually exclusive with other sanitizers. - MemorySanitizer - - Requires linking against MSan-instrumented libraries to prevent false positive reports. More information [here](https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo). + - Requires linking against MSan-instrumented libraries to prevent false positive reports. More information [here](https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo). List of sanitizers available on Windows: + - AddressSanitizer Listed sanitizers can be enabled with appropriate [CMake options](#cmake-standard-options). @@ -133,7 +139,7 @@ UMF comes with predefined pool allocators (see [`include/umf/pools`](include/umf providers that implement a specific interface (see [`include/umf/memory_pool_ops.h`](include/umf/memory_pool_ops.h) and [`include/umf/memory_provider_ops.h`](include/umf/memory_provider_ops.h)). -More detailed documentation is available here: https://oneapi-src.github.io/unified-memory-framework/ +More detailed documentation is available here: ### Memory providers @@ -146,6 +152,7 @@ A memory provider that can provide memory from a given pre-allocated buffer. A memory provider that provides memory from an operating system. OS memory provider supports two types of memory mappings (set by the `visibility` parameter): + 1) private memory mapping (`UMF_MEM_MAP_PRIVATE`) 2) shared memory mapping (`UMF_MEM_MAP_SHARED` - supported on Linux only yet) @@ -159,16 +166,18 @@ Permission to duplicate another process's file descriptor is governed by a ptrac the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: ```sh -$ sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" +sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" ``` There are available two mechanisms for the shared memory mapping: + 1) a named shared memory object (used if the `shm_name` parameter is not NULL) or 2) an anonymous file descriptor (used if the `shm_name` parameter is NULL) The `shm_name` parameter should be a null-terminated string of up to NAME_MAX (i.e., 255) characters none of which are slashes. An anonymous file descriptor for the shared memory mapping will be created using: + 1) `memfd_secret()` syscall - (if it is implemented and) if the `UMF_MEM_FD_FUNC` environment variable does not contain the "memfd_create" string or 2) `memfd_create()` syscall - otherwise (and if it is implemented). @@ -178,7 +187,8 @@ IPC API on Linux requires the `PTRACE_MODE_ATTACH_REALCREDS` permission (see `pt to duplicate another process's file descriptor (see above). Packages required for tests (Linux-only yet): - - libnuma-dev + +- libnuma-dev #### Level Zero memory provider @@ -191,7 +201,7 @@ Permission to duplicate another process's file descriptor is governed by a ptrac the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: ```sh -$ sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" +sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" ``` ##### Requirements @@ -266,7 +276,7 @@ To enable this feature, the `UMF_BUILD_LIBUMF_POOL_DISJOINT` option needs to be #### Jemalloc pool -Jemalloc pool is a [jemalloc](https://github.com/jemalloc/jemalloc)-based memory +Jemalloc pool is a [jemalloc](https://github.com/jemalloc/jemalloc)-based memory pool manager built as a separate static library: libjemalloc_pool.a on Linux and jemalloc_pool.lib on Windows. The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option has to be turned `ON` to build this library. @@ -275,6 +285,7 @@ The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option has to be turned `ON` to build this In case of Linux OS jemalloc is built from the (fetched) sources with the following non-default options enabled: + - `--with-jemalloc-prefix=je_` - adds the `je_` prefix to all public APIs, - `--disable-cxx` - disables C++ integration, it will cause the `new` and the `delete` operators implementations to be omitted. @@ -289,6 +300,7 @@ The default jemalloc package is required on Windows. 1) The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned `ON` 2) jemalloc is required: + - on Linux and MacOS: jemalloc is fetched and built from sources (a custom build), - on Windows: the default jemalloc package is required @@ -300,7 +312,8 @@ It is distributed as part of libumf. To use this pool, TBB must be installed in ##### Requirements Packages required for using this pool and executing tests/benchmarks (not required for build): - - libtbb-dev (libtbbmalloc.so.2) on Linux or tbb (tbbmalloc.dll) on Windows + +- libtbb-dev (libtbbmalloc.so.2) on Linux or tbb (tbbmalloc.dll) on Windows ### Memspaces (Linux-only) @@ -338,10 +351,11 @@ To enable this feature, the `UMF_BUILD_SHARED_LIBRARY` option needs to be turned In case of Linux it can be done without any code changes using the `LD_PRELOAD` environment variable: ```sh -$ LD_PRELOAD=/usr/lib/libumf_proxy.so myprogram +LD_PRELOAD=/usr/lib/libumf_proxy.so myprogram ``` The memory used by the proxy memory allocator is mmap'ed: + 1) with the `MAP_PRIVATE` flag by default or 2) with the `MAP_SHARED` flag if the `UMF_PROXY` environment variable contains one of two following strings: `page.disposition=shared-shm` or `page.disposition=shared-fd`. These two options differ in a mechanism used during IPC: - `page.disposition=shared-shm` - IPC uses the named shared memory. An SHM name is generated using the `umf_proxy_lib_shm_pid_$PID` pattern, where `$PID` is the PID of the process. It creates the `/dev/shm/umf_proxy_lib_shm_pid_$PID` file. @@ -357,6 +371,7 @@ It can be enabled by adding the `size.threshold=` string to the `UMF_PROX #### Windows In case of Windows it requires: + 1) explicitly linking your program dynamically with the `umf_proxy.dll` library 2) (C++ code only) including `proxy_lib_new_delete.h` in a single(!) source file in your project to override also the `new`/`delete` operations. @@ -370,3 +385,7 @@ an issue or a Pull Request, please read [Contribution Guide](./CONTRIBUTING.md). To enable logging in UMF source files please follow the guide in the [web documentation](https://oneapi-src.github.io/unified-memory-framework/introduction.html#logging). + +## Notices + +The contents of this repository may have been developed with support from one or more Intel-operated generative artificial intelligence solutions. From b229a346f36bca356aa2985c798978597c722532 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 9 Jan 2025 10:43:28 +0100 Subject: [PATCH 74/82] Added a condition to check the validity of the starting date and some fixes Changes for this commit: - adding a condition to check the validity of the starting date - broken pipe fix - updating date fix --- .github/workflows/reusable_checks.yml | 4 -- CMakeLists.txt | 2 +- scripts/check_license/check_headers.sh | 80 +++++++++++++++--------- scripts/check_license/file-exceptions.sh | 5 +- src/pool/CMakeLists.txt | 2 +- 5 files changed, 56 insertions(+), 37 deletions(-) diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index de28161a58..6e700cec1c 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -56,10 +56,6 @@ jobs: run: | ./scripts/check_license/check_headers.sh . "Apache-2.0 WITH LLVM-exception" -v - - name: Run copyright-format - run: | - ./scripts/check_license/check_headers.sh . "Apache-2.0 WITH LLVM-exception" -d - - name: Run a spell check uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2 with: diff --git a/CMakeLists.txt b/CMakeLists.txt index 495c70de30..58b189ce17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2024 Intel Corporation +# Copyright (C) 2022-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/scripts/check_license/check_headers.sh b/scripts/check_license/check_headers.sh index d68b0891bc..aeb90e7a28 100755 --- a/scripts/check_license/check_headers.sh +++ b/scripts/check_license/check_headers.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash - # Copyright (C) 2016-2024 Intel Corporation - # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. - # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Copyright (C) 2016-2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # check-headers.sh - check copyright and license in source files @@ -68,10 +68,10 @@ while [ "$1" != "" ]; do done if [ $CHECK_ALL -eq 0 ]; then - CURRENT_COMMIT=$($GIT log --pretty=%H -1) - MERGE_BASE=$($GIT merge-base HEAD origin/master 2>/dev/null) + CURRENT_COMMIT=$($GIT --no-pager log --pretty=%H -1) + MERGE_BASE=$($GIT merge-base HEAD origin/main 2>/dev/null) [ -z $MERGE_BASE ] && \ - MERGE_BASE=$($GIT log --pretty="%cN:%H" | grep GitHub | head -n1 | cut -d: -f2) + MERGE_BASE=$($GIT --no-pager log --pretty="%cN:%H" | grep GitHub 2>/dev/null | head -n1 | cut -d: -f2) [ -z $MERGE_BASE -o "$CURRENT_COMMIT" = "$MERGE_BASE" ] && \ CHECK_ALL=1 fi @@ -127,7 +127,7 @@ for file in $FILES ; do LAST=` tail -n1 $TMP2` YEARS=$(sed ' -/.*Copyright (C) \+.*[0-9-]\+ Intel Corporation/!d +/.*Copyright (C) [0-9-]\+ Intel Corporation/!d s/.*Copyright (C) \([0-9]\+\)-\([0-9]\+\).*/\1-\2/ s/.*Copyright (C) \([0-9]\+\).*/\1/' "$src_path") if [ -z "$YEARS" ]; then @@ -142,29 +142,49 @@ s/.*Copyright (C) \([0-9]\+\).*/\1/' "$src_path") COMMIT_FIRST=`echo $FIRST | cut -d"-" -f1` COMMIT_LAST=` echo $LAST | cut -d"-" -f1` - if [ "$COMMIT_FIRST" != "" -a "$COMMIT_LAST" != "" ]; then - if [[ -n "$COMMIT_FIRST" && -n "$COMMIT_LAST" ]]; then - if [[ $COMMIT_FIRST -eq $COMMIT_LAST ]]; then - NEW=$COMMIT_LAST - else - NEW=$COMMIT_FIRST-$COMMIT_LAST - fi - - if [[ "$YEARS" == "$NEW" ]]; then - continue - else - if [[ ${UPDATE_DATES} -eq 1 ]]; then - sed -i "s/Copyright ${YEARS}/Copyright ${NEW}/g" "${src_path}" - else - echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 - RV=1 - fi - fi - fi - else - echo "error: unknown commit dates in file: $file" >&2 - RV=1 - fi + if [ "$COMMIT_FIRST" != "" -a "$COMMIT_LAST" != "" ]; then + if [ "$COMMIT_FIRST" -lt "$HEADER_FIRST" ]; then + RV=1 + fi + + if [[ -n "$COMMIT_FIRST" && -n "$COMMIT_LAST" ]]; then + if [[ $HEADER_FIRST -le $COMMIT_FIRST ]]; then + if [[ $HEADER_LAST -eq $COMMIT_LAST ]]; then + continue + else + NEW="$HEADER_FIRST-$COMMIT_LAST" + if [[ ${UPDATE_DATES} -eq 1 ]]; then + echo "Updating copyright date in $src_path: $YEARS -> $NEW" + sed -i "s/Copyright (C) ${YEARS}/Copyright (C) ${NEW}/g" "${src_path}" + else + echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 + RV=1 + fi + fi + else + if [[ $COMMIT_FIRST -eq $COMMIT_LAST ]]; then + NEW=$COMMIT_LAST + else + NEW=$COMMIT_FIRST-$COMMIT_LAST + fi + + if [[ "$YEARS" == "$NEW" ]]; then + continue + else + if [[ ${UPDATE_DATES} -eq 1 ]]; then + echo "Updating copyright date in $src_path: $YEARS -> $NEW" + sed -i "s/Copyright (C) ${YEARS}/Copyright (C) ${NEW}/g" "${src_path}" + else + echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 + RV=1 + fi + fi + fi + fi + else + echo "error: unknown commit dates in file: $file" >&2 + RV=1 + fi done rm -f $TMP $TMP2 $TEMPFILE diff --git a/scripts/check_license/file-exceptions.sh b/scripts/check_license/file-exceptions.sh index 144e4b65b4..5a12443d9f 100755 --- a/scripts/check_license/file-exceptions.sh +++ b/scripts/check_license/file-exceptions.sh @@ -13,11 +13,15 @@ grep -v -E -e 'benchmark/ubench.h' \ -e 'docs/config/Doxyfile' \ -e 'include/umf/proxy_lib_new_delete.h' \ -e 'LICENSE.TXT' \ + -e 'licensing/third-party-programs.txt' \ -e 'scripts/assets/images/.*' \ + -e 'scripts/qemu/requirements.txt' \ -e 'src/uthash/.*' \ -e 'src/uthash/utlist.h' \ -e 'src/uthash/uthash.h' \ + -e 'test/ctl/config.txt' \ -e 'test/supp/.*' \ + -e 'third_party/requirements.txt' \ -e '.clang-format$' \ -e '.cmake-format$' \ -e '.cmake.in$' \ @@ -29,6 +33,5 @@ grep -v -E -e 'benchmark/ubench.h' \ -e '.rst$' \ -e '.spellcheck-conf.toml' \ -e '.trivyignore' \ - -e '.txt$' \ -e '.xml$' \ -e '.yml$' diff --git a/src/pool/CMakeLists.txt b/src/pool/CMakeLists.txt index 17be932a45..f54e701857 100644 --- a/src/pool/CMakeLists.txt +++ b/src/pool/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception From 42a95fd3ed047db4c38cd9a68d9a329d6b3c8dd1 Mon Sep 17 00:00:00 2001 From: Katarzyna Luszczewska Date: Mon, 13 Jan 2025 15:56:02 +0100 Subject: [PATCH 75/82] Fix spelling errors in a few files --- docs/config/api.rst | 2 +- docs/config/examples.rst | 2 +- include/umf/memory_pool.h | 4 ++-- include/umf/pools/pool_disjoint.h | 13 +++++++++---- include/umf/providers/provider_os_memory.h | 8 +++++--- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/config/api.rst b/docs/config/api.rst index 3eedc8f1d9..1c20d709c2 100644 --- a/docs/config/api.rst +++ b/docs/config/api.rst @@ -86,7 +86,7 @@ and operate on the provider. Fixed Memory Provider ------------------------------------------ -A memory provider that can provide memory from a given pre-allocated buffer. +A memory provider that can provide memory from a given preallocated buffer. .. doxygenfile:: provider_fixed_memory.h :sections: define enum typedef func var diff --git a/docs/config/examples.rst b/docs/config/examples.rst index c58e7fc223..4eeea6aa95 100644 --- a/docs/config/examples.rst +++ b/docs/config/examples.rst @@ -178,7 +178,7 @@ by a different library and the caller of the :any:`umfGetIPCHandle` function may The :any:`umfGetIPCHandle` function returns the IPC handle and its size. The IPC handle is a byte-copyable opaque data structure. The :any:`umf_ipc_handle_t` type is defined as a pointer to a byte array. The size of the handle might be different for different memory provider types. The code snippet below demonstrates how the IPC handle can -be serialized for marshalling purposes. +be serialized for marshaling purposes. .. code-block:: c diff --git a/include/umf/memory_pool.h b/include/umf/memory_pool.h index de045acf49..ae5e67a969 100644 --- a/include/umf/memory_pool.h +++ b/include/umf/memory_pool.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -140,7 +140,7 @@ umf_result_t umfFree(void *ptr); /// * Implementations *must* store the error code in thread-local /// storage prior to returning NULL from the allocation functions. /// -/// * If the last allocation/de-allocation operation succeeded, the value returned by +/// * If the last allocation/deallocation operation succeeded, the value returned by /// this function is unspecified. /// /// * The application *may* call this function from simultaneous threads. diff --git a/include/umf/pools/pool_disjoint.h b/include/umf/pools/pool_disjoint.h index fdf682ae5b..d268a1dac4 100644 --- a/include/umf/pools/pool_disjoint.h +++ b/include/umf/pools/pool_disjoint.h @@ -1,6 +1,11 @@ -// Copyright (C) 2023-2024 Intel Corporation -// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/* + * + * Copyright (C) 2023-2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ #pragma once #ifdef __cplusplus @@ -87,7 +92,7 @@ umfDisjointPoolParamsSetTrace(umf_disjoint_pool_params_handle_t hParams, /// @brief Set shared limits for disjoint pool. /// @param hParams handle to the parameters of the disjoint pool. -/// @param hSharedLimits handle tp the shared limits. +/// @param hSharedLimits handle to the shared limits. /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. umf_result_t umfDisjointPoolParamsSetSharedLimits( umf_disjoint_pool_params_handle_t hParams, diff --git a/include/umf/providers/provider_os_memory.h b/include/umf/providers/provider_os_memory.h index a6bf43a7d9..90455cad19 100644 --- a/include/umf/providers/provider_os_memory.h +++ b/include/umf/providers/provider_os_memory.h @@ -1,9 +1,11 @@ /* - * Copyright (C) 2022-2024 Intel Corporation + * + * Copyright (C) 2022-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -*/ + * + */ #ifndef UMF_OS_MEMORY_PROVIDER_H #define UMF_OS_MEMORY_PROVIDER_H @@ -23,7 +25,7 @@ extern "C" { /// Not every mode is supported on every system. typedef enum umf_numa_mode_t { /// Default binding mode. Actual binding policy is system-specific. On - /// linux this corresponds to MPOL_DEFAULT. If this mode is specified, + /// Linux this corresponds to MPOL_DEFAULT. If this mode is specified, /// nodemask must be NULL and maxnode must be 0. UMF_NUMA_MODE_DEFAULT, From f72a6130a52f459951768a12d6e1dc086d869bc2 Mon Sep 17 00:00:00 2001 From: Katarzyna Luszczewska Date: Mon, 13 Jan 2025 15:54:42 +0100 Subject: [PATCH 76/82] Add spelling check in the web docs --- .github/workflows/reusable_checks.yml | 9 ++- docs/config/Doxyfile | 2 +- docs/config/conf.py | 5 +- docs/config/spelling_exceptions.txt | 74 ++++++++++++++++++++++++ scripts/check_license/file-exceptions.sh | 1 + third_party/requirements.txt | 3 + 6 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 docs/config/spelling_exceptions.txt diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml index 6e700cec1c..a7602d2696 100644 --- a/.github/workflows/reusable_checks.yml +++ b/.github/workflows/reusable_checks.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev + sudo apt-get install -y black cmake clang-format-15 cmake-format libhwloc-dev doxygen # Latest distros do not allow global pip installation - name: Install Python requirements in venv @@ -29,6 +29,7 @@ jobs: python3 -m venv .venv . .venv/bin/activate echo "$PATH" >> $GITHUB_PATH + python3 -m pip install -r third_party/requirements.txt python3 -m pip install bandit codespell - name: Configure CMake @@ -64,6 +65,12 @@ jobs: - name: Run codespell run: python3 ./.github/scripts/run-codespell.py + - name: Check spelling in docs + run: | + cmake -B build + cmake --build build --target docs + sphinx-build -b spelling ./build/docs_build/config ./build/docs_build/spelling_log -W + # Run Bandit recursively, but omit _deps directory (with 3rd party code) and python's venv - name: Run Bandit run: python3 -m bandit -r . -x '/_deps/,/.venv/' diff --git a/docs/config/Doxyfile b/docs/config/Doxyfile index f23117ff29..6309463745 100644 --- a/docs/config/Doxyfile +++ b/docs/config/Doxyfile @@ -445,7 +445,7 @@ INLINE_SIMPLE_STRUCTS = NO # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. -TYPEDEF_HIDES_STRUCT = NO +TYPEDEF_HIDES_STRUCT = YES # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be diff --git a/docs/config/conf.py b/docs/config/conf.py index fa4788ff42..ae698ba98c 100644 --- a/docs/config/conf.py +++ b/docs/config/conf.py @@ -36,7 +36,10 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["breathe"] +extensions = ["breathe", "sphinxcontrib.spelling"] + +spelling_show_suggestions = True +spelling_word_list_filename = "spelling_exceptions.txt" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/config/spelling_exceptions.txt b/docs/config/spelling_exceptions.txt new file mode 100644 index 0000000000..d4e40a3ec8 --- /dev/null +++ b/docs/config/spelling_exceptions.txt @@ -0,0 +1,74 @@ +addr +allocatable +allocator +allocators +calloc +CXL +copyable +customizable +daxX +deallocation +deallocating +deallocations +Devdax +dev +Globals +hMemtarget +hPool +hProvider +highPtr +io +interprocess +ipc +jemalloc +lowPtr +malloc +maxnode +mem +mempolicies +mempolicy +Mempolicy +memspace +Memspace +memspaces +Memtarget +memtarget +memtargets +middleware +multithreading +Nodemask +nodemask +numa +oneAPI +oneTBB +os +params +partList +pid +poolable +preallocated +providerIpcData +providential +ptr +realloc +Scalable +scalable +stdout +Tiering +tiering +topologies +umf +umfGetIPCHandle +umfMemoryProviderAlloc +umfMemoryProviderGetLastNativeError +umfMemoryProviderOpenIPCHandle +umfOsMemoryProviderParamsDestroy +umfPool +umfPoolCalloc +umfPoolDestroy +umfPoolGetTag +umfPoolMallocUsableSize +umfPoolRealloc +umfMemspaceUserFilter +umfMemspaceMemtargetAdd +unfreed \ No newline at end of file diff --git a/scripts/check_license/file-exceptions.sh b/scripts/check_license/file-exceptions.sh index 5a12443d9f..10c5560614 100755 --- a/scripts/check_license/file-exceptions.sh +++ b/scripts/check_license/file-exceptions.sh @@ -9,6 +9,7 @@ grep -v -E -e 'benchmark/ubench.h' \ -e 'ChangeLog' \ -e 'CODEOWNERS$' \ -e 'docs/assets/.*' \ + -e 'docs/config/spelling_exceptions.txt' \ -e 'docs/config/conf.py' \ -e 'docs/config/Doxyfile' \ -e 'include/umf/proxy_lib_new_delete.h' \ diff --git a/third_party/requirements.txt b/third_party/requirements.txt index 9832bf2f0f..1255dcb929 100644 --- a/third_party/requirements.txt +++ b/third_party/requirements.txt @@ -15,3 +15,6 @@ sphinxcontrib_qthelp==2.0.0 breathe==4.35.0 sphinx==8.1.3 sphinx_book_theme==1.1.3 +# Spelling check in documentation +pyenchant==3.2.2 +sphinxcontrib-spelling==8.0.0 From 39f77a17c341fe4eb9c486de24e064e7146be8f6 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Mon, 2 Dec 2024 18:20:36 +0000 Subject: [PATCH 77/82] L0 provider: implement support for defer and blocking free --- include/umf/providers/provider_level_zero.h | 15 +++++ src/libumf.def | 2 + src/libumf.map | 1 + src/provider/provider_level_zero.c | 60 ++++++++++++++++++- .../provider_level_zero_not_impl.cpp | 4 ++ 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index f760c57244..df6dd7364c 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -68,6 +68,21 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( umf_level_zero_memory_provider_params_handle_t hParams, ze_device_handle_t *hDevices, uint32_t deviceCount); +typedef enum umf_level_zero_memory_provider_free_policy_t { + UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT = + 0, ///< Free memory immediately. Default. + UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_BLOCKING_FREE, ///< Blocks until all commands using the memory are complete before freeing. + UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFER_FREE, ///< Schedules the memory to be freed but does not free immediately. +} umf_level_zero_memory_provider_free_policy_t; + +/// @brief Set the memory free policy. +/// @param hParams handle to the parameters of the Level Zero Memory Provider. +/// @param policy memory free policy. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfLevelZeroMemoryProviderParamsSetFreePolicy( + umf_level_zero_memory_provider_params_handle_t hParams, + umf_level_zero_memory_provider_free_policy_t policy); + umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void); #ifdef __cplusplus diff --git a/src/libumf.def b/src/libumf.def index d053fa2407..42d7cfaf3c 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -120,3 +120,5 @@ EXPORTS umfScalablePoolParamsDestroy umfScalablePoolParamsSetGranularity umfScalablePoolParamsSetKeepAllMemory +; Added in UMF_0.11 + umfLevelZeroMemoryProviderParamsSetFreePolicy diff --git a/src/libumf.map b/src/libumf.map index 9aecf8f53f..c33bb7c10f 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -119,4 +119,5 @@ UMF_0.11 { umfFixedMemoryProviderOps; umfFixedMemoryProviderParamsCreate; umfFixedMemoryProviderParamsDestroy; + umfLevelZeroMemoryProviderParamsSetFreePolicy; } UMF_0.10; diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index f3ce269b25..eaea8abd90 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -75,6 +75,14 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( return UMF_RESULT_ERROR_NOT_SUPPORTED; } +umf_result_t umfLevelZeroMemoryProviderParamsSetFreePolicy( + umf_level_zero_memory_provider_params_handle_t hParams, + umf_level_zero_memory_provider_free_policy_t policy) { + (void)hParams; + (void)policy; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { // not supported LOG_ERR("L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " @@ -107,6 +115,9 @@ typedef struct umf_level_zero_memory_provider_params_t { resident_device_handles; ///< Array of devices for which the memory should be made resident uint32_t resident_device_count; ///< Number of devices for which the memory should be made resident + + umf_level_zero_memory_provider_free_policy_t + freePolicy; ///< Memory free policy } umf_level_zero_memory_provider_params_t; typedef struct ze_memory_provider_t { @@ -118,6 +129,8 @@ typedef struct ze_memory_provider_t { uint32_t resident_device_count; ze_device_properties_t device_properties; + + ze_driver_memory_free_policy_ext_flags_t freePolicyFlags; } ze_memory_provider_t; typedef struct ze_ops_t { @@ -144,6 +157,8 @@ typedef struct ze_ops_t { size_t); ze_result_t (*zeDeviceGetProperties)(ze_device_handle_t, ze_device_properties_t *); + ze_result_t (*zeMemFreeExt)(ze_context_handle_t, + ze_memory_free_ext_desc_t *, void *); } ze_ops_t; static ze_ops_t g_ze_ops; @@ -197,6 +212,8 @@ static void init_ze_global_state(void) { utils_get_symbol_addr(0, "zeContextMakeMemoryResident", lib_name); *(void **)&g_ze_ops.zeDeviceGetProperties = utils_get_symbol_addr(0, "zeDeviceGetProperties", lib_name); + *(void **)&g_ze_ops.zeMemFreeExt = + utils_get_symbol_addr(0, "zeMemFreeExt", lib_name); if (!g_ze_ops.zeMemAllocHost || !g_ze_ops.zeMemAllocDevice || !g_ze_ops.zeMemAllocShared || !g_ze_ops.zeMemFree || @@ -232,6 +249,7 @@ umf_result_t umfLevelZeroMemoryProviderParamsCreate( params->memory_type = UMF_MEMORY_TYPE_UNKNOWN; params->resident_device_handles = NULL; params->resident_device_count = 0; + params->freePolicy = UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT; *hParams = params; @@ -308,6 +326,32 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( return UMF_RESULT_SUCCESS; } +umf_result_t umfLevelZeroMemoryProviderParamsSetFreePolicy( + umf_level_zero_memory_provider_params_handle_t hParams, + umf_level_zero_memory_provider_free_policy_t policy) { + if (!hParams) { + LOG_ERR("Level zero memory provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->freePolicy = policy; + return UMF_RESULT_SUCCESS; +} + +static ze_driver_memory_free_policy_ext_flags_t +umfFreePolicyToZePolicy(umf_level_zero_memory_provider_free_policy_t policy) { + switch (policy) { + case UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT: + return 0; + case UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_BLOCKING_FREE: + return ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_BLOCKING_FREE; + case UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFER_FREE: + return ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_DEFER_FREE; + default: + return 0; + } +} + static umf_result_t ze_memory_provider_initialize(void *params, void **provider) { if (params == NULL) { @@ -351,6 +395,8 @@ static umf_result_t ze_memory_provider_initialize(void *params, ze_provider->context = ze_params->level_zero_context_handle; ze_provider->device = ze_params->level_zero_device_handle; ze_provider->memory_type = (ze_memory_type_t)ze_params->memory_type; + ze_provider->freePolicyFlags = + umfFreePolicyToZePolicy(ze_params->freePolicy); memset(&ze_provider->device_properties, 0, sizeof(ze_provider->device_properties)); @@ -493,8 +539,18 @@ static umf_result_t ze_memory_provider_free(void *provider, void *ptr, } ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; - ze_result_t ze_result = g_ze_ops.zeMemFree(ze_provider->context, ptr); - return ze2umf_result(ze_result); + + if (ze_provider->freePolicyFlags == 0) { + return ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); + } + + ze_memory_free_ext_desc_t desc = { + .stype = ZE_STRUCTURE_TYPE_MEMORY_FREE_EXT_DESC, + .pNext = NULL, + .freePolicy = ze_provider->freePolicyFlags}; + + return ze2umf_result( + g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); } static void ze_memory_provider_get_last_native_error(void *provider, diff --git a/test/providers/provider_level_zero_not_impl.cpp b/test/providers/provider_level_zero_not_impl.cpp index bea1acbe79..c55c236fe7 100644 --- a/test/providers/provider_level_zero_not_impl.cpp +++ b/test/providers/provider_level_zero_not_impl.cpp @@ -31,6 +31,10 @@ TEST_F(test, level_zero_provider_not_implemented) { hDevices, 1); ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); + result = umfLevelZeroMemoryProviderParamsSetFreePolicy( + hParams, UMF_LEVEL_ZERO_MEMORY_PROVIDER_FREE_POLICY_DEFAULT); + ASSERT_EQ(result, UMF_RESULT_ERROR_NOT_SUPPORTED); + umf_memory_provider_ops_t *ops = umfLevelZeroMemoryProviderOps(); ASSERT_EQ(ops, nullptr); } From 330ffd3791bce93d0bdb941a3f1700f04ded6d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Mon, 13 Jan 2025 16:23:32 +0100 Subject: [PATCH 78/82] reduce code duplication in Benchmark by using Apply() to set arguments --- benchmark/benchmark.cpp | 131 +++++++++++++++++----------------------- benchmark/benchmark.hpp | 25 +++++--- 2 files changed, 72 insertions(+), 84 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index df4fe6e5d5..6c8175e1d0 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -1,11 +1,13 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * */ +#include + #include "benchmark.hpp" #define UMF_BENCHMARK_TEMPLATE_DEFINE(BaseClass, Method, ...) \ @@ -18,14 +20,8 @@ #define UMF_BENCHMARK_REGISTER_F(BaseClass, Method) \ BENCHMARK_REGISTER_F(BaseClass, Method) \ - ->ArgNames( \ - BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::argsName()) \ - ->Name(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::name()) \ - ->Iterations( \ - BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::iterations()) - -UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_fix, fixed_alloc_size, - glibc_malloc); + ->Apply( \ + &BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::defaultArgs) // Benchmarks scenarios: @@ -33,54 +29,56 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_fix, fixed_alloc_size, // benchmark arguments, allocator arguments, size generator arguments. // The exact meaning of each argument depends on the benchmark, allocator, and size components used. // Refer to the 'argsName()' function in each component to find detailed descriptions of these arguments. + +static void default_alloc_fix_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 0, 4096}); + benchmark->Args({10000, 100000, 4096}); + benchmark->Threads(4); + benchmark->Threads(1); +} + +static void +default_alloc_uniform_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 0, 8, 64 * 1024, 8}); + benchmark->Threads(4); + benchmark->Threads(1); +} + +UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_fix, fixed_alloc_size, + glibc_malloc); + UMF_BENCHMARK_REGISTER_F(alloc_benchmark, glibc_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_uniform, uniform_alloc_size, glibc_malloc); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, glibc_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, os_provider, fixed_alloc_size, provider_allocator); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, os_provider) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, proxy_pool, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, proxy_pool) - ->Args({1000, 0, 4096}) - ->Args({1000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); #ifdef UMF_POOL_DISJOINT_ENABLED UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, disjoint_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, disjoint_pool_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); // TODO: debug why this crashes /*UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, disjoint_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, disjoint_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - // ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); */ #endif @@ -89,18 +87,13 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, jemalloc_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, jemalloc_pool_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, jemalloc_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, jemalloc_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); #endif #ifdef UMF_POOL_SCALABLE_ENABLED @@ -109,71 +102,67 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, scalable_pool_fix, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, scalable_pool_fix) - ->Args({10000, 0, 4096}) - ->Args({10000, 100000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, scalable_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(alloc_benchmark, scalable_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_alloc_uniform_size); #endif // Multiple allocs/free +static void +default_multiple_alloc_fix_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 4096}); + benchmark->Threads(4); + benchmark->Threads(1); +} + +static void +default_multiple_alloc_uniform_size(benchmark::internal::Benchmark *benchmark) { + benchmark->Args({10000, 8, 64 * 1024, 8}); + benchmark->Threads(4); + benchmark->Threads(1); +} UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, glibc_fix, fixed_alloc_size, glibc_malloc); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, glibc_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, glibc_uniform, uniform_alloc_size, glibc_malloc); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, glibc_uniform) - ->Args({10000, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, proxy_pool, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, proxy_pool) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, os_provider, fixed_alloc_size, provider_allocator); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, os_provider) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); #ifdef UMF_POOL_DISJOINT_ENABLED UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, disjoint_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); // TODO: debug why this crashes /*UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, disjoint_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_uniform) - ->Args({10000, 0, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); */ #endif @@ -182,17 +171,13 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_fix, fixed_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_uniform) - ->Args({1000, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); #endif @@ -202,18 +187,14 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, scalable_pool_fix, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, scalable_pool_fix) - ->Args({10000, 4096}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_fix_size); UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, scalable_pool_uniform, uniform_alloc_size, pool_allocator>); UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, scalable_pool_uniform) - ->Args({10000, 8, 64 * 1024, 8}) - ->Threads(4) - ->Threads(1); + ->Apply(&default_multiple_alloc_uniform_size); #endif BENCHMARK_MAIN(); diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index df5d6a5927..50e75f8fb2 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -162,8 +162,15 @@ struct benchmark_interface : public benchmark::Fixture { return res; } - static std::string name() { return Allocator::name(); } - static int64_t iterations() { return 10000; } + virtual std::string name() { return Allocator::name(); } + virtual int64_t iterations() { return 10000; } + static void defaultArgs(Benchmark *benchmark) { + auto *bench = + static_cast *>(benchmark); + benchmark->ArgNames(bench->argsName()) + ->Name(bench->name()) + ->Iterations(bench->iterations()); + } Size alloc_size; Allocator allocator; }; @@ -260,15 +267,15 @@ class alloc_benchmark : public benchmark_interface { } } - static std::vector argsName() { + virtual std::vector argsName() { auto n = benchmark_interface::argsName(); std::vector res = {"max_allocs", "pre_allocs"}; res.insert(res.end(), n.begin(), n.end()); return res; } - static std::string name() { return base::name() + "/alloc"; } - static int64_t iterations() { return 200000; } + virtual std::string name() { return base::name() + "/alloc"; } + virtual int64_t iterations() { return 200000; } protected: using base = benchmark_interface; @@ -346,18 +353,18 @@ class multiple_malloc_free_benchmark : public alloc_benchmark { } } - static std::string name() { + virtual std::string name() { return base::base::name() + "/multiple_malloc_free"; } - static std::vector argsName() { + virtual std::vector argsName() { auto n = benchmark_interface::argsName(); std::vector res = {"max_allocs"}; res.insert(res.end(), n.begin(), n.end()); return res; } - static int64_t iterations() { return 2000; } + virtual int64_t iterations() { return 2000; } std::default_random_engine generator; distribution dist; From 279b6bf491890cc128ef31f2e62becc9d146cbe5 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 17 Jan 2025 11:23:04 +0100 Subject: [PATCH 79/82] Fix: use cuda_device_handle in cu_memory_provider_initialize cu_provider->device is not set (equals 0) in this place yet, so we have to use cu_params->cuda_device_handle instead of cu_provider->device here. Signed-off-by: Lukasz Dorau --- src/provider/provider_cuda.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index 9bb11327b9..350bd016fd 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -313,7 +313,7 @@ static umf_result_t cu_memory_provider_initialize(void *params, CUmemAllocationProp allocProps = {0}; allocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; allocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - allocProps.location.id = cu_provider->device; + allocProps.location.id = cu_params->cuda_device_handle; CUresult cu_result = g_cu_ops.cuMemGetAllocationGranularity( &min_alignment, &allocProps, CU_MEM_ALLOC_GRANULARITY_MINIMUM); if (cu_result != CUDA_SUCCESS) { From da7706c1d37fb8f6250989136a781fd078e4fdbb Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Fri, 3 Jan 2025 15:21:25 +0100 Subject: [PATCH 80/82] Add info about ptrace permissions Update our documentation with better info about ptrace - IPC users should be aware of required permissions. Add proposed, two possible solutions into our docs. Also, update our examples and tests to work without global change of ptrace_scope. Co-authored-by: sergey.vinogradov@intel.com --- .github/workflows/reusable_basic.yml | 3 --- .github/workflows/reusable_fast.yml | 4 ---- .github/workflows/reusable_proxy_lib.yml | 3 --- .github/workflows/reusable_sanitizers.yml | 4 ---- README.md | 25 ++++++++++++----------- examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh | 14 +++---------- examples/ipc_ipcapi/ipc_ipcapi_producer.c | 19 ++++++++++++++++- scripts/qemu/run-tests.sh | 4 +--- src/utils/utils_posix_common.c | 7 +++---- test/common/ipc_common.c | 10 ++++++++- test/ipc_os_prov_anon_fd.sh | 17 +-------------- test/providers/ipc_level_zero_prov.sh | 17 +-------------- 12 files changed, 49 insertions(+), 78 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 83542efbbe..25d33e2b3a 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -145,9 +145,6 @@ jobs: - name: Install libhwloc run: .github/scripts/install_hwloc.sh - - name: Set ptrace value for IPC test - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Get UMF version run: | VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 5673727ac1..58a172a74f 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -88,10 +88,6 @@ jobs: sudo apt-get install -y cmake libnuma-dev libtbb-dev .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package - - name: Set ptrace value for IPC test (on Linux only) - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-20.04' }} - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Configure CMake if: matrix.simple_cmake == 'OFF' run: > diff --git a/.github/workflows/reusable_proxy_lib.yml b/.github/workflows/reusable_proxy_lib.yml index 27a66267d1..a1f5975fa6 100644 --- a/.github/workflows/reusable_proxy_lib.yml +++ b/.github/workflows/reusable_proxy_lib.yml @@ -34,9 +34,6 @@ jobs: sudo apt-get update sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov - - name: Set ptrace value for IPC test - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Configure build run: > cmake diff --git a/.github/workflows/reusable_sanitizers.yml b/.github/workflows/reusable_sanitizers.yml index 93752ff846..25458da51f 100644 --- a/.github/workflows/reusable_sanitizers.yml +++ b/.github/workflows/reusable_sanitizers.yml @@ -40,10 +40,6 @@ jobs: sudo apt-get update sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp - - - name: Set ptrace value for IPC test - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Configure build run: > ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} diff --git a/README.md b/README.md index b16f35ff64..5bd0b9b2f5 100644 --- a/README.md +++ b/README.md @@ -159,11 +159,12 @@ OS memory provider supports two types of memory mappings (set by the `visibility IPC API requires the `UMF_MEM_MAP_SHARED` memory `visibility` mode (`UMF_RESULT_ERROR_INVALID_ARGUMENT` is returned otherwise). -IPC API uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain -a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported since Linux 5.6). -Permission to duplicate another process's file descriptor is governed by a ptrace access mode -`PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using -the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: +IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain +a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. +Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check +(see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call +`prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. +Alternatively you can change the `ptrace_scope` globally in the system, e.g.: ```sh sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" @@ -194,16 +195,16 @@ Packages required for tests (Linux-only yet): A memory provider that provides memory from L0 device. -IPC API uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain -a duplicate of another process's file descriptor (`pidfd_getfd(2)` is supported since Linux 5.6). -Permission to duplicate another process's file descriptor is governed by a ptrace access mode -`PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using -the `/proc/sys/kernel/yama/ptrace_scope` interface in the following way: +IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain +a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. +Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check +(see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call +`prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. +Alternatively you can change the `ptrace_scope` globally in the system, e.g.: ```sh sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" ``` - ##### Requirements 1) Linux or Windows OS @@ -359,7 +360,7 @@ The memory used by the proxy memory allocator is mmap'ed: 1) with the `MAP_PRIVATE` flag by default or 2) with the `MAP_SHARED` flag if the `UMF_PROXY` environment variable contains one of two following strings: `page.disposition=shared-shm` or `page.disposition=shared-fd`. These two options differ in a mechanism used during IPC: - `page.disposition=shared-shm` - IPC uses the named shared memory. An SHM name is generated using the `umf_proxy_lib_shm_pid_$PID` pattern, where `$PID` is the PID of the process. It creates the `/dev/shm/umf_proxy_lib_shm_pid_$PID` file. - - `page.disposition=shared-fd` - IPC uses the file descriptor duplication. It requires using `pidfd_getfd(2)` to obtain a duplicate of another process's file descriptor. Permission to duplicate another process's file descriptor is governed by a ptrace access mode `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`) that can be changed using the `/proc/sys/kernel/yama/ptrace_scope` interface. `pidfd_getfd(2)` is supported since Linux 5.6. + - `page.disposition=shared-fd` - IPC API uses file descriptor duplication, which requires the `pidfd_getfd(2)` system call to obtain a duplicate of another process's file descriptor. This system call is supported since Linux 5.6. Required permission ("restricted ptrace") is governed by the `PTRACE_MODE_ATTACH_REALCREDS` check (see `ptrace(2)`). To allow file descriptor duplication in a binary that opens IPC handle, you can call `prctl(PR_SET_PTRACER, ...)` in the producer binary that gets the IPC handle. Alternatively you can change the `ptrace_scope` globally in the system, e.g.: `sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"`. **Size threshold** diff --git a/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh b/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh index 615271eebb..2eb9409daf 100755 --- a/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh +++ b/examples/ipc_ipcapi/ipc_ipcapi_anon_fd.sh @@ -1,5 +1,5 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -16,16 +16,8 @@ PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) # to obtain a duplicate of another process's file descriptor. # Permission to duplicate another process's file descriptor # is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) -# that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. -PTRACE_SCOPE_FILE="/proc/sys/kernel/yama/ptrace_scope" -VAL=0 -if [ -f $PTRACE_SCOPE_FILE ]; then - PTRACE_SCOPE_VAL=$(cat $PTRACE_SCOPE_FILE) - if [ $PTRACE_SCOPE_VAL -ne $VAL ]; then - echo "SKIP: ptrace_scope is not set to 0 (classic ptrace permissions) - skipping the test" - exit 125 # skip code defined in CMakeLists.txt - fi -fi +# In the producer binary used in this example prctl(PR_SET_PTRACER, getppid()) is used +# to allow consumer to duplicate file descriptor of producer. UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" diff --git a/examples/ipc_ipcapi/ipc_ipcapi_producer.c b/examples/ipc_ipcapi/ipc_ipcapi_producer.c index 4157e8284f..9082302ac9 100644 --- a/examples/ipc_ipcapi/ipc_ipcapi_producer.c +++ b/examples/ipc_ipcapi/ipc_ipcapi_producer.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -69,6 +70,21 @@ int main(int argc, char *argv[]) { int port = atoi(argv[1]); + // The prctl() function with PR_SET_PTRACER is used here to allow parent process and its children + // to ptrace the current process. This is necessary because UMF's memory providers on Linux (except CUDA) + // use the pidfd_getfd(2) system call to duplicate another process's file descriptor, which is + // governed by ptrace permissions. By default on Ubuntu /proc/sys/kernel/yama/ptrace_scope is + // set to 1 ("restricted ptrace"), which prevents pidfd_getfd from working unless ptrace_scope + // is set to 0. + // To overcome this limitation without requiring users to change the ptrace_scope + // setting (which requires root privileges), we use prctl() to allow the consumer process + // to copy producer's file descriptor, even when ptrace_scope is set to 1. + ret = prctl(PR_SET_PTRACER, getppid()); + if (ret == -1) { + perror("PR_SET_PTRACER may be not supported. prctl() call failed"); + goto err_end; + } + umf_memory_provider_handle_t OS_memory_provider = NULL; umf_os_memory_provider_params_handle_t os_params = NULL; enum umf_result_t umf_result; @@ -259,6 +275,7 @@ int main(int argc, char *argv[]) { err_destroy_OS_params: umfOsMemoryProviderParamsDestroy(os_params); +err_end: if (ret == 0) { fprintf(stderr, "[producer] Shutting down (status OK) ...\n"); } else if (ret == 1) { diff --git a/scripts/qemu/run-tests.sh b/scripts/qemu/run-tests.sh index 9d855590ba..341e2f9ab8 100755 --- a/scripts/qemu/run-tests.sh +++ b/scripts/qemu/run-tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -23,8 +23,6 @@ UMF_DIR=$(pwd) # Drop caches, restores free memory on NUMA nodes echo password | sudo sync; echo password | sudo sh -c "/usr/bin/echo 3 > /proc/sys/vm/drop_caches" -# Set ptrace value for IPC test -echo password | sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" numactl -H diff --git a/src/utils/utils_posix_common.c b/src/utils/utils_posix_common.c index 4a60cbb1f2..613b8ea41d 100644 --- a/src/utils/utils_posix_common.c +++ b/src/utils/utils_posix_common.c @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -91,9 +91,8 @@ umf_result_t utils_duplicate_fd(int pid, int fd_in, int *fd_out) { return UMF_RESULT_ERROR_NOT_SUPPORTED; #else // pidfd_getfd(2) is used to obtain a duplicate of another process's file descriptor. - // Permission to duplicate another process's file descriptor - // is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) - // that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. + // Calling prctl(PR_SET_PTRACER, getppid()) in a producer binary that creates IPC handle + // allows file descriptor duplication for parent process and its children. // pidfd_getfd(2) is supported since Linux 5.6 // pidfd_open(2) is supported since Linux 5.3 errno = 0; diff --git a/test/common/ipc_common.c b/test/common/ipc_common.c index 140927079b..1590dd3c45 100644 --- a/test/common/ipc_common.c +++ b/test/common/ipc_common.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024-2025 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -336,6 +337,12 @@ int run_producer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, int producer_socket = -1; char consumer_message[MSG_SIZE]; + ret = prctl(PR_SET_PTRACER, getppid()); + if (ret == -1) { + perror("PR_SET_PTRACER may be not supported. prctl() call failed"); + goto err_end; + } + // create OS memory provider umf_result = umfMemoryProviderCreate(provider_ops, provider_params, &provider); @@ -528,6 +535,7 @@ int run_producer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, err_umfMemoryProviderDestroy: umfMemoryProviderDestroy(provider); +err_end: if (ret == 0) { fprintf(stderr, "[producer] Shutting down (status OK) ...\n"); } else if (ret == 1) { diff --git a/test/ipc_os_prov_anon_fd.sh b/test/ipc_os_prov_anon_fd.sh index c5738e9893..a42d820a25 100755 --- a/test/ipc_os_prov_anon_fd.sh +++ b/test/ipc_os_prov_anon_fd.sh @@ -1,5 +1,5 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -12,21 +12,6 @@ set -e # port should be a number from the range <1024, 65535> PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) -# The ipc_os_prov_anon_fd example requires using pidfd_getfd(2) -# to obtain a duplicate of another process's file descriptor. -# Permission to duplicate another process's file descriptor -# is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) -# that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. -PTRACE_SCOPE_FILE="/proc/sys/kernel/yama/ptrace_scope" -VAL=0 -if [ -f $PTRACE_SCOPE_FILE ]; then - PTRACE_SCOPE_VAL=$(cat $PTRACE_SCOPE_FILE) - if [ $PTRACE_SCOPE_VAL -ne $VAL ]; then - echo "SKIP: ptrace_scope is not set to 0 (classic ptrace permissions) - skipping the test" - exit 125 # skip code defined in CMakeLists.txt - fi -fi - UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" echo "Starting ipc_os_prov_anon_fd CONSUMER on port $PORT ..." diff --git a/test/providers/ipc_level_zero_prov.sh b/test/providers/ipc_level_zero_prov.sh index d6bcef4f3a..4d29677257 100755 --- a/test/providers/ipc_level_zero_prov.sh +++ b/test/providers/ipc_level_zero_prov.sh @@ -1,5 +1,5 @@ # -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -12,21 +12,6 @@ set -e # port should be a number from the range <1024, 65535> PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) -# The ipc_level_zero_prov test requires using pidfd_getfd(2) -# to obtain a duplicate of another process's file descriptor. -# Permission to duplicate another process's file descriptor -# is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) -# that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. -PTRACE_SCOPE_FILE="/proc/sys/kernel/yama/ptrace_scope" -VAL=0 -if [ -f $PTRACE_SCOPE_FILE ]; then - PTRACE_SCOPE_VAL=$(cat $PTRACE_SCOPE_FILE) - if [ $PTRACE_SCOPE_VAL -ne $VAL ]; then - echo "SKIP: ptrace_scope is not set to 0 (classic ptrace permissions) - skipping the test" - exit 125 # skip code defined in CMakeLists.txt - fi -fi - UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" echo "Starting ipc_level_zero_prov CONSUMER on port $PORT ..." From 958aceb7cc905aa8dd9ff098d450c62bd6c5f203 Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Mon, 20 Jan 2025 11:47:25 +0100 Subject: [PATCH 81/82] Test static hwloc linking on macos --- .github/workflows/reusable_basic.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 25d33e2b3a..a106472c36 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -471,8 +471,12 @@ jobs: echo "$PATH" >> $GITHUB_PATH python3 -m pip install -r third_party/requirements.txt + - name: Install dependencies + run: brew install jemalloc tbb automake + - name: Install hwloc - run: brew install hwloc tbb automake + if: matrix.os == 'macos-14' + run: brew install hwloc - name: Get UMF version run: | @@ -492,6 +496,7 @@ jobs: -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON + ${{ matrix.os != 'macos-14' && '-DUMF_LINK_HWLOC_STATICALLY=ON' || '' }} - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(sysctl -n hw.logicalcpu) From 3ec8f5be721e18cc939ae0750db210be8dc2690e Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Mon, 20 Jan 2025 16:05:28 +0100 Subject: [PATCH 82/82] Add dependent macOS libraries for hwloc build --- src/CMakeLists.txt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fe05ef0b7b..c0072be7e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -182,6 +182,18 @@ add_dependencies(umf coarse) if(UMF_LINK_HWLOC_STATICALLY) add_dependencies(umf ${UMF_HWLOC_NAME}) + # On Darwin, link with the IOKit and Foundation frameworks, if they are + # available in the system. This is to comply with hwloc which links these, + # if available. There is no option to disable these frameworks on Darwin + # hwloc builds. + if(MACOSX) + find_library(IOKIT_LIBRARY IOKit) + find_library(FOUNDATION_LIBRARY Foundation) + if(IOKIT_LIBRARY OR FOUNDATION_LIBRARY) + target_link_libraries(umf PRIVATE ${IOKIT_LIBRARY} + ${FOUNDATION_LIBRARY}) + endif() + endif() endif() if(NOT WINDOWS AND UMF_POOL_JEMALLOC_ENABLED)