From 88cabc0552c6b52829c9062bac7bf69f12e907d1 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 7 Jan 2022 16:08:39 -0600 Subject: [PATCH 01/16] Initial integration of the Hermes VFD --- CMakeLists.txt | 1 + adapter/CMakeLists.txt | 8 + adapter/vfd/CMakeLists.txt | 235 +++++++ adapter/vfd/H5FDhermes.c | 984 +++++++++++++++++++++++++++++ adapter/vfd/H5FDhermes.h | 37 ++ adapter/vfd/H5FDhermes_err.h | 199 ++++++ adapter/vfd/README.md | 50 ++ adapter/vfd/test/CMakeLists.txt | 48 ++ adapter/vfd/test/hermes.conf | 112 ++++ adapter/vfd/test/hermes_driver.c | 130 ++++ adapter/vfd/test/hermes_env.c | 124 ++++ adapter/vfd/test/hermes_set_fapl.c | 125 ++++ wrapper/CMakeLists.txt | 2 +- wrapper/hermes_wrapper.cpp | 4 +- wrapper/hermes_wrapper.h | 4 +- 15 files changed, 2058 insertions(+), 5 deletions(-) create mode 100644 adapter/vfd/CMakeLists.txt create mode 100644 adapter/vfd/H5FDhermes.c create mode 100644 adapter/vfd/H5FDhermes.h create mode 100644 adapter/vfd/H5FDhermes_err.h create mode 100644 adapter/vfd/README.md create mode 100644 adapter/vfd/test/CMakeLists.txt create mode 100644 adapter/vfd/test/hermes.conf create mode 100644 adapter/vfd/test/hermes_driver.c create mode 100644 adapter/vfd/test/hermes_env.c create mode 100644 adapter/vfd/test/hermes_set_fapl.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 21c2cb287..11f77120f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,7 @@ option(HERMES_BUILD_BENCHMARKS "Build the Hermes benchmark suite." OFF) option(HERMES_ENABLE_TIMING "Turn on timing of selected functions." OFF) option(HERMES_ENABLE_COVERAGE "Enable code coverage." OFF) option(HERMES_ENABLE_ADAPTERS "Enable hermes adapters." ON) +option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF) option(HERMES_ENABLE_WRAPPER "Enable hermes C API wrapper." ON) option(HERMES_INSTALL_TESTS "Enable installation of tests." OFF) # Calculate code coverage with debug mode diff --git a/adapter/CMakeLists.txt b/adapter/CMakeLists.txt index e2bbbf200..f00c3aa73 100644 --- a/adapter/CMakeLists.txt +++ b/adapter/CMakeLists.txt @@ -7,6 +7,14 @@ add_subdirectory(stdio) add_subdirectory(posix) add_subdirectory(mpiio) +if(HERMES_ENABLE_VFD) + if(HERMES_ENABLE_WRAPPER) + add_subdirectory(vfd) + else() + message(FATAL_ERROR "The Hermes VFD requires HERMES_ENABLE_WRAPPER=ON") + endif() +endif() + if(BUILD_TESTING) enable_testing() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) diff --git a/adapter/vfd/CMakeLists.txt b/adapter/vfd/CMakeLists.txt new file mode 100644 index 000000000..752db721b --- /dev/null +++ b/adapter/vfd/CMakeLists.txt @@ -0,0 +1,235 @@ +# CMakeLists files in this project can +# refer to the root source directory of the project as ${HDF5_HERMES_SOURCE_DIR} and +# to the root binary directory of the project as ${HDF5_HERMES_BINARY_DIR}. + +project(HDF5_HERMES_VFD) + +# include_directories(${PROJECT_BINARY_DIR}) + +#------------------------------------------------------------------------------ +# Version information +#------------------------------------------------------------------------------ +set(HDF5_HERMES_VFD_VERSION_MAJOR "0") +set(HDF5_HERMES_VFD_VERSION_MINOR "1") +set(HDF5_HERMES_VFD_VERSION_PATCH "0") +set(HDF5_HERMES_VFD_PACKAGE "hdf5_hermes_vfd") +set(HDF5_HERMES_VFD_PACKAGE_NAME "HDF5_HERMES_VFD") +set(HDF5_HERMES_VFD_PACKAGE_VERSION "${HDF5_HERMES_VFD_VERSION_MAJOR}.${HDF5_HERMES_VFD_VERSION_MINOR}.${HDF5_HERMES_VFD_VERSION_PATCH}") +set(HDF5_HERMES_VFD_PACKAGE_VERSION_MAJOR "${HDF5_HERMES_VFD_VERSION_MAJOR}.${HDF5_HERMES_VFD_VERSION_MINOR}") +set(HDF5_HERMES_VFD_PACKAGE_VERSION_MINOR "${HDF5_HERMES_VFD_VERSION_PATCH}") +set(HDF5_HERMES_VFD_PACKAGE_STRING "${HDF5_HERMES_VFD_PACKAGE_NAME} ${HDF5_HERMES_VFD_PACKAGE_VERSION}") +set(HDF5_HERMES_VFD_PACKAGE_TARNAME "${HDF5_HERMES_VFD_PACKAGE}") + +# Dynamically loaded VFDs must be shared libraries +set(HDF5_HERMES_VFD_LIBTYPE SHARED) + +#----------------------------------------------------------------------------- +# Targets built within this project are exported at Install time for use +# by other projects. +#----------------------------------------------------------------------------- +if(NOT HDF5_HERMES_VFD_EXPORTED_TARGETS) + set(HDF5_HERMES_VFD_EXPORTED_TARGETS "${HDF5_HERMES_VFD_PACKAGE}-targets") +endif() + +#------------------------------------------------------------------------------ +# External dependencies +#------------------------------------------------------------------------------ +#HDF5 +find_package(HDF5 1.13.0 NO_MODULE NAMES hdf5 COMPONENTS C shared) +if(HDF5_FOUND) + set(HDF5_C_SHARED_LIBRARY hdf5-shared) +# if(NOT TARGET ${HDF5_C_SHARED_LIBRARY}) +# message(FATAL_ERROR "Could not find hdf5 shared target, please make " +#"sure that HDF5 has ben compiled with shared libraries enabled.") +# endif() + set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ${HDF5_INCLUDE_DIR} + ) + set(HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} + ${HDF5_C_SHARED_LIBRARY} + ) +else() +# Allow for HDF5 autotools builds + find_package(HDF5 1.13.0 MODULE REQUIRED) + if(HDF5_FOUND) + set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ${HDF5_INCLUDE_DIRS} + ) + set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_LIBRARIES} + ) + else() + message(FATAL_ERROR "Could not find HDF5, please check HDF5_DIR.") + endif() +endif() + + +set(HDF5_HERMES_VFD_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.h + ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.c + ) + +add_library(hdf5_hermes_vfd ${HDF5_HERMES_VFD_SRCS}) +target_include_directories(hdf5_hermes_vfd PRIVATE ${CMAKE_SOURCE_DIR}/wrapper) +target_include_directories(hdf5_hermes_vfd + SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ) + +target_link_libraries(hdf5_hermes_vfd + hermes_wrapper + ${HDF5_HERMES_VFD_EXPORTED_LIBS} + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} + ) + +set(HDF5_HERMES_VFD_EXPORTED_LIBS hdf5_hermes_vfd ${HDF5_HERMES_VFD_EXPORTED_LIBS} PARENT_SCOPE) + +#----------------------------------------------------------------------------- +# Specify project header files to be installed +#----------------------------------------------------------------------------- +set(HDF5_HERMES_VFD_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR}/H5FDhermes.h +) + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +install( + FILES + ${HDF5_HERMES_VFD_HEADERS} + DESTINATION + ${HERMES_INSTALL_INCLUDE_DIR} + COMPONENT + headers +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install +#----------------------------------------------------------------------------- +install( + TARGETS + hdf5_hermes_vfd + EXPORT + ${HDF5_HERMES_VFD_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) + +#----------------------------------------------------------------------------- +# Add Target(s) to CMake Install for import into other projects +#----------------------------------------------------------------------------- +install( + EXPORT + ${HDF5_HERMES_VFD_EXPORTED_TARGETS} + DESTINATION + ${HERMES_INSTALL_DATA_DIR}/cmake/hdf5_hermes_vfd + FILE + ${HDF5_HERMES_VFD_EXPORTED_TARGETS}.cmake +) + +#----------------------------------------------------------------------------- +# Export all exported targets to the build tree for use by parent project + +if(NOT HDF5_HERMES_VFD_EXTERNALLY_CONFIGURED) + export( + TARGETS + ${HDF5_HERMES_VFD_EXPORTED_LIBS} + FILE + ${HDF5_HERMES_VFD_EXPORTED_TARGETS}.cmake + ) +endif() + +#------------------------------------------------------------------------------ +# Set variables for parent scope +#------------------------------------------------------------------------------ + +# Pkg-config configuration +if(CMAKE_BUILD_TYPE) + string(TOLOWER ${CMAKE_BUILD_TYPE} lower_cmake_build_type) +endif() + +# Hermes VFD package dependencies +foreach(pkg_dep ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES}) + set(HDF5_HERMES_VFD_PKG_DEPENDENCIES ${HDF5_HERMES_VFD_PKG_DEPENDENCIES} ${pkg_dep}) +endforeach() +set(HDF5_HERMES_VFD_PKG_DEPENDENCIES ${HDF5_HERMES_VFD_PKG_DEPENDENCIES} PARENT_SCOPE) + +# Hermes VFD private library dependencies +foreach(exported_lib ${HDF5_HERMES_VFD_EXPORTED_LIBS}) + if(lower_cmake_build_type MATCHES "debug") + get_target_property(HDF5_HERMES_VFD_LIBRARY ${exported_lib} DEBUG_OUTPUT_NAME) + else() + get_target_property(HDF5_HERMES_VFD_LIBRARY ${exported_lib} RELEASE_OUTPUT_NAME) + endif() + set(HDF5_HERMES_VFD_LIBRARIES "${HDF5_HERMES_VFD_LIBRARIES} -l${HDF5_HERMES_VFD_LIBRARY}") +endforeach() +set(HDF5_HERMES_VFD_LIBRARIES ${HDF5_HERMES_VFD_LIBRARIES} PARENT_SCOPE) +# Hermes VFD external library dependencies +# Need to generate -lib if not already passed +set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_LIB_DEPENDENCIES} + PARENT_SCOPE +) +foreach(lib_dep ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}) + # get library name + get_filename_component(lib_name ${lib_dep} NAME_WE) + if(lib_name MATCHES "^-l") + # lib_name found is -lxxx + set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST} ${lib_name}) + else() + # lib_name is /path/to/lib so get library path and name + get_filename_component(lib_path ${lib_dep} PATH) + string(REGEX REPLACE "^lib" "" lib_name ${lib_name}) + set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST} -L${lib_path} -l${lib_name}) + endif() +endforeach() +if(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST) + list(REMOVE_DUPLICATES HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST) +endif() +foreach(lib_dep ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES_LIST}) + set(HDF5_HERMES_VFD_LIB_DEPENDENCIES "${HDF5_HERMES_VFD_LIB_DEPENDENCIES} ${lib_dep}") +endforeach() +set(HDF5_HERMES_VFD_LIB_DEPENDENCIES ${HDF5_HERMES_VFD_LIB_DEPENDENCIES} PARENT_SCOPE) + +# External include dependencies +set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) +if(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES) + list(REMOVE_DUPLICATES HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES) +endif() +foreach(inc_dep ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES}) + set(HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES "${HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES} -I${inc_dep}") +endforeach() +set(HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES ${HDF5_HERMES_VFD_INCLUDE_DEPENDENCIES} PARENT_SCOPE) + +set(HDF5_HERMES_VFD_INCLUDES_BUILD_TIME + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) + +set(HDF5_HERMES_VFD_INCLUDES_INSTALL_TIME + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + PARENT_SCOPE +) +#----------------------------------------------------------------------------- +# Testing +#----------------------------------------------------------------------------- +# option(BUILD_TESTING "Build testing." ON) +# if(NOT HDF5_HERMES_VFD_EXTERNALLY_CONFIGURED AND BUILD_TESTING) +# enable_testing() +# include(CTest) +# add_subdirectory(test) +# endif() diff --git a/adapter/vfd/H5FDhermes.c b/adapter/vfd/H5FDhermes.c new file mode 100644 index 000000000..293cdc8fe --- /dev/null +++ b/adapter/vfd/H5FDhermes.c @@ -0,0 +1,984 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Kimmy Mu + * March 2021 + * + * Purpose: The hermes file driver using only the HDF5 public API + * and buffer datasets in Hermes buffering systems with + * multiple storage tiers. + */ + +#include +#include +#include + +#include +#include +#include +#include + +/* HDF5 header for dynamic plugin loading */ +#include "H5PLextern.h" + +#include "H5FDhermes.h" /* Hermes file driver */ +#include "H5FDhermes_err.h" /* error handling */ + +/* Necessary hermes headers */ +#include "hermes_wrapper.h" + +#define H5FD_HERMES (H5FD_hermes_init()) + +/* HDF5 doesn't currently have a driver init callback. Use + * macro to initialize driver if loaded as a plugin. + */ +#define H5FD_HERMES_INIT \ + do { \ + if (H5FD_HERMES_g < 0) \ + H5FD_HERMES_g = H5FD_HERMES; \ + } while (0) + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_HERMES_g = H5I_INVALID_HID; + +/* Identifiers for HDF5's error API */ +hid_t H5FDhermes_err_stack_g = H5I_INVALID_HID; +hid_t H5FDhermes_err_class_g = H5I_INVALID_HID; + +/* Whether Hermes is initialized */ +static htri_t hermes_initialized = FAIL; + +/* File operations */ +#define OP_UNKNOWN 0 +#define OP_READ 1 +#define OP_WRITE 2 + +/* POSIX I/O mode used as the third parameter to open/_open + * when creating a new file (O_CREAT is set). */ +#if defined(H5_HAVE_WIN32_API) +#define H5FD_HERMES_POSIX_CREATE_MODE_RW (_S_IREAD | _S_IWRITE) +#else +#define H5FD_HERMES_POSIX_CREATE_MODE_RW 0666 +#endif + +/* Define length of blob name, which is converted from page index */ +#define LEN_BLOB_NAME 10 + +#define BIT_SIZE_OF_UNSIGNED (sizeof(uint)*8) + +/* kHermesConf env variable is used to define path to kHermesConf in adapters. + * This is used for initialization of Hermes. */ +const char *kHermesConf = "HERMES_CONF"; + +/* The description of bit representation of blobs in Hermes buffering system. */ +typedef struct bitv_t { + uint *blobs; + size_t capacity; + size_t end_pos; +} bitv_t; + +/* The description of a file/bucket belonging to this driver. */ +typedef struct H5FD_hermes_t { + H5FD_t pub; /* public stuff, must be first */ + haddr_t eoa; /* end of allocated region */ + haddr_t eof; /* end of file; current file size */ + haddr_t pos; /* current file I/O position */ + int op; /* last operation */ + hbool_t persistence; /* write to file name on close */ + int fd; /* the filesystem file descriptor */ + size_t buf_size; + char *bktname; /* Copy of file name from open operation */ + BucketClass *bkt_handle; + int ref_count; + unsigned char *page_buf; + bitv_t blob_in_bucket; +} H5FD_hermes_t; + +/* Driver-specific file access properties */ +typedef struct H5FD_hermes_fapl_t { + hbool_t persistence; /* write to file name on flush */ + size_t page_size; /* page size */ +} H5FD_hermes_fapl_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1 << (8 * sizeof(off_t) - 1)) - 1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF == (A) || ((A) & ~(haddr_t)MAXADDR)) +#define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) +#define REGION_OVERFLOW(A, Z) \ + (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || HADDR_UNDEF == (A) + (Z) || \ + (off_t)((A) + (Z)) < (off_t)(A)) + +/* Prototypes */ +static herr_t H5FD__hermes_term(void); +static herr_t H5FD__hermes_fapl_free(void *_fa); +static H5FD_t *H5FD__hermes_open(const char *name, unsigned flags, + hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__hermes_close(H5FD_t *_file); +static int H5FD__hermes_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD__hermes_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD__hermes_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__hermes_set_eoa(H5FD_t *_file, H5FD_mem_t type, + haddr_t addr); +static haddr_t H5FD__hermes_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__hermes_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD__hermes_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); + +static const H5FD_class_t H5FD_hermes_g = { + H5FD_HERMES_VALUE, /* value */ + H5FD_HERMES_NAME, /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_STRONG, /* fc_degree */ + H5FD__hermes_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_hermes_fapl_t),/* fapl_size */ + NULL, /* fapl_get */ + NULL, /* fapl_copy */ + H5FD__hermes_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__hermes_open, /* open */ + H5FD__hermes_close, /* close */ + H5FD__hermes_cmp, /* cmp */ + H5FD__hermes_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__hermes_get_eoa, /* get_eoa */ + H5FD__hermes_set_eoa, /* set_eoa */ + H5FD__hermes_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD__hermes_read, /* read */ + H5FD__hermes_write, /* write */ + NULL, /* flush */ + NULL, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + NULL, /* del */ + NULL, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Check if the blob at POS is set */ +static bool check_blob(bitv_t *bits, size_t bit_pos) { + bool result = false; + + if (bit_pos >= bits->capacity) + return false; + + size_t unit_pos = bit_pos / BIT_SIZE_OF_UNSIGNED; + size_t blob_pos_in_unit = bit_pos % BIT_SIZE_OF_UNSIGNED; + result = bits->blobs[unit_pos] & (1 << blob_pos_in_unit); + + return result; +} + +/* Set the bit at POS and reallocate 2*capacity for blobs as needed */ +static void set_blob(bitv_t *bits, size_t bit_pos) { + if (bit_pos >= bits->capacity) { + size_t current_units = bits->capacity/BIT_SIZE_OF_UNSIGNED; + size_t need_units = bit_pos/BIT_SIZE_OF_UNSIGNED + 1; + bits->capacity = need_units * BIT_SIZE_OF_UNSIGNED * 2; + bits->blobs = realloc(bits->blobs, bits->capacity); + memset(&bits->blobs[current_units], 0, + sizeof(uint)*(need_units*2-current_units+1)); + } + + size_t unit_pos = bit_pos / BIT_SIZE_OF_UNSIGNED; + size_t blob_pos_in_unit = bit_pos % BIT_SIZE_OF_UNSIGNED; + bits->blobs[unit_pos] |= 1 << blob_pos_in_unit; + if (bit_pos > bits->end_pos) + bits->end_pos = bit_pos; +} + +/*------------------------------------------------------------------------- + * Function: H5FD_hermes_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the hermes driver + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_hermes_init(void) { + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + /* Initialize error reporting */ + if ((H5FDhermes_err_stack_g = H5Ecreate_stack()) < 0) + H5FD_HERMES_GOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, + "can't create HDF5 error stack"); + if ((H5FDhermes_err_class_g = H5Eregister_class(H5FD_HERMES_ERR_CLS_NAME, + H5FD_HERMES_ERR_LIB_NAME, + H5FD_HERMES_ERR_VER)) < 0) + H5FD_HERMES_GOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, + "can't register error class with HDF5 error API"); + + if (H5I_VFL != H5Iget_type(H5FD_HERMES_g)) + H5FD_HERMES_g = H5FDregister(&H5FD_hermes_g); + + /* Set return value */ + ret_value = H5FD_HERMES_g; + +done: + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD_hermes_init() */ + +/*--------------------------------------------------------------------------- + * Function: H5FD__hermes_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * + *--------------------------------------------------------------------------- + */ +static herr_t +H5FD__hermes_term(void) { + herr_t ret_value = SUCCEED; + + if ((H5OPEN hermes_initialized) == TRUE) { + HermesFinalize(); + hermes_initialized = FALSE; + } + + /* Unregister from HDF5 error API */ + if (H5FDhermes_err_class_g >= 0) { + if (H5Eunregister_class(H5FDhermes_err_class_g) < 0) + H5FD_HERMES_GOTO_ERROR( + H5E_VFL, H5E_CLOSEERROR, FAIL, + "can't unregister error class from HDF5 error API"); + + /* Print the current error stack before destroying it */ + PRINT_ERROR_STACK; + + /* Destroy the error stack */ + if (H5Eclose_stack(H5FDhermes_err_stack_g) < 0) { + H5FD_HERMES_GOTO_ERROR(H5E_VFL, H5E_CLOSEERROR, FAIL, + "can't close HDF5 error stack"); + PRINT_ERROR_STACK; + } /* end if */ + + H5FDhermes_err_stack_g = H5I_INVALID_HID; + H5FDhermes_err_class_g = H5I_INVALID_HID; + } + + /* Reset VFL ID */ + H5FD_HERMES_g = H5I_INVALID_HID; + +done: + H5FD_HERMES_FUNC_LEAVE_API; +} /* end H5FD__hermes_term() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_hermes + * + * Purpose: Modify the file access property list to use the H5FD_HERMES + * driver defined in this source file. There are no driver + * specific properties. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_hermes(hid_t fapl_id, hbool_t persistence, size_t page_size) { + H5FD_hermes_fapl_t fa; /* Hermes VFD info */ + herr_t ret_value = SUCCEED; /* Return value */ + + /* Check argument */ + if (H5I_GENPROP_LST != H5Iget_type(fapl_id) || + TRUE != H5Pisa_class(fapl_id, H5P_FILE_ACCESS)) { + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, + "not a file access property list"); + } + + /* Set VFD info values */ + memset(&fa, 0, sizeof(H5FD_hermes_fapl_t)); + fa.persistence = persistence; + fa.page_size = page_size; + + /* Set the property values & the driver for the FAPL */ + if (H5Pset_driver(fapl_id, H5FD_HERMES, &fa) < 0) { + H5FD_HERMES_GOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, + "can't set Hermes VFD as driver"); + } + +done: + H5FD_HERMES_FUNC_LEAVE_API; +} /* end H5Pset_fapl_hermes() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_fapl_free + * + * Purpose: Frees the family-specific file access properties. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD__hermes_fapl_free(void *_fa) { + H5FD_hermes_fapl_t *fa = (H5FD_hermes_fapl_t *)_fa; + herr_t ret_value = SUCCEED; /* Return value */ + + free(fa); + + H5FD_HERMES_FUNC_LEAVE; +} + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_open + * + * Purpose: Create and/or opens a bucket in Hermes. + * + * Return: Success: A pointer to a new bucket data structure. + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__hermes_open(const char *name, unsigned flags, hid_t fapl_id, + haddr_t maxaddr) { + H5FD_hermes_t *file = NULL; /* hermes VFD info */ + int fd = -1; /* File descriptor */ + int o_flags; /* Flags for open() call */ + struct stat sb; + const H5FD_hermes_fapl_t *fa = NULL; + H5FD_hermes_fapl_t new_fa; + char *hermes_config = NULL; + H5FD_t *ret_value = NULL; /* Return value */ + + /* Sanity check on file offsets */ + assert(sizeof(off_t) >= sizeof(size_t)); + + H5FD_HERMES_INIT; + + /* Check arguments */ + if (!name || !*name) + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name"); + if (0 == maxaddr || HADDR_UNDEF == maxaddr) + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr"); + if (ADDR_OVERFLOW(maxaddr)) + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr"); + + /* Get the driver specific information */ + H5E_BEGIN_TRY { + fa = H5Pget_driver_info(fapl_id); + } + H5E_END_TRY; + if (!fa || (H5P_FILE_ACCESS_DEFAULT == fapl_id)) { + ssize_t config_str_len = 0; + char config_str_buf[128]; + if ((config_str_len = + H5Pget_driver_config_str(fapl_id, config_str_buf, 128)) < 0) { + printf("H5Pget_driver_config_str() error\n"); + } + char *saveptr = NULL; + char* token = strtok_r(config_str_buf, " ", &saveptr); + if (!strcmp(token, "true") || !strcmp(token, "TRUE") || + !strcmp(token, "True")) { + new_fa.persistence = true; + } + token = strtok_r(0, " ", &saveptr); + sscanf(token, "%zu", &(new_fa.page_size)); + fa = &new_fa; + } + + /* Initialize Hermes */ + if ((H5OPEN hermes_initialized) == FAIL) { + hermes_config = getenv(kHermesConf); + if (HermesInitHermes(hermes_config) < 0) { + H5FD_HERMES_GOTO_ERROR(H5E_SYM, H5E_UNINITIALIZED, NULL, + "Hermes initialization failed"); + } else { + hermes_initialized = TRUE; + } + } + + /* Create the new file struct */ + if (NULL == (file = calloc(1, sizeof(H5FD_hermes_t)))) { + H5FD_HERMES_GOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "unable to allocate file struct"); + } + + if (name && *name) { + file->bktname = strdup(name); + } + file->persistence = fa->persistence; + file->fd = -1; + file->bkt_handle = HermesBucketCreate(name); + file->page_buf = malloc(fa->page_size); + file->buf_size = fa->page_size; + file->ref_count = 1; + file->op = OP_UNKNOWN; + file->blob_in_bucket.capacity = BIT_SIZE_OF_UNSIGNED; + file->blob_in_bucket.blobs = (uint *)calloc(1, sizeof(uint)); + file->blob_in_bucket.end_pos = 0; + + if (fa->persistence) { + /* Build the open flags */ + o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY; + if (H5F_ACC_TRUNC & flags) + o_flags |= O_TRUNC; + if (H5F_ACC_CREAT & flags) + o_flags |= O_CREAT; + if (H5F_ACC_EXCL & flags) + o_flags |= O_EXCL; + + /* Open the file */ + if ((fd = open(name, o_flags, H5FD_HERMES_POSIX_CREATE_MODE_RW)) < 0) { + int myerrno = errno; + H5FD_HERMES_GOTO_ERROR( + H5E_FILE, H5E_CANTOPENFILE, NULL, + "unable to open file: name = '%s', errno = %d, error message = '%s'," + "flags = %x, o_flags = %x", name, myerrno, strerror(myerrno), flags, + (unsigned)o_flags); + } + + if (fstat(fd, &sb) < 0) { + H5FD_HERMES_SYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, + "unable to fstat file"); + } + + /* FIXME: Possible overflow! */ + file->eof = (haddr_t)sb.st_size; + file->fd = fd; + } + + /* Set return value */ + ret_value = (H5FD_t *)file; + +done: + if (NULL == ret_value) { + if (fd >= 0) + close(fd); + if (file) { + HermesBucketDestroy(file->bkt_handle); + free(file->blob_in_bucket.blobs); + free(file); + } + } /* end if */ + + H5FD_HERMES_FUNC_LEAVE_API; +} /* end H5FD__hermes_open() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_close + * + * Purpose: Closes an HDF5 file. + * + * Return: Success: SUCCEED + * Failure: FAIL, file not closed. + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD__hermes_close(H5FD_t *_file) { + H5FD_hermes_t *file = (H5FD_hermes_t *)_file; + size_t blob_size = file->buf_size; + size_t i; + herr_t ret_value = SUCCEED; /* Return value */ + + /* Sanity check */ + assert(file); + + if (file->persistence) { + if (file->op == OP_WRITE) { + /* TODO: if there is user blobk, the logic is not working, + including offset, but not 0 */ + for (i = 0; i <= file->blob_in_bucket.end_pos; i++) { + /* Check if this blob exists */ + bool blob_exists = check_blob(&file->blob_in_bucket, i); + if (!blob_exists) + continue; + + char i_blob[LEN_BLOB_NAME]; + snprintf(i_blob, sizeof(i_blob), "%zu\n", i); + /* Read blob back */ + HermesBucketGet(file->bkt_handle, i_blob, blob_size, file->page_buf); + ssize_t bytes_wrote; + if (i == file->blob_in_bucket.end_pos) { + size_t bytes_in = file->eof%blob_size; + if (bytes_in == 0) + bytes_in = blob_size; + bytes_wrote = pwrite(file->fd, file->page_buf, + bytes_in, i*blob_size); + assert(bytes_wrote == bytes_in); + } else { + bytes_wrote = pwrite(file->fd, file->page_buf, + blob_size, i*blob_size); + assert(bytes_wrote == blob_size); + } + } + } + if (close(file->fd) < 0) + H5FD_HERMES_SYS_GOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, + "unable to close file"); + } + if (file->ref_count == 1) + HermesBucketDestroy(file->bkt_handle); + else + HermesBucketClose(file->bkt_handle); + + /* Release the file info */ + free(file->blob_in_bucket.blobs); + free(file); + +done: + H5FD_HERMES_FUNC_LEAVE_API; +} /* end H5FD__hermes_close() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_cmp + * + * Purpose: Compares two buckets belonging to this driver using an + * arbitrary (but consistent) ordering. + * + * Return: Success: A value like strcmp() + * Failure: never fails (arguments were checked by the + * caller). + * + *------------------------------------------------------------------------- + */ +static int H5FD__hermes_cmp(const H5FD_t *_f1, const H5FD_t *_f2) { + const H5FD_hermes_t *f1 = (const H5FD_hermes_t *)_f1; + const H5FD_hermes_t *f2 = (const H5FD_hermes_t *)_f2; + int ret_value = 0; + + ret_value = strcmp(f1->bktname, f2->bktname); + + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD__hermes_cmp() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Return: SUCCEED (Can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD__hermes_query(const H5FD_t *_file, + unsigned long *flags /* out */) { + /* Set the VFL feature flags that this driver supports */ + /* Notice: the Mirror VFD Writer currently uses only the hermes driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * these feature flags as its own. Any modifications made here must be + * reflected in H5FDmirror.c + * -- JOS 2020-01-13 + */ + herr_t ret_value = SUCCEED; + + if (flags) { + *flags = 0; + } /* end if */ + + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD__hermes_query() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_get_eoa + * + * Purpose: Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Return: The end-of-address marker. + * + *------------------------------------------------------------------------- + */ +static haddr_t H5FD__hermes_get_eoa(const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) { + haddr_t ret_value = HADDR_UNDEF; + + const H5FD_hermes_t *file = (const H5FD_hermes_t *)_file; + + ret_value = file->eoa; + + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD__hermes_get_eoa() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED (Can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD__hermes_set_eoa(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t addr) { + herr_t ret_value = SUCCEED; + + H5FD_hermes_t *file = (H5FD_hermes_t *)_file; + + file->eoa = addr; + + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD__hermes_set_eoa() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_get_eof + * + * Purpose: Returns the end-of-file marker, which is the greater of + * either the filesystem end-of-file or the HDF5 end-of-address + * markers. + * + * Return: End of file address, the first address past the end of the + * "file", either the filesystem file or the HDF5 file. + * + *------------------------------------------------------------------------- + */ +static haddr_t H5FD__hermes_get_eof(const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) { + haddr_t ret_value = HADDR_UNDEF; + + const H5FD_hermes_t *file = (const H5FD_hermes_t *)_file; + + ret_value = file->eof; + + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD__hermes_get_eof() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_read + * + * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in + * DXPL_ID. Determine the number of file pages affected by this + * call from ADDR and SIZE. Utilize transfer buffer PAGE_BUF to + * read the data from Blobs. Exercise care for the first and last + * pages to prevent overwriting existing data. + * + * Return: Success: SUCCEED. Result is stored in caller-supplied + * buffer BUF. + * Failure: FAIL, Contents of buffer BUF are undefined. + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD__hermes_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, + size_t size, void *buf /*out*/) { + H5FD_hermes_t *file = (H5FD_hermes_t *)_file; + size_t num_pages; /* Number of pages of transfer buffer */ + size_t start_page_index; /* First page index of tranfer buffer */ + size_t end_page_index; /* End page index of tranfer buffer */ + size_t transfer_size = 0; + size_t blob_size = file->buf_size; + size_t k; + haddr_t addr_end = addr+size-1; + herr_t ret_value = SUCCEED; /* Return value */ + + assert(file && file->pub.cls); + assert(buf); + + /* Check for overflow conditions */ + if (HADDR_UNDEF == addr) { + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "addr undefined, addr = %llu", + (unsigned long long)addr); + } + if (REGION_OVERFLOW(addr, size)) { + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, + "addr overflow, addr = %llu", + (unsigned long long)addr); + } + if (NULL == file->page_buf) { + H5FD_HERMES_GOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, FAIL, + "transfer buffer not initialized"); + } + + /* Check easy cases */ + if (0 == size) + return 0; + if ((haddr_t)addr >= file->eof) { + memset(buf, 0, size); + return 0; + } + + start_page_index = addr/blob_size; + end_page_index = addr_end/blob_size; + num_pages = end_page_index - start_page_index + 1; + + for (k = start_page_index; k <= end_page_index; ++k) { + size_t bytes_in; + char k_blob[LEN_BLOB_NAME]; + snprintf(k_blob, sizeof(k_blob), "%zu\n", k); + /* Check if this blob exists */ + bool blob_exists = check_blob(&file->blob_in_bucket, k); + + /* Check if addr is in the range of (k*blob_size, (k+1)*blob_size) */ + /* NOTE: The range does NOT include the start address of page k, + but includes the end address of page k */ + if (addr > k*blob_size && addr < (k+1)*blob_size) { + /* Calculate the starting address of transfer buffer update within page + * k */ + size_t offset = addr - k*blob_size; + assert(offset > 0); + + if (addr_end <= (k+1)*blob_size-1) + bytes_in = size; + else + bytes_in = (k+1)*blob_size-addr; + + if (!blob_exists) { + size_t bytes_copy; + if (file->eof < (k+1)*blob_size-1) + bytes_copy = file->eof-k*blob_size; + else + bytes_copy = blob_size; + + size_t bytes_read = pread(file->fd, file->page_buf, bytes_copy, + k*blob_size); + if (bytes_read != bytes_copy) + H5FD_HERMES_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "pread failed"); + memcpy(buf, file->page_buf+offset, bytes_in); + + /* Write Blob k to Hermes buffering system */ + HermesBucketPut(file->bkt_handle, k_blob, file->page_buf, blob_size); + set_blob(&file->blob_in_bucket, k); + } else { + /* Read blob back to transfer buffer */ + HermesBucketGet(file->bkt_handle, k_blob, blob_size, file->page_buf); + + memcpy(buf, file->page_buf+offset, bytes_in); + } + transfer_size += bytes_in; + /* Check if addr_end is in the range of [k*blob_size, + * (k+1)*blob_size-1) */ + /* NOTE: The range includes the start address of page k, + but does NOT include the end address of page k */ + } else if (addr_end >= k*blob_size && addr_end < (k+1)*blob_size-1) { + bytes_in = addr_end-k*blob_size+1; + if (!blob_exists) { + size_t bytes_copy; + if (file->eof < (k+1)*blob_size-1) + bytes_copy = file->eof-k*blob_size; + else + bytes_copy = blob_size; + + ssize_t bytes_read = pread(file->fd, file->page_buf, bytes_copy, + k*blob_size); + if (bytes_read != bytes_copy) + H5FD_HERMES_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "pread failed"); + + memcpy(buf+transfer_size, file->page_buf, bytes_in); + + /* Write Blob k to Hermes buffering system */ + HermesBucketPut(file->bkt_handle, k_blob, file->page_buf, blob_size); + set_blob(&file->blob_in_bucket, k); + } else { + /* Read blob back to transfer buffer */ + HermesBucketGet(file->bkt_handle, k_blob, blob_size, file->page_buf); + + /* Update transfer buffer */ + memcpy(buf+transfer_size, file->page_buf, bytes_in); + } + transfer_size += bytes_in; + /* Page/Blob k is within the range of (addr, addr+size) */ + /* addr <= k*blob_size && addr_end >= (k+1)*blob_size-1 */ + } else { + if (!blob_exists) { + ssize_t bytes_read = pread(file->fd, buf+transfer_size, blob_size, + addr+transfer_size); + if (bytes_read != blob_size) + H5FD_HERMES_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "pread failed"); + + /* Write Blob k to Hermes buffering system */ + HermesBucketPut(file->bkt_handle, k_blob, buf+transfer_size, blob_size); + set_blob(&file->blob_in_bucket, k); + } else { + /* Read blob back directly */ + HermesBucketGet(file->bkt_handle, k_blob, blob_size, buf+transfer_size); + } + transfer_size += blob_size; + } + } + + /* Update current position */ + file->pos = addr+size; + file->op = OP_READ; + +done: + if (ret_value < 0) { + /* Reset last file I/O information */ + file->pos = HADDR_UNDEF; + file->op = OP_UNKNOWN; + } /* end if */ + + H5FD_HERMES_FUNC_LEAVE_API; +} /* end H5FD__hermes_read() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__hermes_write + * + * Purpose: Writes SIZE bytes of data contained in buffer BUF to Hermes + * buffering system according to data transfer properties in + * DXPL_ID. Determine the number of file pages affected by this + * call from ADDR and SIZE. Utilize transfer buffer PAGE_BUF to + * put the data into Blobs. Exercise care for the first and last + * pages to prevent overwriting existing data. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD__hermes_write(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, + size_t size, const void *buf) { + H5FD_hermes_t *file = (H5FD_hermes_t *)_file; + size_t num_pages; /* Number of pages of transfer buffer */ + size_t start_page_index; /* First page index of tranfer buffer */ + size_t end_page_index; /* End page index of tranfer buffer */ + size_t transfer_size = 0; + size_t blob_size = file->buf_size; + size_t k; + haddr_t addr_end = addr+size-1; + herr_t ret_value = SUCCEED; /* Return value */ + + assert(file && file->pub.cls); + assert(buf); + + /* Check for overflow conditions */ + if (HADDR_UNDEF == addr) { + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "addr undefined, addr = %llu", + (unsigned long long)addr); + } + if (REGION_OVERFLOW(addr, size)) { + H5FD_HERMES_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, + "addr overflow, addr = %llu, size = %llu", + (unsigned long long)addr, (unsigned long long)size); + } + if (NULL == file->page_buf) { + H5FD_HERMES_GOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, FAIL, + "transfer buffer not initialized"); + } + start_page_index = addr/blob_size; + end_page_index = addr_end/blob_size; + num_pages = end_page_index - start_page_index + 1; + + /* Assume only using one page now */ + for (k = start_page_index; k <= end_page_index; ++k) { + char k_blob[LEN_BLOB_NAME]; + snprintf(k_blob, sizeof(k_blob), "%zu\n", k); + /* Check if addr is in the range of (k*blob_size, (k+1)*blob_size) */ + /* NOTE: The range does NOT include the start address of page k, + but includes the end address of page k */ + if (addr > k*blob_size && addr < (k+1)*blob_size) { + /* Check if this blob exists */ + bool blob_exists = check_blob(&file->blob_in_bucket, k); + /* Read blob back to transfer buffer */ + if (blob_exists) { + HermesBucketGet(file->bkt_handle, k_blob, blob_size, file->page_buf); + } + /* Calculate the starting address of transfer buffer update within page + * k */ + size_t offset = addr - k*blob_size; + assert(offset > 0); + + /* Update transfer buffer */ + /* addr+size is within the same page (only one page) */ + if (addr_end <= (k+1)*blob_size-1) { + memcpy(file->page_buf+offset, buf+transfer_size, size); + transfer_size += size; + } else { + /* More than one page */ + /* Copy data from addr to the end of the address in page k */ + memcpy(file->page_buf+offset, buf+transfer_size, (k+1)*blob_size-addr); + transfer_size += (k+1)*blob_size-addr; + } + /* Write Blob k to Hermes buffering system */ + HermesBucketPut(file->bkt_handle, k_blob, file->page_buf, blob_size); + set_blob(&file->blob_in_bucket, k); + } else if (addr_end >= k*blob_size && addr_end < (k+1)*blob_size-1) { + /* Check if addr_end is in the range of [k*blob_size, + * (k+1)*blob_size-1) */ + /* NOTE: The range includes the start address of page k, + but does NOT include the end address of page k */ + /* Check if this blob exists */ + bool blob_exists = check_blob(&file->blob_in_bucket, k); + /* Read blob back */ + if (blob_exists) { + HermesBucketGet(file->bkt_handle, k_blob, blob_size, file->page_buf); + } + /* Update transfer buffer */ + memcpy(file->page_buf, buf+transfer_size, addr_end-k*blob_size+1); + transfer_size += addr_end-k*blob_size+1; + /* Write Blob k to Hermes buffering system */ + HermesBucketPut(file->bkt_handle, k_blob, file->page_buf, blob_size); + set_blob(&file->blob_in_bucket, k); + } else if (addr <= k*blob_size && addr_end >= (k+1)*blob_size-1) { + /* Page/Blob k is within the range of (addr, addr+size) */ + /* Update transfer buffer */ + /* Write Blob k to Hermes buffering system */ + HermesBucketPut(file->bkt_handle, k_blob, buf+transfer_size, blob_size); + set_blob(&(file->blob_in_bucket), k); + transfer_size += blob_size; + } + } + + /* Update current position and eof */ + file->pos = addr+size; + file->op = OP_WRITE; + if (file->pos > file->eof) + file->eof = file->pos; + +done: + if (ret_value < 0) { + /* Reset last file I/O information */ + file->pos = HADDR_UNDEF; + file->op = OP_UNKNOWN; + } /* end if */ + + H5FD_HERMES_FUNC_LEAVE; +} /* end H5FD__hermes_write() */ + +/* + * Stub routines for dynamic plugin loading + */ +H5PL_type_t +H5PLget_plugin_type(void) { + return H5PL_TYPE_VFD; +} + +const void* +H5PLget_plugin_info(void) { + return &H5FD_hermes_g; +} diff --git a/adapter/vfd/H5FDhermes.h b/adapter/vfd/H5FDhermes.h new file mode 100644 index 000000000..cd7d1a647 --- /dev/null +++ b/adapter/vfd/H5FDhermes.h @@ -0,0 +1,37 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Kimmy Mu + * April 2021 + * + * Purpose: The public header file for the Hermes driver. + */ +#ifndef H5FDhermes_H +#define H5FDhermes_H + +#define H5FD_HERMES_NAME "hermes" +#define H5FD_HERMES_VALUE ((H5FD_class_value_t)(513)) + +#ifdef __cplusplus +extern "C" { +#endif + +hid_t H5FD_hermes_init(void); +herr_t H5Pset_fapl_hermes(hid_t fapl_id, hbool_t persistence, size_t page_size); + +#ifdef __cplusplus +} +#endif + +#endif /* end H5FDhermes_H */ diff --git a/adapter/vfd/H5FDhermes_err.h b/adapter/vfd/H5FDhermes_err.h new file mode 100644 index 000000000..3d910b670 --- /dev/null +++ b/adapter/vfd/H5FDhermes_err.h @@ -0,0 +1,199 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 HERMES Virtual File Driver. The full * + * copyright notice, including terms governing use, modification, and * + * redistribution, is contained in the COPYING file, which can be found at * + * the root of the source code distribution tree. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Error handling for the HERMES VFD + */ + +#ifndef H5FDhermes_err_h +#define H5FDhermes_err_h + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include "H5Epublic.h" + +extern hid_t H5FDhermes_err_stack_g; +extern hid_t H5FDhermes_err_class_g; + +#define H5FD_HERMES_ERR_CLS_NAME "HDF5 HERMES VFD" +#define H5FD_HERMES_ERR_LIB_NAME "HDF5 HERMES VFD" +#define H5FD_HERMES_ERR_VER "0.1.0" + +#define SUCCEED 0 +#define FAIL (-1) + +#ifndef FALSE + #define FALSE false +#endif +#ifndef TRUE + #define TRUE true +#endif + +#define H5_ATTR_UNUSED __attribute__((unused)) + +/* Error macros */ + +/* + * Macro to push the current function to the current error stack + * and then goto the "done" label, which should appear inside the + * function. (compatible with v1 and v2 errors) + */ +#define H5FD_HERMES_GOTO_ERROR(err_major, err_minor, ret_val, ...) \ + do { \ + unsigned is_v2_err; \ + union { \ + H5E_auto1_t err_func_v1; \ + H5E_auto2_t err_func_v2; \ + } err_func; \ + \ + /* Determine version of error */ \ + (void)H5Eauto_is_v2(H5E_DEFAULT, &is_v2_err); \ + \ + if (is_v2_err) { \ + (void)H5Eget_auto2(H5E_DEFAULT, &err_func.err_func_v2, NULL); \ + } else { \ + (void)H5Eget_auto1(&err_func.err_func_v1, NULL); \ + } \ + /* Check whether automatic error reporting has been disabled */ \ + if ( (is_v2_err && err_func.err_func_v2) || \ + (!is_v2_err && err_func.err_func_v1) ) { \ + if (H5FDhermes_err_stack_g >= 0 && H5FDhermes_err_class_g >= 0) { \ + H5Epush2(H5FDhermes_err_stack_g, __FILE__, __func__, __LINE__, \ + H5FDhermes_err_class_g, err_major, err_minor, __VA_ARGS__); \ + } else { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } \ + } \ + \ + ret_value = ret_val; \ + goto done; \ + } while (0) + +/* + * Macro to push the current function to the current error stack + * without calling goto. This is used for handling the case where + * an error occurs during cleanup past the "done" label inside a + * function so that an infinite loop does not occur where goto + * continually branches back to the label. (compatible with v1 + * and v2 errors) + */ +#define H5FD_HERMES_DONE_ERROR(err_major, err_minor, ret_val, ...) \ + do { \ + unsigned is_v2_err; \ + union { \ + H5E_auto1_t err_func_v1; \ + H5E_auto2_t err_func_v2; \ + } err_func; \ + \ + /* Determine version of error */ \ + (void)H5Eauto_is_v2(H5E_DEFAULT, &is_v2_err); \ + \ + if (is_v2_err) { \ + (void)H5Eget_auto2(H5E_DEFAULT, &err_func.err_func_v2, NULL); \ + } else { \ + (void)H5Eget_auto1(&err_func.err_func_v1, NULL); \ + } \ + /* Check whether automatic error reporting has been disabled */ \ + if ( (is_v2_err && err_func.err_func_v2) || \ + (!is_v2_err && err_func.err_func_v1) ) { \ + if (H5FDhermes_err_stack_g >= 0 && H5FDhermes_err_class_g >= 0) { \ + H5Epush2(H5FDhermes_err_stack_g, __FILE__, __func__, __LINE__, \ + H5FDhermes_err_class_g, err_major, err_minor, __VA_ARGS__); \ + } else { \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } \ + } \ + \ + ret_value = ret_val; \ + } while (0) + +/* + * Macro to print out the current error stack and then clear it + * for future use. (compatible with v1 and v2 errors) + */ +#define PRINT_ERROR_STACK \ + do { \ + unsigned is_v2_err; \ + union { \ + H5E_auto1_t err_func_v1; \ + H5E_auto2_t err_func_v2; \ + } err_func; \ + \ + /* Determine version of error */ \ + (void)H5Eauto_is_v2(H5E_DEFAULT, &is_v2_err); \ + \ + if (is_v2_err) { \ + (void)H5Eget_auto2(H5E_DEFAULT, &err_func.err_func_v2, NULL); \ + } else { \ + (void)H5Eget_auto1(&err_func.err_func_v1, NULL); \ + } \ + /* Check whether automatic error reporting has been disabled */ \ + if ( (is_v2_err && err_func.err_func_v2) || \ + (!is_v2_err && err_func.err_func_v1) ) { \ + if ((H5FDhermes_err_stack_g >= 0) && \ + (H5Eget_num(H5FDhermes_err_stack_g) > 0)) { \ + H5Eprint2(H5FDhermes_err_stack_g, NULL); \ + H5Eclear2(H5FDhermes_err_stack_g); \ + } \ + } \ + } while (0) + +#define H5FD_HERMES_SYS_GOTO_ERROR(err_major, err_minor, ret_val, str) \ + do { \ + int myerrno = errno; \ + H5FD_HERMES_GOTO_ERROR(err_major, err_minor, ret_val, \ + "%s, errno = %d, error message = '%s'", str, \ + myerrno, strerror(myerrno)); \ + } while (0) + +/* + * Macro to simply jump to the "done" label inside the function, + * setting ret_value to the given value. This is often used for + * short circuiting in functions when certain conditions arise. + */ +#define H5FD_HERMES_GOTO_DONE(ret_val) \ + do { \ + ret_value = ret_val; \ + goto done; \ + } while (0) + +/* + * Macro to return from a top-level API function, printing + * out the error stack on the way out. + * It should be ensured that this macro is only called once + * per HDF5 operation. If it is called multiple times per + * operation (e.g. due to calling top-level API functions + * internally), the error stack will be inconsistent/incoherent. + */ +#define H5FD_HERMES_FUNC_LEAVE_API \ + do { \ + PRINT_ERROR_STACK; \ + return ret_value; \ + } while (0) + +/* + * Macro to return from internal functions. + */ +#define H5FD_HERMES_FUNC_LEAVE \ + do { \ + return ret_value; \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDhermes_err_h */ diff --git a/adapter/vfd/README.md b/adapter/vfd/README.md new file mode 100644 index 000000000..8ff7a0ee8 --- /dev/null +++ b/adapter/vfd/README.md @@ -0,0 +1,50 @@ +# HDF5 Hermes VFD +## 1. Description +The HDF5 Hermes VFD is a Virtual File Driver (VFD) for HDF5 that can be used to interface with Hermes) API. The driver is built as a plugin library that is external to HDF5. + +## 2. Dependencies +To build the HDF5 Hermes VFD, the following libraries are required: +* [Hermes](https://github.com/HDFGroup/hermes) - A heterogeneous aware, multi-tiered, dynamic, and distributed I/O buffering system that aims to significantly accelerate I/O performance. Make sure to build with HERMES_ENABLE_WRAPPER=ON. +* [HDF5](https://github.com/HDFGroup/hdf5) - HDF5 is built for fast I/O processing and storage. + +## 3. Building + +### CMake +Hermes VFD makes use of the CMake build system and requires an out of source build. +``` +cd /path/to/Hermes_VFD +mkdir build +cd build +ccmake .. +``` + +Type 'c' to configure until there are no errors, then generate the makefile with 'g'. The default options should suffice for most use cases. In addition, we recommend the following options. + +``` +-DCMAKE_INSTALL_PREFIX=/installation/prefix +-DHDF5_DIR=/paht/to/hdf5 +-DHERMES_DIR=/path/to/hermes +``` +After the makefile has been generated, you can type `make -j 2` or `cmake --build . -- -j 2`. Add `VERBOSE=1` to see detailed compiler output. + +Assuming that the `CMAKE_INSTALL_PREFIX` has been set and that you have write permissions to the destination directory, you can install the driver by simply doing: +``` +make install +``` + +## 4. Usage +To use the HDF5 Hermes VFD in an HDF5 application, the driver can either be linked into the application, or it can be dynamically loaded as a plugin. If dynamically loading the Hermes VFD, users should ensure that the HDF5_PLUGIN_PATH environment variable points to the directory containing the built VFD library if the VFD has been installed to a non-standard location. + +### Method 1: Linked into application +To link the Hermes VFD into an HDF5 application, the application should include the H5FDhermes.h header that gets installed on the system and should link the installed VFD library (libhdf5_hermes_vfd.so) into the application. Once this has been done, Hermes VFD access can be setup by calling `H5Pset_fapl_hermes(...)` on a FAPL within the HDF5 application. The test `hermes_set_fapl` is using this method, in which it calls `H5Pset_fapl_hermes` directly. + +### Method 2: Dynamically loaded by FAPL +To explicitly load the Hermes VFD inside an HDF5 application, a call to the `H5Pset_driver_by_name(...)` routine should be made to setup Hermes VFD access on a FAPL. This will cause HDF5 to load the VFD as a plugin and set the VFD on the given FAPL. The string "hermes" should be given for the driver_name parameter. A string should be given for the `driver_config` parameter (last parameter in `H5Pset_driver_by_name`), as the driver requires additional parameters to config Hermes. An example string like "false,1024" (comma dilemma) is matching the second and third parameter as in `H5Pset_fapl_hermes`. User also needs to set up the enrivonment variable `HDF5_PLUGIN_PATH`, which points to directory containing the built Hermes VFD library. Test `hermes_driver` is using this method. + +### Method 3: Dynamically loaded by environment variable +To implicitly load the Hermes VFD inside an HDF5 application, the HDF5_DRIVER environment variable may be set to the string "hermes". During library initialization, HDF5 will check this environment variable, load the Hermes VFD as a plugin and set the VFD as the default file driver on File Access Property Lists. Therefore, any file access that uses H5P_DEFAULT for its FAPL, or which uses a FAPL that hasn't had a specific VFD set on it, will automatically use the Hermes VFD for file access. User can simply setup HDF5_DRIVER environment variable to "hermes" and HDF5_DRIVER_CONFIG the same as `driver_config` parameter without calling `H5Pset_driver_by_name()`, compared to Method 2 Dynamically loaded by FAPL. Test `hermes_env` is using this method. + +## 5. More Information +[More about Hermes](https://github.com/HDFGroup/hermes/wiki) + +[HDF5 VFD Plugins RFC](https://github.com/HDFGroup/hdf5doc/blob/master/RFCs/HDF5_Library/VFL_DriverPlugins/RFC__A_Plugin_Interface_for_HDF5_Virtual_File_Drivers.pdf) diff --git a/adapter/vfd/test/CMakeLists.txt b/adapter/vfd/test/CMakeLists.txt new file mode 100644 index 000000000..c890812c6 --- /dev/null +++ b/adapter/vfd/test/CMakeLists.txt @@ -0,0 +1,48 @@ +#------------------------------------------------------------------------------ +# Include source and build directories +#------------------------------------------------------------------------------ +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +#----------------------------------------------------------------------------- +# Define Sources and tests +#----------------------------------------------------------------------------- +set(hermes_vfd_tests + hermes_set_fapl + hermes_driver + hermes_env +) + +foreach(vfd_test ${hermes_vfd_tests}) + add_executable(${vfd_test} + ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.c + ) + target_include_directories(${vfd_test} + PUBLIC "$" + $ + ) + target_include_directories(${vfd_test} + SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ) + target_link_libraries(${vfd_test} + ${HDF5_HERMES_VFD_EXPORTED_LIBS} + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} + ) +endforeach() + +#----------------------------------------------------------------------------- +# Tests +#----------------------------------------------------------------------------- +add_test(NAME "Testhermes_set_fapl" COMMAND hermes_set_fapl) +set_tests_properties(Testhermes_set_fapl PROPERTIES + ENVIRONMENT "HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf" +) + +add_test(NAME "Testhermes_driver" + COMMAND hermes_driver -E env HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + +add_test(NAME "Testhermes_env" + COMMAND hermes_driver -E env HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} HDF5_DRIVER_CONFIG="true 1024" HDF5_DRIVER=hermes) diff --git a/adapter/vfd/test/hermes.conf b/adapter/vfd/test/hermes.conf new file mode 100644 index 000000000..91e6cf989 --- /dev/null +++ b/adapter/vfd/test/hermes.conf @@ -0,0 +1,112 @@ +mple Hermes configuration file + +# TODO(chogan): Allow specifying capacity values in bytes, KiB, or GiB. + +# The number of buffering tiers available. For example, RAM, NVMe, burst +# buffer, and parallel file system would be 4 tiers. +num_devices = 4; +# For now this should be the same as num_devices. +num_targets = 4; + +# The maximum buffering capacity in MiB of each device. Here we say that all 4 +# devices get 50 MiB of buffering capacity. +capacities_mb = {50, 50, 50, 50}; +# The size of the smallest available buffer in KiB. In general this should be +# the page size of your system for byte addressable storage, and the block size +# of the storage device for block addressable storage. +block_sizes_kb = {1, 1, 1, 1}; +# The nummber of size categories for each device. Here we say that each of our 4 +# devices should have 4 different sizes of buffers. +num_slabs = {1, 1, 1, 1}; + +# The number of blocks (the size of which is chosen in block_sizes_kb) that each +# device should contain for each slab (controlled by num_slabs). This allows for +# precise control of the distibution of buffer sizes. +slab_unit_sizes = { + {1}, + {1}, + {1}, + {1}, +}; + +# The percentage of buffering capacity per device to allocate for each slab. +# Each row should add up to 1. In this example, we have 4 devices, each with 4 +# slabs, and each slab is allotted 25% of the device's total buffering capacity. +desired_slab_percentages = { + {1}, + {1}, + {1}, + {1}, +}; + +# The maximum theoretical bandwidth (as advertised by the manufacturer) in +# MiB/sec. of each device. +bandwidths_mbps = {6000, 300, 150, 70}; +# The latency in microseconds of each device (as advertised by the manufactuerer). +latencies_us = {15, 250000, 500000, 1000000}; + +# Hermes memory management. The following 4 values should add up to 1. +# The percentage of Hermes memory to reserve for RAM buffers. +buffer_pool_arena_percentage = 0.85; +# The percentage of Hermes memory to reserve for metadata. +metadata_arena_percentage = 0.04; +# The percentage of Hermes memory to reserve for data transfers. +transfer_window_arena_percentage = 0.08; +# The percentage of Hermes memory to reserve for short term storage. +transient_arena_percentage = 0.03; + +# The maxiumum number of buckets that can be created. +max_buckets_per_node = 16; +# The maxiumum number of virtual buckets that can be created. +max_vbuckets_per_node = 8; +# The interval in milliseconds at which to update the global system view. +system_view_state_update_interval_ms = 1000; + +# The mount point of each device. RAM should be the empty string. For block +# devices, this is the directory where Hermes will create buffering files. For +# object storage or cloud targets, this will be a url. +mount_points = {"", "./", "./", "./"}; +# For each device, indicate '1' if it is shared among nodes (e.g., burst +# buffers), or '0' if it is per node (e.g., local NVMe). +is_shared_device = {0, 0, 0, 0}; +# The mount point of a PFS or object store for swap space, in the event that +# Hermes buffers become full. +swap_mount = "./"; +# The number of times the buffer organizer will attempt to place a blob from +# swap space into the hierarchy before giving up. +num_buffer_organizer_retries = 3; +# Base hostname for the RPC servers. +rpc_server_base_name = "localhost"; +# RPC server name suffix. This is appended to the the base name plus host +# number. +rpc_server_suffix = ""; +# The RPC protocol. This must come from the documentation of the specific RPC +# library in use. +rpc_protocol = "ofi+sockets"; +# RPC domain name for verbs transport. Blank for tcp. +rpc_domain = ""; +# Desired RPC port number. +rpc_port = 8080; +# Desired RPC port number for buffer organizer. +buffer_organizer_port = 8081; +# An inclusive range of the first and last server numbers. This is a convenience +# feature for generating long lists of server names. For example, if your +# servers are called server-1-40g, server-2-40g, server-3-40g, all the way to +# server-100-40g, then you would set rpc_server_base_name to 'server', +# rpc_server_suffix to '-40g', and rpc_host_number_range to {1, 100}. +# TODO(chogan): Support reading server names from file. +rpc_host_number_range = {0, 0}; +# The number of handler threads for each RPC server. +rpc_num_threads = 1; +# The number of threads used in the background organization of internal Hermes buffers. +buffer_organizer_num_threads = 4; +# The shared memory prefix for the hermes shared memory segment. A user name +# will be automatically appended. +buffer_pool_shmem_name = "/hermes_buffer_pool_"; + +# Choose Random, RoundRobin, or MinimizeIoTime +default_placement_policy = "RoundRobin"; + +# If true (1) the RoundRobin placement policy algorithm will split each Blob +# into a random number of smaller Blobs. +default_rr_split = 0; diff --git a/adapter/vfd/test/hermes_driver.c b/adapter/vfd/test/hermes_driver.c new file mode 100644 index 000000000..79f9e0308 --- /dev/null +++ b/adapter/vfd/test/hermes_driver.c @@ -0,0 +1,130 @@ +#include +#include +#include +#include +#include +#include + +#include "hdf5.h" +#include "H5FDhermes.h" + +#define DATASETNAME "IntArray" +#define NX 128 /* dataset dimensions */ +#define NY 128 + +int main(int argc, char *argv[]) { + hid_t file_id, fapl_id; + hid_t dset_id, dcpl_id, dataspace_id; + hid_t dapl; + hsize_t dims[2]; + char *file_name = "hermes_test.h5"; + int data_in[NX][NY]; /* data to write */ + int data_out[NX][NY]; /* data to read */ + int i, j, k; + + const char *const config_str = "true 1024"; + + int mpi_threads_provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); + if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { + fprintf(stderr, + "Didn't receive appropriate MPI threading specification\n"); + return 1; + } + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + printf("H5Pcreate() error\n"); + else + printf("H5Pcreate() succeeded\n"); + + if (H5Pset_driver_by_name(fapl_id, "hermes", config_str) < 0) + printf("H5Pset_driver_by_name() error\n"); + else + printf("H5Pset_driver_by_name() succeeded\n"); + + if ((file_id = + H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) { + printf("H5Fcreate() error\n"); + } else { + printf("H5Fcreate() succeeded\n"); + } + + dims[0] = NX; + dims[1] = NY; + dataspace_id = H5Screate_simple(2, dims, NULL); + + dset_id = H5Dcreate(file_id, DATASETNAME, H5T_NATIVE_INT, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + for (j = 0; j < NX; j++) { + for (i = 0; i < NY; i++) { + data_in[j][i] = i + j; + data_out[j][i] = 0; + } + } + + if (H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_in) < 0) { + printf("H5Dwrite() error\n"); + } else { + printf("H5Dwrite() succeeded\n"); + } + + if (H5Dclose(dset_id) < 0) + printf("H5Dclose() error\n"); + else + printf("H5Dclose() succeeded\n"); + + if (H5Fclose(file_id) < 0) + printf("H5Fclose() error\n"); + else + printf("H5Fclose() succeeded\n"); + + hid_t fid; + if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) + printf("H5Fopen() error\n"); + else + printf("H5Fopen() succeeded\n"); + + if ((dset_id = H5Dopen(fid, DATASETNAME, H5P_DEFAULT)) < 0) + printf("H5Dopen() error\n"); + else + printf("H5Dopen() succeeded\n"); + + if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_out) < 0) { + printf("H5Dread() error\n"); + } else { + printf("H5Dread() succeeded\n"); + } + + for (j = 0; j < NX; j++) { + for (i = 0; i < NY; i++) { + assert(data_out[j][i] == data_in[j][i]); + } + } + + if (H5Dclose(dset_id) < 0) + printf("H5Dclose() error\n"); + else + printf("H5Dclose() succeeded\n"); + + if (H5Fclose(fid) < 0) + printf("H5Fclose() error\n"); + else + printf("H5Fclose() succeeded\n"); + + if (H5Sclose(dataspace_id) < 0) + printf("H5Sclose() error\n"); + else + printf("H5Sclose() succeeded\n"); + + if (H5Pclose(fapl_id) < 0) + printf("H5Pclose() error\n"); + else + printf("H5Pclose() succeeded\n"); + + printf("Done with Hermes set driver test!\n"); + + return 0; +} diff --git a/adapter/vfd/test/hermes_env.c b/adapter/vfd/test/hermes_env.c new file mode 100644 index 000000000..022f9f289 --- /dev/null +++ b/adapter/vfd/test/hermes_env.c @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include + +#include "hdf5.h" +#include "H5FDhermes.h" + +/* HDF5 header for dynamic plugin loading */ +#include "H5PLextern.h" + +#define DATASETNAME "IntArray" +#define NX 128 /* dataset dimensions */ +#define NY 128 + +int main(int argc, char *argv[]) { + hid_t file_id, fapl_id; + hid_t dset_id, dcpl_id, dataspace_id; + hid_t dapl; + hsize_t dims[2]; + char *file_name = "hermes_test.h5"; + int data_in[NX][NY]; /* data to write */ + int data_out[NX][NY]; /* data to read */ + int i, j, k; + + int mpi_threads_provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); + if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { + fprintf(stderr, + "Didn't receive appropriate MPI threading specification\n"); + return 1; + } + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + printf("H5Pcreate() error\n"); + else + printf("H5Pcreate() succeeded\n"); + + if ((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) + < 0) { + printf("H5Fcreate() error\n"); + } else { + printf("H5Fcreate() succeeded\n"); + } + + dims[0] = NX; + dims[1] = NY; + dataspace_id = H5Screate_simple(2, dims, NULL); + + dset_id = H5Dcreate(file_id, DATASETNAME, H5T_NATIVE_INT, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + for (j = 0; j < NX; j++) { + for (i = 0; i < NY; i++) { + data_in[j][i] = i + j; + data_out[j][i] = 0; + } + } + + if (H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_in) < 0) + printf("H5Dwrite() error\n"); + else + printf("H5Dwrite() succeeded\n"); + + if (H5Dclose(dset_id) < 0) + printf("H5Dclose() error\n"); + else + printf("H5Dclose() succeeded\n"); + + if (H5Fclose(file_id) < 0) + printf("H5Fclose() error\n"); + else + printf("H5Fclose() succeeded\n"); + + hid_t fid; + if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) + printf("H5Fopen() error\n"); + else + printf("H5Fopen() succeeded\n"); + + if ((dset_id = H5Dopen(fid, DATASETNAME, H5P_DEFAULT)) < 0) + printf("H5Dopen() error\n"); + else + printf("H5Dopen() succeeded\n"); + + if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_out) < 0) + printf("H5Dread() error\n"); + else + printf("H5Dread() succeeded\n"); + + for (j = 0; j < NX; j++) { + for (i = 0; i < NY; i++) { + assert(data_out[j][i] == data_in[j][i]); + } + } + + if (H5Dclose(dset_id) < 0) + printf("H5Dclose() error\n"); + else + printf("H5Dclose() succeeded\n"); + + if (H5Fclose(fid) < 0) + printf("H5Fclose() error\n"); + else + printf("H5Fclose() succeeded\n"); + + if (H5Sclose(dataspace_id) < 0) + printf("H5Sclose() error\n"); + else + printf("H5Sclose() succeeded\n"); + + if (H5Pclose(fapl_id) < 0) + printf("H5Pclose() error\n"); + else + printf("H5Pclose() succeeded\n"); + + printf("Done with Hermes set driver test!\n"); + + return 0; +} diff --git a/adapter/vfd/test/hermes_set_fapl.c b/adapter/vfd/test/hermes_set_fapl.c new file mode 100644 index 000000000..84cea02b8 --- /dev/null +++ b/adapter/vfd/test/hermes_set_fapl.c @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include + +#include "hdf5.h" +#include "H5FDhermes.h" + +#define DATASETNAME "IntArray" +#define NX 128 /* dataset dimensions */ +#define NY 128 + +int main(int argc, char *argv[]) { + hid_t file_id, fapl_id; + hid_t dset_id, dcpl_id, dataspace_id; + hid_t dapl; + hsize_t dims[2]; + char *file_name = "hermes_test.h5"; + int data_in[NX][NY]; /* data to write */ + int data_out[NX][NY]; /* data to read */ + int i, j, k; + + int mpi_threads_provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); + if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { + fprintf(stderr, + "Didn't receive appropriate MPI threading specification\n"); + return 1; + } + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + printf("H5Pcreate() error\n"); + else + printf("H5Pcreate() succeeded\n"); + + if (H5Pset_fapl_hermes(fapl_id, true, 1024) < 0) + printf("H5Pset_fapl_hermes() error\n"); + else + printf("H5Pset_fapl_hermes() succeeded\n"); + + if ((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) + < 0) + printf("H5Fcreate() error\n"); + else + printf("H5Fcreate() succeeded\n"); + + dims[0] = NX; + dims[1] = NY; + dataspace_id = H5Screate_simple(2, dims, NULL); + + dset_id = H5Dcreate(file_id, DATASETNAME, H5T_NATIVE_INT, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + for (j = 0; j < NX; j++) { + for (i = 0; i < NY; i++) { + data_in[j][i] = i + j; + data_out[j][i] = 0; + } + } + + if (H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_in) < 0) + printf("H5Dwrite() error\n"); + else + printf("H5Dwrite() succeeded\n"); + + if (H5Dclose(dset_id) < 0) + printf("H5Dclose() error\n"); + else + printf("H5Dclose() succeeded\n"); + + if (H5Fclose(file_id) < 0) + printf("H5Fclose() error\n"); + else + printf("H5Fclose() succeeded\n"); + + hid_t fid; + if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) + printf("H5Fopen() error\n"); + else + printf("H5Fopen() succeeded\n"); + + if ((dset_id = H5Dopen(fid, DATASETNAME, H5P_DEFAULT)) < 0) + printf("H5Dopen() error\n"); + else + printf("H5Dopen() succeeded\n"); + + if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_out) < 0) + printf("H5Dread() error\n"); + else + printf("H5Dread() succeeded\n"); + + for (j = 0; j < NX; j++) { + for (i = 0; i < NY; i++) { + assert(data_out[j][i] == data_in[j][i]); + } + } + + if (H5Dclose(dset_id) < 0) + printf("H5Dclose() error\n"); + else + printf("H5Dclose() succeeded\n"); + + if (H5Fclose(fid) < 0) + printf("H5Fclose() error\n"); + else + printf("H5Fclose() succeeded\n"); + + if (H5Sclose(dataspace_id) < 0) + printf("H5Sclose() error\n"); + else + printf("H5Sclose() succeeded\n"); + + if (H5Pclose(fapl_id) < 0) + printf("H5Pclose() error\n"); + else + printf("H5Pclose() succeeded\n"); + + printf("Done with Hermes set driver test!\n"); + + return 0; +} diff --git a/wrapper/CMakeLists.txt b/wrapper/CMakeLists.txt index b2033f1a0..f5e3eca89 100644 --- a/wrapper/CMakeLists.txt +++ b/wrapper/CMakeLists.txt @@ -14,7 +14,7 @@ add_library(hermes_wrapper hermes_wrapper.cpp) target_link_libraries(hermes_wrapper ${LIBRT} hermes) target_compile_definitions(hermes_wrapper PRIVATE $<$:HERMES_RPC_THALLIUM>) - + set(HERMES_EXPORTED_LIBS hermes_wrapper ${HERMES_EXPORTED_LIBS}) #----------------------------------------------------------------------------- diff --git a/wrapper/hermes_wrapper.cpp b/wrapper/hermes_wrapper.cpp index 463c0ec2c..a0ed26845 100644 --- a/wrapper/hermes_wrapper.cpp +++ b/wrapper/hermes_wrapper.cpp @@ -102,8 +102,8 @@ bool HermesBucketContainsBlob(BucketClass *bkt, char *name) { return bucket->ContainsBlob(name); } -void HermesBucketPut(BucketClass *bkt, char *name, unsigned char *put_data, - size_t size) { +void HermesBucketPut(BucketClass *bkt, char *name, + const unsigned char *put_data, size_t size) { hermes::api::Bucket *bucket = (hermes::api::Bucket *)bkt; LOG(INFO) << "Hermes Wrapper: Putting Blob " << name << " to bucket " << diff --git a/wrapper/hermes_wrapper.h b/wrapper/hermes_wrapper.h index 63efeed53..c88ea89f7 100644 --- a/wrapper/hermes_wrapper.h +++ b/wrapper/hermes_wrapper.h @@ -45,8 +45,8 @@ void HermesBucketDestroy(BucketClass *bucket_ptr); bool HermesBucketContainsBlob(BucketClass *bkt, char *name); -void HermesBucketPut(BucketClass *bkt, char *name, unsigned char *put_data, - size_t size); +void HermesBucketPut(BucketClass *bkt, char *name, + const unsigned char *put_data, size_t size); void HermesBucketGet(BucketClass *bkt, char *blob_name, size_t kPageSize, unsigned char *buf); From d7c5af010c5ed930762a39e2cc7c6663bb48ddcd Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Tue, 11 Jan 2022 15:53:44 -0600 Subject: [PATCH 02/16] Restructure VFD tests --- CMakeLists.txt | 2 +- adapter/test/CMakeLists.txt | 3 ++ adapter/test/vfd/CMakeLists.txt | 30 ++++++++++++ adapter/{vfd/test => test/vfd}/hermes.conf | 0 .../{vfd/test => test/vfd}/hermes_driver.c | 0 adapter/{vfd/test => test/vfd}/hermes_env.c | 3 +- .../{vfd/test => test/vfd}/hermes_set_fapl.c | 0 adapter/vfd/CMakeLists.txt | 9 ---- adapter/vfd/test/CMakeLists.txt | 48 ------------------- 9 files changed, 36 insertions(+), 59 deletions(-) create mode 100644 adapter/test/vfd/CMakeLists.txt rename adapter/{vfd/test => test/vfd}/hermes.conf (100%) rename adapter/{vfd/test => test/vfd}/hermes_driver.c (100%) rename adapter/{vfd/test => test/vfd}/hermes_env.c (99%) rename adapter/{vfd/test => test/vfd}/hermes_set_fapl.c (100%) delete mode 100644 adapter/vfd/test/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 11f77120f..0100c06a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -287,7 +287,7 @@ if(HERMES_INTERCEPT_IO) endif() if(HERMES_COMMUNICATION_MPI) - find_package(MPI REQUIRED COMPONENTS CXX) + find_package(MPI REQUIRED COMPONENTS C CXX) message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") endif() diff --git a/adapter/test/CMakeLists.txt b/adapter/test/CMakeLists.txt index 8ff4a5952..ffdb8e425 100644 --- a/adapter/test/CMakeLists.txt +++ b/adapter/test/CMakeLists.txt @@ -46,6 +46,9 @@ enable_testing() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/posix) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/stdio) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mpiio) +if(HERMES_ENABLE_VFD) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/vfd) +endif() install( TARGETS diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt new file mode 100644 index 000000000..78a7fdabb --- /dev/null +++ b/adapter/test/vfd/CMakeLists.txt @@ -0,0 +1,30 @@ +set(HERMES_VFD_DIR ${HERMES_ADAPTER_DIR}/vfd) + +set(hermes_vfd_tests + hermes_set_fapl + hermes_driver + hermes_env +) + +foreach(vfd_test ${hermes_vfd_tests}) + add_executable(${vfd_test} ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.c) + target_include_directories(${vfd_test} PRIVATE ${HERMES_VFD_DIR}) + target_include_directories(${vfd_test} + SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} + ) + target_link_libraries(${vfd_test} + MPI::MPI_C + ${HDF5_HERMES_VFD_EXPORTED_LIBS} + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} + ) + add_test(NAME "Test${vfd_test}" COMMAND ${vfd_test}) + set_property(TEST "Test${vfd_test}" + PROPERTY ENVIRONMENT HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf) + set_property(TEST "Test${vfd_test}" APPEND + PROPERTY ENVIRONMENT HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set_property(TEST "Test${vfd_test}" APPEND + PROPERTY ENVIRONMENT HDF5_DRIVER_CONFIG="true 1024") + set_property(TEST "Test${vfd_test}" APPEND + PROPERTY ENVIRONMENT HDF5_DRIVER=hermes) +endforeach() diff --git a/adapter/vfd/test/hermes.conf b/adapter/test/vfd/hermes.conf similarity index 100% rename from adapter/vfd/test/hermes.conf rename to adapter/test/vfd/hermes.conf diff --git a/adapter/vfd/test/hermes_driver.c b/adapter/test/vfd/hermes_driver.c similarity index 100% rename from adapter/vfd/test/hermes_driver.c rename to adapter/test/vfd/hermes_driver.c diff --git a/adapter/vfd/test/hermes_env.c b/adapter/test/vfd/hermes_env.c similarity index 99% rename from adapter/vfd/test/hermes_env.c rename to adapter/test/vfd/hermes_env.c index 022f9f289..c5c56291c 100644 --- a/adapter/vfd/test/hermes_env.c +++ b/adapter/test/vfd/hermes_env.c @@ -5,7 +5,8 @@ #include #include -#include "hdf5.h" +#include + #include "H5FDhermes.h" /* HDF5 header for dynamic plugin loading */ diff --git a/adapter/vfd/test/hermes_set_fapl.c b/adapter/test/vfd/hermes_set_fapl.c similarity index 100% rename from adapter/vfd/test/hermes_set_fapl.c rename to adapter/test/vfd/hermes_set_fapl.c diff --git a/adapter/vfd/CMakeLists.txt b/adapter/vfd/CMakeLists.txt index 752db721b..89587ac6c 100644 --- a/adapter/vfd/CMakeLists.txt +++ b/adapter/vfd/CMakeLists.txt @@ -224,12 +224,3 @@ set(HDF5_HERMES_VFD_INCLUDES_INSTALL_TIME ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} PARENT_SCOPE ) -#----------------------------------------------------------------------------- -# Testing -#----------------------------------------------------------------------------- -# option(BUILD_TESTING "Build testing." ON) -# if(NOT HDF5_HERMES_VFD_EXTERNALLY_CONFIGURED AND BUILD_TESTING) -# enable_testing() -# include(CTest) -# add_subdirectory(test) -# endif() diff --git a/adapter/vfd/test/CMakeLists.txt b/adapter/vfd/test/CMakeLists.txt deleted file mode 100644 index c890812c6..000000000 --- a/adapter/vfd/test/CMakeLists.txt +++ /dev/null @@ -1,48 +0,0 @@ -#------------------------------------------------------------------------------ -# Include source and build directories -#------------------------------------------------------------------------------ -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} -) - -#----------------------------------------------------------------------------- -# Define Sources and tests -#----------------------------------------------------------------------------- -set(hermes_vfd_tests - hermes_set_fapl - hermes_driver - hermes_env -) - -foreach(vfd_test ${hermes_vfd_tests}) - add_executable(${vfd_test} - ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.c - ) - target_include_directories(${vfd_test} - PUBLIC "$" - $ - ) - target_include_directories(${vfd_test} - SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} - ) - target_link_libraries(${vfd_test} - ${HDF5_HERMES_VFD_EXPORTED_LIBS} - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} - ) -endforeach() - -#----------------------------------------------------------------------------- -# Tests -#----------------------------------------------------------------------------- -add_test(NAME "Testhermes_set_fapl" COMMAND hermes_set_fapl) -set_tests_properties(Testhermes_set_fapl PROPERTIES - ENVIRONMENT "HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf" -) - -add_test(NAME "Testhermes_driver" - COMMAND hermes_driver -E env HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - -add_test(NAME "Testhermes_env" - COMMAND hermes_driver -E env HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY} HDF5_DRIVER_CONFIG="true 1024" HDF5_DRIVER=hermes) From ee6fa91198b9011e782a0ee4577b35ddbec6ebdb Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Wed, 12 Jan 2022 10:37:42 -0600 Subject: [PATCH 03/16] Update vfd hermes.conf to new format and remove statically compiled vfd test --- adapter/test/vfd/CMakeLists.txt | 1 - adapter/test/vfd/hermes.conf | 68 +--------------- adapter/test/vfd/hermes_set_fapl.c | 125 ----------------------------- 3 files changed, 2 insertions(+), 192 deletions(-) delete mode 100644 adapter/test/vfd/hermes_set_fapl.c diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt index 78a7fdabb..c078d791f 100644 --- a/adapter/test/vfd/CMakeLists.txt +++ b/adapter/test/vfd/CMakeLists.txt @@ -1,7 +1,6 @@ set(HERMES_VFD_DIR ${HERMES_ADAPTER_DIR}/vfd) set(hermes_vfd_tests - hermes_set_fapl hermes_driver hermes_env ) diff --git a/adapter/test/vfd/hermes.conf b/adapter/test/vfd/hermes.conf index 91e6cf989..8dc8b138d 100644 --- a/adapter/test/vfd/hermes.conf +++ b/adapter/test/vfd/hermes.conf @@ -1,27 +1,10 @@ -mple Hermes configuration file - -# TODO(chogan): Allow specifying capacity values in bytes, KiB, or GiB. - -# The number of buffering tiers available. For example, RAM, NVMe, burst -# buffer, and parallel file system would be 4 tiers. num_devices = 4; -# For now this should be the same as num_devices. num_targets = 4; -# The maximum buffering capacity in MiB of each device. Here we say that all 4 -# devices get 50 MiB of buffering capacity. capacities_mb = {50, 50, 50, 50}; -# The size of the smallest available buffer in KiB. In general this should be -# the page size of your system for byte addressable storage, and the block size -# of the storage device for block addressable storage. block_sizes_kb = {1, 1, 1, 1}; -# The nummber of size categories for each device. Here we say that each of our 4 -# devices should have 4 different sizes of buffers. num_slabs = {1, 1, 1, 1}; -# The number of blocks (the size of which is chosen in block_sizes_kb) that each -# device should contain for each slab (controlled by num_slabs). This allows for -# precise control of the distibution of buffer sizes. slab_unit_sizes = { {1}, {1}, @@ -29,9 +12,6 @@ slab_unit_sizes = { {1}, }; -# The percentage of buffering capacity per device to allocate for each slab. -# Each row should add up to 1. In this example, we have 4 devices, each with 4 -# slabs, and each slab is allotted 25% of the device's total buffering capacity. desired_slab_percentages = { {1}, {1}, @@ -39,74 +19,30 @@ desired_slab_percentages = { {1}, }; -# The maximum theoretical bandwidth (as advertised by the manufacturer) in -# MiB/sec. of each device. bandwidths_mbps = {6000, 300, 150, 70}; -# The latency in microseconds of each device (as advertised by the manufactuerer). latencies_us = {15, 250000, 500000, 1000000}; -# Hermes memory management. The following 4 values should add up to 1. -# The percentage of Hermes memory to reserve for RAM buffers. buffer_pool_arena_percentage = 0.85; -# The percentage of Hermes memory to reserve for metadata. metadata_arena_percentage = 0.04; -# The percentage of Hermes memory to reserve for data transfers. -transfer_window_arena_percentage = 0.08; -# The percentage of Hermes memory to reserve for short term storage. -transient_arena_percentage = 0.03; +transient_arena_percentage = 0.11; -# The maxiumum number of buckets that can be created. max_buckets_per_node = 16; -# The maxiumum number of virtual buckets that can be created. max_vbuckets_per_node = 8; -# The interval in milliseconds at which to update the global system view. system_view_state_update_interval_ms = 1000; -# The mount point of each device. RAM should be the empty string. For block -# devices, this is the directory where Hermes will create buffering files. For -# object storage or cloud targets, this will be a url. mount_points = {"", "./", "./", "./"}; -# For each device, indicate '1' if it is shared among nodes (e.g., burst -# buffers), or '0' if it is per node (e.g., local NVMe). is_shared_device = {0, 0, 0, 0}; -# The mount point of a PFS or object store for swap space, in the event that -# Hermes buffers become full. swap_mount = "./"; -# The number of times the buffer organizer will attempt to place a blob from -# swap space into the hierarchy before giving up. num_buffer_organizer_retries = 3; -# Base hostname for the RPC servers. rpc_server_base_name = "localhost"; -# RPC server name suffix. This is appended to the the base name plus host -# number. rpc_server_suffix = ""; -# The RPC protocol. This must come from the documentation of the specific RPC -# library in use. rpc_protocol = "ofi+sockets"; -# RPC domain name for verbs transport. Blank for tcp. rpc_domain = ""; -# Desired RPC port number. rpc_port = 8080; -# Desired RPC port number for buffer organizer. buffer_organizer_port = 8081; -# An inclusive range of the first and last server numbers. This is a convenience -# feature for generating long lists of server names. For example, if your -# servers are called server-1-40g, server-2-40g, server-3-40g, all the way to -# server-100-40g, then you would set rpc_server_base_name to 'server', -# rpc_server_suffix to '-40g', and rpc_host_number_range to {1, 100}. -# TODO(chogan): Support reading server names from file. -rpc_host_number_range = {0, 0}; -# The number of handler threads for each RPC server. +rpc_host_number_range = {}; rpc_num_threads = 1; -# The number of threads used in the background organization of internal Hermes buffers. buffer_organizer_num_threads = 4; -# The shared memory prefix for the hermes shared memory segment. A user name -# will be automatically appended. buffer_pool_shmem_name = "/hermes_buffer_pool_"; - -# Choose Random, RoundRobin, or MinimizeIoTime default_placement_policy = "RoundRobin"; - -# If true (1) the RoundRobin placement policy algorithm will split each Blob -# into a random number of smaller Blobs. default_rr_split = 0; diff --git a/adapter/test/vfd/hermes_set_fapl.c b/adapter/test/vfd/hermes_set_fapl.c deleted file mode 100644 index 84cea02b8..000000000 --- a/adapter/test/vfd/hermes_set_fapl.c +++ /dev/null @@ -1,125 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "hdf5.h" -#include "H5FDhermes.h" - -#define DATASETNAME "IntArray" -#define NX 128 /* dataset dimensions */ -#define NY 128 - -int main(int argc, char *argv[]) { - hid_t file_id, fapl_id; - hid_t dset_id, dcpl_id, dataspace_id; - hid_t dapl; - hsize_t dims[2]; - char *file_name = "hermes_test.h5"; - int data_in[NX][NY]; /* data to write */ - int data_out[NX][NY]; /* data to read */ - int i, j, k; - - int mpi_threads_provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); - if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { - fprintf(stderr, - "Didn't receive appropriate MPI threading specification\n"); - return 1; - } - - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) - printf("H5Pcreate() error\n"); - else - printf("H5Pcreate() succeeded\n"); - - if (H5Pset_fapl_hermes(fapl_id, true, 1024) < 0) - printf("H5Pset_fapl_hermes() error\n"); - else - printf("H5Pset_fapl_hermes() succeeded\n"); - - if ((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) - < 0) - printf("H5Fcreate() error\n"); - else - printf("H5Fcreate() succeeded\n"); - - dims[0] = NX; - dims[1] = NY; - dataspace_id = H5Screate_simple(2, dims, NULL); - - dset_id = H5Dcreate(file_id, DATASETNAME, H5T_NATIVE_INT, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - for (j = 0; j < NX; j++) { - for (i = 0; i < NY; i++) { - data_in[j][i] = i + j; - data_out[j][i] = 0; - } - } - - if (H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data_in) < 0) - printf("H5Dwrite() error\n"); - else - printf("H5Dwrite() succeeded\n"); - - if (H5Dclose(dset_id) < 0) - printf("H5Dclose() error\n"); - else - printf("H5Dclose() succeeded\n"); - - if (H5Fclose(file_id) < 0) - printf("H5Fclose() error\n"); - else - printf("H5Fclose() succeeded\n"); - - hid_t fid; - if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) - printf("H5Fopen() error\n"); - else - printf("H5Fopen() succeeded\n"); - - if ((dset_id = H5Dopen(fid, DATASETNAME, H5P_DEFAULT)) < 0) - printf("H5Dopen() error\n"); - else - printf("H5Dopen() succeeded\n"); - - if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data_out) < 0) - printf("H5Dread() error\n"); - else - printf("H5Dread() succeeded\n"); - - for (j = 0; j < NX; j++) { - for (i = 0; i < NY; i++) { - assert(data_out[j][i] == data_in[j][i]); - } - } - - if (H5Dclose(dset_id) < 0) - printf("H5Dclose() error\n"); - else - printf("H5Dclose() succeeded\n"); - - if (H5Fclose(fid) < 0) - printf("H5Fclose() error\n"); - else - printf("H5Fclose() succeeded\n"); - - if (H5Sclose(dataspace_id) < 0) - printf("H5Sclose() error\n"); - else - printf("H5Sclose() succeeded\n"); - - if (H5Pclose(fapl_id) < 0) - printf("H5Pclose() error\n"); - else - printf("H5Pclose() succeeded\n"); - - printf("Done with Hermes set driver test!\n"); - - return 0; -} From b46d4109c68749e165d1aca664c573934045601c Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Thu, 13 Jan 2022 16:05:26 -0600 Subject: [PATCH 04/16] Let GLOG shut itself down. --- CMakeLists.txt | 39 +++++++++++++++------------------ adapter/test/vfd/CMakeLists.txt | 3 +-- adapter/test/vfd/hermes_env.c | 29 ++++++++---------------- src/buffer_pool.cc | 9 +++++++- 4 files changed, 36 insertions(+), 44 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0100c06a8..cd7cbc990 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -325,6 +325,24 @@ if(HERMES_ENABLE_COVERAGE) endmacro() endif() +#----------------------------------------------------------------------------- +# Testing +#----------------------------------------------------------------------------- +if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) + include(CTest) + find_package(Catch2 REQUIRED) + find_program(IOR_EXE ior) + if(IOR_EXE) + message(STATUS "Found ior at ${IOR_EXE}") + set(HERMES_HAVE_IOR "YES") + else() + message(WARNING "Couldn't find the 'ior' executable. Some tests will be skipped.") + set(HERMES_HAVE_IOR "NO") + endif() + enable_testing() + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) +endif() + #----------------------------------------------------------------------------- # Source #----------------------------------------------------------------------------- @@ -366,27 +384,6 @@ if(HERMES_ENABLE_DOXYGEN) ) endif() -#----------------------------------------------------------------------------- -# Testing -#----------------------------------------------------------------------------- -if(CMAKE_PROJECT_NAME STREQUAL HERMES) - include(CTest) -endif() - -if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) - find_package(Catch2 REQUIRED) - find_program(IOR_EXE ior) - if(IOR_EXE) - message(STATUS "Found ior at ${IOR_EXE}") - set(HERMES_HAVE_IOR "YES") - else() - message(WARNING "Couldn't find the 'ior' executable. Some tests will be skipped.") - set(HERMES_HAVE_IOR "NO") - endif() - enable_testing() - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) -endif() - #----------------------------------------------------------------------------- # Benchmarks #----------------------------------------------------------------------------- diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt index c078d791f..4514a82e8 100644 --- a/adapter/test/vfd/CMakeLists.txt +++ b/adapter/test/vfd/CMakeLists.txt @@ -1,7 +1,6 @@ set(HERMES_VFD_DIR ${HERMES_ADAPTER_DIR}/vfd) set(hermes_vfd_tests - hermes_driver hermes_env ) @@ -23,7 +22,7 @@ foreach(vfd_test ${hermes_vfd_tests}) set_property(TEST "Test${vfd_test}" APPEND PROPERTY ENVIRONMENT HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set_property(TEST "Test${vfd_test}" APPEND - PROPERTY ENVIRONMENT HDF5_DRIVER_CONFIG="true 1024") + PROPERTY ENVIRONMENT HDF5_DRIVER_CONFIG=true\ 1024) set_property(TEST "Test${vfd_test}" APPEND PROPERTY ENVIRONMENT HDF5_DRIVER=hermes) endforeach() diff --git a/adapter/test/vfd/hermes_env.c b/adapter/test/vfd/hermes_env.c index c5c56291c..9e2dc6e41 100644 --- a/adapter/test/vfd/hermes_env.c +++ b/adapter/test/vfd/hermes_env.c @@ -2,24 +2,22 @@ #include #include #include -#include #include +#include #include +/* HDF5 header for dynamic plugin loading */ +#include #include "H5FDhermes.h" -/* HDF5 header for dynamic plugin loading */ -#include "H5PLextern.h" - #define DATASETNAME "IntArray" #define NX 128 /* dataset dimensions */ #define NY 128 int main(int argc, char *argv[]) { - hid_t file_id, fapl_id; - hid_t dset_id, dcpl_id, dataspace_id; - hid_t dapl; + hid_t file_id; + hid_t dset_id, dataspace_id; hsize_t dims[2]; char *file_name = "hermes_test.h5"; int data_in[NX][NY]; /* data to write */ @@ -34,12 +32,8 @@ int main(int argc, char *argv[]) { return 1; } - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) - printf("H5Pcreate() error\n"); - else - printf("H5Pcreate() succeeded\n"); - - if ((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) + if ((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { printf("H5Fcreate() error\n"); } else { @@ -77,7 +71,7 @@ int main(int argc, char *argv[]) { printf("H5Fclose() succeeded\n"); hid_t fid; - if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) + if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) printf("H5Fopen() error\n"); else printf("H5Fopen() succeeded\n"); @@ -114,12 +108,7 @@ int main(int argc, char *argv[]) { else printf("H5Sclose() succeeded\n"); - if (H5Pclose(fapl_id) < 0) - printf("H5Pclose() error\n"); - else - printf("H5Pclose() succeeded\n"); - - printf("Done with Hermes set driver test!\n"); + remove(file_name); return 0; } diff --git a/src/buffer_pool.cc b/src/buffer_pool.cc index e1c4e065c..add821d35 100644 --- a/src/buffer_pool.cc +++ b/src/buffer_pool.cc @@ -109,7 +109,14 @@ void Finalize(SharedMemoryContext *context, CommunicationContext *comm, HERMES_DEBUG_SERVER_CLOSE(); } DestroyArena(trans_arena); - google::ShutdownGoogleLogging(); + // TODO(chogan): The VFD calls H5FD__hermes_term in an atexit() registered + // function. This means that the global static objects in GLOG have already + // been destroyed, so when we try to shutdown GLOG here it dereferences NULL + // pointers. The correct thing to do is to protect the shutdown call with an + // IsGoogleLoggingInitialized `if` statement. However, that API call was only + // added in a version of GLOG that is incompatible with OR-tools (they + // reimplemented their own version of GLOG). + // google::ShutdownGoogleLogging(); } void LockBuffer(BufferHeader *header) { From a0877e89379939e61eba98b0e155ce76caa36174 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Wed, 19 Jan 2022 13:32:46 -0600 Subject: [PATCH 05/16] Rearrange some CMake commands and disable GLOG init/shutdown --- CMakeLists.txt | 38 +++++++++++++++++++------------------- src/api/hermes.cc | 22 +++++++++++----------- src/rpc_thallium.cc | 4 ++-- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cd7cbc990..e174d22ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -325,11 +325,29 @@ if(HERMES_ENABLE_COVERAGE) endmacro() endif() +#----------------------------------------------------------------------------- +# Source +#----------------------------------------------------------------------------- + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) + +if(HERMES_HAVE_GOTCHA) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/gotcha_intercept) +endif() + +if(HERMES_ENABLE_WRAPPER) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/wrapper) +endif() + +if(HERMES_BUILD_BUFFER_POOL_VISUALIZER AND SDL2_FOUND) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/buffer_pool_visualizer) +endif() + #----------------------------------------------------------------------------- # Testing #----------------------------------------------------------------------------- +include(CTest) if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) - include(CTest) find_package(Catch2 REQUIRED) find_program(IOR_EXE ior) if(IOR_EXE) @@ -343,28 +361,10 @@ if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) endif() -#----------------------------------------------------------------------------- -# Source -#----------------------------------------------------------------------------- - -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) - -if(HERMES_HAVE_GOTCHA) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/gotcha_intercept) -endif() - if(HERMES_ENABLE_ADAPTERS) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/adapter) endif() -if(HERMES_ENABLE_WRAPPER) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/wrapper) -endif() - -if(HERMES_BUILD_BUFFER_POOL_VISUALIZER AND SDL2_FOUND) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/buffer_pool_visualizer) -endif() - #----------------------------------------------------------------------------- # Documentation #----------------------------------------------------------------------------- diff --git a/src/api/hermes.cc b/src/api/hermes.cc index 103980cb5..ec9f41834 100644 --- a/src/api/hermes.cc +++ b/src/api/hermes.cc @@ -250,23 +250,23 @@ BootstrapSharedMemory(Arena *arenas, Config *config, CommunicationContext *comm, return result; } -static void InitGlog() { - FLAGS_logtostderr = 1; - const char kMinLogLevel[] = "GLOG_minloglevel"; - char *min_log_level = getenv(kMinLogLevel); +// static void InitGlog() { +// FLAGS_logtostderr = 1; +// const char kMinLogLevel[] = "GLOG_minloglevel"; +// char *min_log_level = getenv(kMinLogLevel); - if (!min_log_level) { - FLAGS_minloglevel = 0; - } +// if (!min_log_level) { +// FLAGS_minloglevel = 0; +// } - FLAGS_v = 0; +// FLAGS_v = 0; - google::InitGoogleLogging("hermes"); -} +// google::InitGoogleLogging("hermes"); +// } std::shared_ptr InitHermes(Config *config, bool is_daemon, bool is_adapter) { - InitGlog(); + // InitGlog(); std::string base_shmem_name(config->buffer_pool_shmem_name); MakeFullShmemName(config->buffer_pool_shmem_name, base_shmem_name.c_str()); diff --git a/src/rpc_thallium.cc b/src/rpc_thallium.cc index 29d58ac0f..d147a5087 100644 --- a/src/rpc_thallium.cc +++ b/src/rpc_thallium.cc @@ -706,7 +706,7 @@ void RunDaemon(SharedMemoryContext *context, RpcContext *rpc, HERMES_DEBUG_SERVER_CLOSE(); DestroyArena(trans_arena); - google::ShutdownGoogleLogging(); + // google::ShutdownGoogleLogging(); } void FinalizeClient(SharedMemoryContext *context, RpcContext *rpc, @@ -731,7 +731,7 @@ void FinalizeClient(SharedMemoryContext *context, RpcContext *rpc, ReleaseSharedMemoryContext(context); HERMES_DEBUG_CLIENT_CLOSE(); DestroyArena(trans_arena); - google::ShutdownGoogleLogging(); + // google::ShutdownGoogleLogging(); } std::string GetRpcAddress(Config *config, const std::string &host_number, From b1e75c0d1eb2a256f62c253ff69d518ffb055108 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Thu, 20 Jan 2022 13:41:16 -0600 Subject: [PATCH 06/16] Build HDF5 in CI and enable VFD --- ci/install_deps.sh | 18 ++++++++++++++++++ ci/install_hermes.sh | 1 + 2 files changed, 19 insertions(+) diff --git a/ci/install_deps.sh b/ci/install_deps.sh index 5ba7f2247..675956f43 100755 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -12,9 +12,27 @@ GOTCHA_VERSION=develop CATCH2_VERSION=2.13.3 ORTOOLS_VERSION=7.7 SPACK_VERSION=0.16.3 +HDF5_VERSION=1_13_0 echo "Installing dependencies at ${INSTALL_DIR}" mkdir -p ${INSTALL_DIR} + +# HDF5 +# TODO: Use spack package once 1.13.0 is available in a release (currently +# only on the develop branch) +git clone https://github.com/HDFGroup/hdf5 +pushd hdf5 +git checkout hdf5-${HDF5_VERSION} +./autogen.sh +mkdir -p build +pushd build +CXXFLAGS=-I"${INSTALL_DIR}/include" LDFLAGS="-L${INSTALL_DIR}/lib -Wl,-rpath,${INSTALL_DIR}/lib" \ + ../configure --prefix=${INSTALL_DIR} +make && make install +popd +popd + +# Spack git clone https://github.com/spack/spack ${SPACK_DIR} pushd ${SPACK_DIR} git checkout v${SPACK_VERSION} diff --git a/ci/install_hermes.sh b/ci/install_hermes.sh index 55a7a6788..402380c12 100755 --- a/ci/install_hermes.sh +++ b/ci/install_hermes.sh @@ -29,6 +29,7 @@ cmake \ -DHERMES_USE_THREAD_SANITIZER=OFF \ -DHERMES_RPC_THALLIUM=ON \ -DHERMES_DEBUG_HEAP=OFF \ + -DHERMES_ENABLE_VFD=ON \ -DBUILD_TESTING=ON \ .. From a6b9b6507f6bfe3a8a8b3012b98b31e61694eada Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Thu, 20 Jan 2022 13:46:11 -0600 Subject: [PATCH 07/16] Add libtool to CI machine --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 088c3cb5b..9a3b73d00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,7 @@ jobs: sudo apt update sudo apt-get install -y autoconf sudo apt-get install -y automake + sudo apt-get install -y libtool sudo apt-get install -y mpich sudo apt-get install -y lcov sudo apt-get install -y zlib1g-dev From 3d120e3495080b63b3bd561ca4d059b6b0f4c497 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Thu, 20 Jan 2022 13:56:33 -0600 Subject: [PATCH 08/16] Run HDF5 autogen in verbose mode --- ci/install_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_deps.sh b/ci/install_deps.sh index 675956f43..ed5a63398 100755 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -23,7 +23,7 @@ mkdir -p ${INSTALL_DIR} git clone https://github.com/HDFGroup/hdf5 pushd hdf5 git checkout hdf5-${HDF5_VERSION} -./autogen.sh +bash autogen.sh -v mkdir -p build pushd build CXXFLAGS=-I"${INSTALL_DIR}/include" LDFLAGS="-L${INSTALL_DIR}/lib -Wl,-rpath,${INSTALL_DIR}/lib" \ From 032f600b55ed992697163fd3e3a0f0dd9bef6fa2 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Thu, 20 Jan 2022 14:01:03 -0600 Subject: [PATCH 09/16] Add libtool-bin to CI machines --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a3b73d00..0267f20f5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,6 +53,7 @@ jobs: sudo apt-get install -y autoconf sudo apt-get install -y automake sudo apt-get install -y libtool + sudo apt-get install -y libtool-bin sudo apt-get install -y mpich sudo apt-get install -y lcov sudo apt-get install -y zlib1g-dev From caf26f17dffb8579fa86ef3e499ed7f7973c92fd Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Thu, 20 Jan 2022 15:10:22 -0600 Subject: [PATCH 10/16] Reference issue for commented out init/shutdown of GLOG --- adapter/test/vfd/CMakeLists.txt | 13 +++++++++++++ ci/install_deps.sh | 4 ++-- src/api/hermes.cc | 26 +++++++++++++++----------- src/buffer_pool.cc | 8 +------- src/rpc_thallium.cc | 2 ++ 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt index 4514a82e8..489ebaf00 100644 --- a/adapter/test/vfd/CMakeLists.txt +++ b/adapter/test/vfd/CMakeLists.txt @@ -26,3 +26,16 @@ foreach(vfd_test ${hermes_vfd_tests}) set_property(TEST "Test${vfd_test}" APPEND PROPERTY ENVIRONMENT HDF5_DRIVER=hermes) endforeach() + +# IOR tests +if(HERMES_HAVE_IOR) + add_test(NAME "TestVFDWrite" COMMAND ${IOR_EXE} -a HDF5 -w -W -r -R) + set_property(TEST "TestVFDWrite" + PROPERTY ENVIRONMENT HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf) + set_property(TEST "TestVFDWrite" APPEND + PROPERTY ENVIRONMENT HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set_property(TEST "TestVFDWrite" APPEND + PROPERTY ENVIRONMENT HDF5_DRIVER_CONFIG=true\ 1024) + set_property(TEST "TestVFDWrite" APPEND + PROPERTY ENVIRONMENT HDF5_DRIVER=hermes) +endif() diff --git a/ci/install_deps.sh b/ci/install_deps.sh index ed5a63398..3f116d38c 100755 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -23,12 +23,12 @@ mkdir -p ${INSTALL_DIR} git clone https://github.com/HDFGroup/hdf5 pushd hdf5 git checkout hdf5-${HDF5_VERSION} -bash autogen.sh -v +bash autogen.sh mkdir -p build pushd build CXXFLAGS=-I"${INSTALL_DIR}/include" LDFLAGS="-L${INSTALL_DIR}/lib -Wl,-rpath,${INSTALL_DIR}/lib" \ ../configure --prefix=${INSTALL_DIR} -make && make install +make -j 4 && make install popd popd diff --git a/src/api/hermes.cc b/src/api/hermes.cc index ec9f41834..6446a4942 100644 --- a/src/api/hermes.cc +++ b/src/api/hermes.cc @@ -250,22 +250,26 @@ BootstrapSharedMemory(Arena *arenas, Config *config, CommunicationContext *comm, return result; } -// static void InitGlog() { -// FLAGS_logtostderr = 1; -// const char kMinLogLevel[] = "GLOG_minloglevel"; -// char *min_log_level = getenv(kMinLogLevel); - -// if (!min_log_level) { -// FLAGS_minloglevel = 0; -// } +// TODO(chogan): https://github.com/HDFGroup/hermes/issues/323 +#if 0 +static void InitGlog() { + FLAGS_logtostderr = 1; + const char kMinLogLevel[] = "GLOG_minloglevel"; + char *min_log_level = getenv(kMinLogLevel); + + if (!min_log_level) { + FLAGS_minloglevel = 0; + } -// FLAGS_v = 0; + FLAGS_v = 0; -// google::InitGoogleLogging("hermes"); -// } + google::InitGoogleLogging("hermes"); +} +#endif std::shared_ptr InitHermes(Config *config, bool is_daemon, bool is_adapter) { + // TODO(chogan): https://github.com/HDFGroup/hermes/issues/323 // InitGlog(); std::string base_shmem_name(config->buffer_pool_shmem_name); diff --git a/src/buffer_pool.cc b/src/buffer_pool.cc index add821d35..b0150bd8c 100644 --- a/src/buffer_pool.cc +++ b/src/buffer_pool.cc @@ -109,13 +109,7 @@ void Finalize(SharedMemoryContext *context, CommunicationContext *comm, HERMES_DEBUG_SERVER_CLOSE(); } DestroyArena(trans_arena); - // TODO(chogan): The VFD calls H5FD__hermes_term in an atexit() registered - // function. This means that the global static objects in GLOG have already - // been destroyed, so when we try to shutdown GLOG here it dereferences NULL - // pointers. The correct thing to do is to protect the shutdown call with an - // IsGoogleLoggingInitialized `if` statement. However, that API call was only - // added in a version of GLOG that is incompatible with OR-tools (they - // reimplemented their own version of GLOG). + // TODO(chogan): https://github.com/HDFGroup/hermes/issues/323 // google::ShutdownGoogleLogging(); } diff --git a/src/rpc_thallium.cc b/src/rpc_thallium.cc index d147a5087..123a29e7a 100644 --- a/src/rpc_thallium.cc +++ b/src/rpc_thallium.cc @@ -706,6 +706,7 @@ void RunDaemon(SharedMemoryContext *context, RpcContext *rpc, HERMES_DEBUG_SERVER_CLOSE(); DestroyArena(trans_arena); + // TODO(chogan): https://github.com/HDFGroup/hermes/issues/323 // google::ShutdownGoogleLogging(); } @@ -731,6 +732,7 @@ void FinalizeClient(SharedMemoryContext *context, RpcContext *rpc, ReleaseSharedMemoryContext(context); HERMES_DEBUG_CLIENT_CLOSE(); DestroyArena(trans_arena); + // TODO(chogan): https://github.com/HDFGroup/hermes/issues/323 // google::ShutdownGoogleLogging(); } From 8acfd6690a98896d0981c1f237675f2d3bc272a9 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 21 Jan 2022 10:42:28 -0600 Subject: [PATCH 11/16] Use static VFD --- adapter/test/vfd/CMakeLists.txt | 22 +++--- adapter/test/vfd/hermes_env.c | 114 ----------------------------- adapter/test/vfd/hermes_vfd_test.c | 90 +++++++++++++++++++++++ 3 files changed, 99 insertions(+), 127 deletions(-) delete mode 100644 adapter/test/vfd/hermes_env.c create mode 100644 adapter/test/vfd/hermes_vfd_test.c diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt index 489ebaf00..f225d8a0f 100644 --- a/adapter/test/vfd/CMakeLists.txt +++ b/adapter/test/vfd/CMakeLists.txt @@ -1,30 +1,26 @@ set(HERMES_VFD_DIR ${HERMES_ADAPTER_DIR}/vfd) set(hermes_vfd_tests - hermes_env + hermes_vfd_test ) foreach(vfd_test ${hermes_vfd_tests}) add_executable(${vfd_test} ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.c) + add_dependencies(${vfd_test} hermes) target_include_directories(${vfd_test} PRIVATE ${HERMES_VFD_DIR}) target_include_directories(${vfd_test} SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} ) - target_link_libraries(${vfd_test} - MPI::MPI_C - ${HDF5_HERMES_VFD_EXPORTED_LIBS} - ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} - ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} - ) +target_link_libraries(${vfd_test} + MPI::MPI_C + hermes + ${HDF5_HERMES_VFD_EXPORTED_LIBS} + ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} + ${HDF5_HERMES_VFD_EXT_PKG_DEPENDENCIES} +) add_test(NAME "Test${vfd_test}" COMMAND ${vfd_test}) set_property(TEST "Test${vfd_test}" PROPERTY ENVIRONMENT HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf) - set_property(TEST "Test${vfd_test}" APPEND - PROPERTY ENVIRONMENT HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - set_property(TEST "Test${vfd_test}" APPEND - PROPERTY ENVIRONMENT HDF5_DRIVER_CONFIG=true\ 1024) - set_property(TEST "Test${vfd_test}" APPEND - PROPERTY ENVIRONMENT HDF5_DRIVER=hermes) endforeach() # IOR tests diff --git a/adapter/test/vfd/hermes_env.c b/adapter/test/vfd/hermes_env.c deleted file mode 100644 index 9e2dc6e41..000000000 --- a/adapter/test/vfd/hermes_env.c +++ /dev/null @@ -1,114 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -/* HDF5 header for dynamic plugin loading */ -#include - -#include "H5FDhermes.h" - -#define DATASETNAME "IntArray" -#define NX 128 /* dataset dimensions */ -#define NY 128 - -int main(int argc, char *argv[]) { - hid_t file_id; - hid_t dset_id, dataspace_id; - hsize_t dims[2]; - char *file_name = "hermes_test.h5"; - int data_in[NX][NY]; /* data to write */ - int data_out[NX][NY]; /* data to read */ - int i, j, k; - - int mpi_threads_provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); - if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { - fprintf(stderr, - "Didn't receive appropriate MPI threading specification\n"); - return 1; - } - - if ((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, - H5P_DEFAULT)) - < 0) { - printf("H5Fcreate() error\n"); - } else { - printf("H5Fcreate() succeeded\n"); - } - - dims[0] = NX; - dims[1] = NY; - dataspace_id = H5Screate_simple(2, dims, NULL); - - dset_id = H5Dcreate(file_id, DATASETNAME, H5T_NATIVE_INT, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - for (j = 0; j < NX; j++) { - for (i = 0; i < NY; i++) { - data_in[j][i] = i + j; - data_out[j][i] = 0; - } - } - - if (H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data_in) < 0) - printf("H5Dwrite() error\n"); - else - printf("H5Dwrite() succeeded\n"); - - if (H5Dclose(dset_id) < 0) - printf("H5Dclose() error\n"); - else - printf("H5Dclose() succeeded\n"); - - if (H5Fclose(file_id) < 0) - printf("H5Fclose() error\n"); - else - printf("H5Fclose() succeeded\n"); - - hid_t fid; - if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) - printf("H5Fopen() error\n"); - else - printf("H5Fopen() succeeded\n"); - - if ((dset_id = H5Dopen(fid, DATASETNAME, H5P_DEFAULT)) < 0) - printf("H5Dopen() error\n"); - else - printf("H5Dopen() succeeded\n"); - - if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data_out) < 0) - printf("H5Dread() error\n"); - else - printf("H5Dread() succeeded\n"); - - for (j = 0; j < NX; j++) { - for (i = 0; i < NY; i++) { - assert(data_out[j][i] == data_in[j][i]); - } - } - - if (H5Dclose(dset_id) < 0) - printf("H5Dclose() error\n"); - else - printf("H5Dclose() succeeded\n"); - - if (H5Fclose(fid) < 0) - printf("H5Fclose() error\n"); - else - printf("H5Fclose() succeeded\n"); - - if (H5Sclose(dataspace_id) < 0) - printf("H5Sclose() error\n"); - else - printf("H5Sclose() succeeded\n"); - - remove(file_name); - - return 0; -} diff --git a/adapter/test/vfd/hermes_vfd_test.c b/adapter/test/vfd/hermes_vfd_test.c new file mode 100644 index 000000000..1fca7a98b --- /dev/null +++ b/adapter/test/vfd/hermes_vfd_test.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include + +#include +#include +/* HDF5 header for dynamic plugin loading */ +#include + +#include "H5FDhermes.h" + +void AssertFunc(bool expr, const char *file, int lineno, const char *message) { + if (!expr) { + fprintf(stderr, "Assertion failed at %s: line %d: %s\n", file, lineno, + message); + exit(-1); + } +} + +#define Assert(expr) AssertFunc((expr), __FILE__, __LINE__, #expr) + +int main(int argc, char *argv[]) { + hid_t file_id; + hid_t dset_id; + hid_t dataspace_id; + hid_t fapl_id; + hsize_t dims[2]; + char *file_name = "hermes_test.h5"; + const int kNx = 128; + const int kNy = 128; + int data_in[kNx][kNy]; /* data to write */ + int data_out[kNx][kNy]; /* data to read */ + int i, j, k; + + int mpi_threads_provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); + if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { + fprintf(stderr, + "Didn't receive appropriate MPI threading specification\n"); + return 1; + } + + const char kDatasetName[] = "IntArray"; + dims[0] = kNx; + dims[1] = kNy; + + Assert((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) >= 0); + Assert(H5Pset_fapl_hermes(fapl_id, true, 1024) >= 0); + + Assert((file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, + fapl_id)) >= 0); + Assert((dataspace_id = H5Screate_simple(2, dims, NULL)) >= 0); + dset_id = H5Dcreate(file_id, kDatasetName, H5T_NATIVE_INT, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + Assert(dset_id >= 0); + + for (j = 0; j < kNx; j++) { + for (i = 0; i < kNy; i++) { + data_in[j][i] = i + j; + data_out[j][i] = 0; + } + } + + Assert(H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_in) >= 0); + Assert(H5Dclose(dset_id) >= 0); + Assert(H5Fclose(file_id) >= 0); + + hid_t fid; + Assert((fid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) >= 0); + Assert((dset_id = H5Dopen(fid, kDatasetName, H5P_DEFAULT)) >= 0); + Assert(H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_out) >= 0); + + for (j = 0; j < kNx; j++) { + for (i = 0; i < kNy; i++) { + Assert(data_out[j][i] == data_in[j][i]); + } + } + + Assert(H5Dclose(dset_id) >= 0); + Assert(H5Fclose(fid) >= 0); + Assert(H5Sclose(dataspace_id) >= 0); + Assert(H5Pclose(fapl_id) >= 0); + remove(file_name); + + return 0; +} From e4c1ba0f930419fa4f5b66762dd329d35d6bf617 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 21 Jan 2022 10:46:52 -0600 Subject: [PATCH 12/16] Remove redundant test --- adapter/test/vfd/hermes_driver.c | 130 ------------------------------- 1 file changed, 130 deletions(-) delete mode 100644 adapter/test/vfd/hermes_driver.c diff --git a/adapter/test/vfd/hermes_driver.c b/adapter/test/vfd/hermes_driver.c deleted file mode 100644 index 79f9e0308..000000000 --- a/adapter/test/vfd/hermes_driver.c +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "hdf5.h" -#include "H5FDhermes.h" - -#define DATASETNAME "IntArray" -#define NX 128 /* dataset dimensions */ -#define NY 128 - -int main(int argc, char *argv[]) { - hid_t file_id, fapl_id; - hid_t dset_id, dcpl_id, dataspace_id; - hid_t dapl; - hsize_t dims[2]; - char *file_name = "hermes_test.h5"; - int data_in[NX][NY]; /* data to write */ - int data_out[NX][NY]; /* data to read */ - int i, j, k; - - const char *const config_str = "true 1024"; - - int mpi_threads_provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); - if (mpi_threads_provided < MPI_THREAD_MULTIPLE) { - fprintf(stderr, - "Didn't receive appropriate MPI threading specification\n"); - return 1; - } - - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) - printf("H5Pcreate() error\n"); - else - printf("H5Pcreate() succeeded\n"); - - if (H5Pset_driver_by_name(fapl_id, "hermes", config_str) < 0) - printf("H5Pset_driver_by_name() error\n"); - else - printf("H5Pset_driver_by_name() succeeded\n"); - - if ((file_id = - H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) { - printf("H5Fcreate() error\n"); - } else { - printf("H5Fcreate() succeeded\n"); - } - - dims[0] = NX; - dims[1] = NY; - dataspace_id = H5Screate_simple(2, dims, NULL); - - dset_id = H5Dcreate(file_id, DATASETNAME, H5T_NATIVE_INT, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - for (j = 0; j < NX; j++) { - for (i = 0; i < NY; i++) { - data_in[j][i] = i + j; - data_out[j][i] = 0; - } - } - - if (H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data_in) < 0) { - printf("H5Dwrite() error\n"); - } else { - printf("H5Dwrite() succeeded\n"); - } - - if (H5Dclose(dset_id) < 0) - printf("H5Dclose() error\n"); - else - printf("H5Dclose() succeeded\n"); - - if (H5Fclose(file_id) < 0) - printf("H5Fclose() error\n"); - else - printf("H5Fclose() succeeded\n"); - - hid_t fid; - if ((fid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) - printf("H5Fopen() error\n"); - else - printf("H5Fopen() succeeded\n"); - - if ((dset_id = H5Dopen(fid, DATASETNAME, H5P_DEFAULT)) < 0) - printf("H5Dopen() error\n"); - else - printf("H5Dopen() succeeded\n"); - - if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data_out) < 0) { - printf("H5Dread() error\n"); - } else { - printf("H5Dread() succeeded\n"); - } - - for (j = 0; j < NX; j++) { - for (i = 0; i < NY; i++) { - assert(data_out[j][i] == data_in[j][i]); - } - } - - if (H5Dclose(dset_id) < 0) - printf("H5Dclose() error\n"); - else - printf("H5Dclose() succeeded\n"); - - if (H5Fclose(fid) < 0) - printf("H5Fclose() error\n"); - else - printf("H5Fclose() succeeded\n"); - - if (H5Sclose(dataspace_id) < 0) - printf("H5Sclose() error\n"); - else - printf("H5Sclose() succeeded\n"); - - if (H5Pclose(fapl_id) < 0) - printf("H5Pclose() error\n"); - else - printf("H5Pclose() succeeded\n"); - - printf("Done with Hermes set driver test!\n"); - - return 0; -} From 706853ea0f60124d8216316a83850405fcbe1e9f Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 21 Jan 2022 12:58:09 -0600 Subject: [PATCH 13/16] Compile VFD test as C++ --- adapter/test/vfd/CMakeLists.txt | 4 ++-- adapter/test/vfd/{hermes_vfd_test.c => hermes_vfd_test.cc} | 4 ++-- adapter/vfd/CMakeLists.txt | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) rename adapter/test/vfd/{hermes_vfd_test.c => hermes_vfd_test.cc} (97%) diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt index f225d8a0f..f3ec15568 100644 --- a/adapter/test/vfd/CMakeLists.txt +++ b/adapter/test/vfd/CMakeLists.txt @@ -5,14 +5,14 @@ set(hermes_vfd_tests ) foreach(vfd_test ${hermes_vfd_tests}) - add_executable(${vfd_test} ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.c) + add_executable(${vfd_test} ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.cc) add_dependencies(${vfd_test} hermes) target_include_directories(${vfd_test} PRIVATE ${HERMES_VFD_DIR}) target_include_directories(${vfd_test} SYSTEM PUBLIC ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES} ) target_link_libraries(${vfd_test} - MPI::MPI_C + MPI::MPI_CXX hermes ${HDF5_HERMES_VFD_EXPORTED_LIBS} ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES} diff --git a/adapter/test/vfd/hermes_vfd_test.c b/adapter/test/vfd/hermes_vfd_test.cc similarity index 97% rename from adapter/test/vfd/hermes_vfd_test.c rename to adapter/test/vfd/hermes_vfd_test.cc index 1fca7a98b..1057ed6c9 100644 --- a/adapter/test/vfd/hermes_vfd_test.c +++ b/adapter/test/vfd/hermes_vfd_test.cc @@ -27,12 +27,12 @@ int main(int argc, char *argv[]) { hid_t dataspace_id; hid_t fapl_id; hsize_t dims[2]; - char *file_name = "hermes_test.h5"; + const char *file_name = "hermes_test.h5"; const int kNx = 128; const int kNy = 128; int data_in[kNx][kNy]; /* data to write */ int data_out[kNx][kNy]; /* data to read */ - int i, j, k; + int i, j; int mpi_threads_provided; MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_threads_provided); diff --git a/adapter/vfd/CMakeLists.txt b/adapter/vfd/CMakeLists.txt index 89587ac6c..3de478b4b 100644 --- a/adapter/vfd/CMakeLists.txt +++ b/adapter/vfd/CMakeLists.txt @@ -4,8 +4,6 @@ project(HDF5_HERMES_VFD) -# include_directories(${PROJECT_BINARY_DIR}) - #------------------------------------------------------------------------------ # Version information #------------------------------------------------------------------------------ From b74a79389a9caf737179dbbbe36ddeb86d190848 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 21 Jan 2022 14:25:34 -0600 Subject: [PATCH 14/16] Fix memory leaks --- adapter/vfd/H5FDhermes.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/adapter/vfd/H5FDhermes.c b/adapter/vfd/H5FDhermes.c index 293cdc8fe..a9f595a16 100644 --- a/adapter/vfd/H5FDhermes.c +++ b/adapter/vfd/H5FDhermes.c @@ -486,6 +486,8 @@ H5FD__hermes_open(const char *name, unsigned flags, hid_t fapl_id, if (file) { HermesBucketDestroy(file->bkt_handle); free(file->blob_in_bucket.blobs); + free(file->bktname); + free(file->page_buf); free(file); } } /* end if */ @@ -552,6 +554,8 @@ static herr_t H5FD__hermes_close(H5FD_t *_file) { /* Release the file info */ free(file->blob_in_bucket.blobs); + free(file->page_buf); + free(file->bktname); free(file); done: From 87b447d9ca982dbd96cf77bc7542c1e65e0d1061 Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 21 Jan 2022 14:25:56 -0600 Subject: [PATCH 15/16] Fix logic of unsetting -fsanitize=address --- CMakeLists.txt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e174d22ef..4d5493585 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,10 +184,11 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug") CACHE STRING "" FORCE) endif() - if (HERMES_USE_ADDRESS_SANITIZER AND - NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=address.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address" + if (HERMES_USE_ADDRESS_SANITIZER) + if(NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=address.*") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address" CACHE STRING "" FORCE) + endif() else() string(REPLACE "-fsanitize=address" "" FLAGS_NO_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG}") @@ -198,10 +199,11 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug") message(FATAL_ERROR "Cannont use -fsanitize=address and -fsanitize=thread " "at the same time") else() - if (HERMES_USE_THREAD_SANITIZER AND - NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=thread.*") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=thread" - CACHE STRING "" FORCE) + if (HERMES_USE_THREAD_SANITIZER) + if(NOT "${CMAKE_CXX_FLAGS_DEBUG}" MATCHES ".*-fsanitize=thread.*") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=thread" + CACHE STRING "" FORCE) + endif() else() string(REPLACE "-fsanitize=thread" "" FLAGS_NO_SANITIZE "${CMAKE_CXX_FLAGS_DEBUG}") From 370a978fc29de7c3faf7c6e8ff811f4cc79412ae Mon Sep 17 00:00:00 2001 From: Chris Hogan Date: Fri, 21 Jan 2022 14:26:17 -0600 Subject: [PATCH 16/16] Properly set asan env vars in VFD test --- adapter/test/vfd/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/adapter/test/vfd/CMakeLists.txt b/adapter/test/vfd/CMakeLists.txt index f3ec15568..0dda6e914 100644 --- a/adapter/test/vfd/CMakeLists.txt +++ b/adapter/test/vfd/CMakeLists.txt @@ -21,6 +21,8 @@ target_link_libraries(${vfd_test} add_test(NAME "Test${vfd_test}" COMMAND ${vfd_test}) set_property(TEST "Test${vfd_test}" PROPERTY ENVIRONMENT HERMES_CONF=${CMAKE_CURRENT_SOURCE_DIR}/hermes.conf) + set_property(TEST "Test${vfd_test}" APPEND + PROPERTY ENVIRONMENT LSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/test/data/asan.supp) endforeach() # IOR tests