From 2b20db30787351c8a6646d4826716c9bd1210703 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 27 Jul 2023 22:44:48 +0200 Subject: [PATCH 01/45] Fix build with MariaDB connector --- src/drivers/mysql/drv_mysql.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index 5e9afba2..e048dbaa 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -415,9 +415,10 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con) { DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_COMPRESS", "NULL"); mysql_options(con, MYSQL_OPT_COMPRESS, NULL); - +#ifdef MYSQL_OPT_COMPRESSION_ALGORITHMS DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_COMPRESSION_ALGORITHMS", args.compression_alg); mysql_options(con, MYSQL_OPT_COMPRESSION_ALGORITHMS, args.compression_alg); +#endif } DEBUG("mysql_real_connect(%p, \"%s\", \"%s\", \"%s\", \"%s\", %u, \"%s\", %s)", From e876cef8bfdc88c6803222d953c0a1701f09a3b0 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 02/45] sysbench cmake port With this patch, sysbench can be built (on Linux, macOS, or FreeBSD at least) using cmake. Packaging using cpack works, too. --- CMakeLists.txt | 34 ++++++ cmake/BuildConcurrencyKit.cmake | 27 +++++ cmake/BuildLuaJit.cmake | 41 +++++++ cmake/FindConcurrencyKit.cmake | 27 +++++ cmake/FindLuaJit.cmake | 53 +++++++++ cmake/FindMySQL.cmake | 140 ++++++++++++++++++++++ cmake/compile_flags.cmake | 25 ++++ cmake/githash.cmake | 26 ++++ cmake/sanitizer.cmake | 38 ++++++ cmake/systemchecks.cmake | 45 +++++++ src/CMakeLists.txt | 203 ++++++++++++++++++++++++++++++++ src/config.h.in | 67 +++++++++++ src/lua/CMakeLists.txt | 17 +++ src/lua/internal/CMakeLists.txt | 28 +++++ src/sb_timer.h | 2 +- src/sysbenchConfig.cmake.in | 15 +++ 16 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 CMakeLists.txt create mode 100644 cmake/BuildConcurrencyKit.cmake create mode 100644 cmake/BuildLuaJit.cmake create mode 100644 cmake/FindConcurrencyKit.cmake create mode 100644 cmake/FindLuaJit.cmake create mode 100644 cmake/FindMySQL.cmake create mode 100644 cmake/compile_flags.cmake create mode 100644 cmake/githash.cmake create mode 100644 cmake/sanitizer.cmake create mode 100644 cmake/systemchecks.cmake create mode 100644 src/CMakeLists.txt create mode 100644 src/config.h.in create mode 100644 src/lua/CMakeLists.txt create mode 100644 src/lua/internal/CMakeLists.txt create mode 100644 src/sysbenchConfig.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..b7a6699d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.12..3.26) + +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) +if(APPLE) + # Add homebrew search paths on Mac + foreach(pkg openssl mysql-client mariadb-connector-c libpq) + set(pkgpath /usr/local/opt/${pkg}) + if(EXISTS ${pkgpath}) + list(APPEND CMAKE_PREFIX_PATH ${pkg}) + endif() + endforeach() +endif() + +project( + sysbench + VERSION "1.1.0" + LANGUAGES C) + +# Set default build type, if none is given +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(default_build_type Release) + message( + STATUS + "Setting build type to '${default_build_type}' as none was specified.") + set(CMAKE_BUILD_TYPE + "${default_build_type}" + CACHE STRING "Choose the type of build." FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") +endif() +include(compile_flags) +include(sanitizer) +add_subdirectory(src) diff --git a/cmake/BuildConcurrencyKit.cmake b/cmake/BuildConcurrencyKit.cmake new file mode 100644 index 00000000..a35a7c84 --- /dev/null +++ b/cmake/BuildConcurrencyKit.cmake @@ -0,0 +1,27 @@ +# Compile concurrency kit from source Currently, we do not even need to link the +# library we only use functionality from headers only + +set(build_dir ${PROJECT_BINARY_DIR}/build/third_party/concurrency_kit/ck) +set(install_dir ${build_dir}/install) + +include(ExternalProject) +ExternalProject_Add( + buildConcurrencyKit + URL ${PROJECT_SOURCE_DIR}/third_party/concurrency_kit/ck + SOURCE_DIR ${build_dir} + CONFIGURE_COMMAND ./configure --prefix=${install_dir} + BUILD_BYPRODUCTS ${install_dir}/lib/libck.a + BUILD_COMMAND make + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + BUILD_ALWAYS FALSE) + +add_library(ConcurrencyKit STATIC IMPORTED) +add_dependencies(ConcurrencyKit buildConcurrencyKit) +make_directory(${install_dir}/include) +set_target_properties( + ConcurrencyKit + PROPERTIES IMPORTED_LOCATION + ${install_dir}/lib/libck.a + INTERFACE_INCLUDE_DIRECTORIES + ${install_dir}/include) diff --git a/cmake/BuildLuaJit.cmake b/cmake/BuildLuaJit.cmake new file mode 100644 index 00000000..d10586ef --- /dev/null +++ b/cmake/BuildLuaJit.cmake @@ -0,0 +1,41 @@ +# Build the bundled luajit library and create imported library. +# luajit::libluajit. +# +# This follows the instructions on the luajit page +include(ExternalProject) + +set(build_dir ${PROJECT_BINARY_DIR}/build/third_party/luajit) +set(install_dir ${build_dir}/install) +set(url ${PROJECT_SOURCE_DIR}/third_party/luajit/luajit) + + +# Note : luajit's Makefile is non-portable (GNU)it could fail on BSD, unless +# gmake is there´ +find_program(MAKE_EXECUTABLE NAMES gmake make REQUIRED) +# Makefile requires MACOSX_DEPLOYMENT_TARGET on Apple +if(CMAKE_OSX_DEPLOYMENT_TARGET) + set(MACOSX_DEPLOYMENT_TARGET + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") +endif() +set(lubluajit_static_lib ${install_dir}/lib/libluajit-5.1.a) +ExternalProject_Add( + buildLuaJit + URL ${url} + SOURCE_DIR ${build_dir} + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_ALWAYS FALSE + BUILD_COMMAND ${MAKE_EXECUTABLE} install PREFIX=${install_dir} ${MACOSX_DEPLOYMENT_TARGET} + BUILD_BYPRODUCTS ${libluajit_static_lib}) +add_library(luajit::libluajit STATIC IMPORTED) +make_directory(${install_dir}/include/luajit-2.1) +set_target_properties( + luajit::libluajit + PROPERTIES IMPORTED_LOCATION + ${lubluajit_static_lib} + INTERFACE_INCLUDE_DIRECTORIES + ${install_dir}/include/luajit-2.1 + INTERFACE_LINK_LIBRARIES "m;${CMAKE_DL_LIBS}") +add_dependencies(luajit::libluajit buildLuaJit) + diff --git a/cmake/FindConcurrencyKit.cmake b/cmake/FindConcurrencyKit.cmake new file mode 100644 index 00000000..723ad6d4 --- /dev/null +++ b/cmake/FindConcurrencyKit.cmake @@ -0,0 +1,27 @@ +find_package(PkgConfig) +if(PKG_CONFIG_FOUND) + pkg_check_modules(PC_CK QUIET ck) +endif() + +find_path( + CK_INCLUDE_DIR + NAMES ck_md.h + PATHS ${PC_CK_INCLUDE_DIRS} + PATH_SUFFIXES ck) +find_library( + CK_LIBRARY + NAMES ck + PATHS ${PC_CK_LIBRARY_DIRS}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ConcurrencyKit + FOUND_VAR ConcurrencyKit_FOUND + REQUIRED_VARS CK_LIBRARY CK_INCLUDE_DIR) +mark_as_advanced(CK_LIBRARY CK_INCLUDE_DIR) +if(ConcurrencyKit_FOUND AND NOT TARGET ConcurrencyKit) + add_library(ConcurrencyKit UNKNOWN IMPORTED) + set_target_properties( + ConcurrencyKit PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CK_INCLUDE_DIR} + IMPORTED_LOCATION ${CK_LIBRARY}) +endif() diff --git a/cmake/FindLuaJit.cmake b/cmake/FindLuaJit.cmake new file mode 100644 index 00000000..41e4dff3 --- /dev/null +++ b/cmake/FindLuaJit.cmake @@ -0,0 +1,53 @@ +# Locate LuaJIT library +# This module defines +# LUAJIT_FOUND, if false, do not try to link to Lua +# LUA_LIBRARY, where to find the lua library +# LUA_INCLUDE_DIR, where to find lua.h +# +# This module is similar to FindLua51.cmake except that it finds LuaJit instead. + +FIND_PATH(LUA_INCLUDE_DIR luajit.h + HINTS + $ENV{LUA_DIR} + PATH_SUFFIXES include/luajit-2.1 include/luajit-2.0 include/luajit-5_1-2.1 include/luajit-5_1-2.0 include luajit + PATHS + ~/Library/Frameworks + /Library/Frameworks + /sw # Fink + /opt/local # DarwinPorts + /opt/csw # Blastwave + /opt +) + + +FIND_LIBRARY(LUA_LIBRARY + NAMES + luajit-5.1 + HINTS + $ENV{LUA_DIR} + PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /sw + /opt/local + /opt/csw + /opt + ) + +INCLUDE(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set LuaJit_FOUND to TRUE if +# all listed variables exist +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LuaJit + REQUIRED_VARS LUA_LIBRARY LUA_INCLUDE_DIR) + +MARK_AS_ADVANCED(LUA_INCLUDE_DIR LUA_LIBRARY) + +if (LuaJit_FOUND AND NOT TARGET luajit::libluajit) + add_library(luajit::libluajit UNKNOWN IMPORTED) + set_target_properties(luajit::libluajit PROPERTIES + IMPORTED_LOCATION "${LUA_LIBRARY}" + INTERFACE_LINK_LIBRARIES "m;${CMAKE_DL_LIBS}" # Linuxism? + set_target_properties(luajit::libluajit PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LUA_INCLUDE_DIR}") +endif() diff --git a/cmake/FindMySQL.cmake b/cmake/FindMySQL.cmake new file mode 100644 index 00000000..104c3a9f --- /dev/null +++ b/cmake/FindMySQL.cmake @@ -0,0 +1,140 @@ +# This comes from https://github.com/Kitware/VTK/blob/master/CMake/FindMySQL.cmake + +#[==[ +Provides the following variables: + + * `MySQL_INCLUDE_DIRS`: Include directories necessary to use MySQL. + * `MySQL_LIBRARIES`: Libraries necessary to use MySQL. + * A `MySQL::MySQL` imported target. +#]==] + +# No .pc files are shipped with MySQL on Windows. +set(_MySQL_use_pkgconfig 0) +if (NOT WIN32) + find_package(PkgConfig) + if (PkgConfig_FOUND) + set(_MySQL_use_pkgconfig 1) + endif () +endif () + +if (_MySQL_use_pkgconfig) + pkg_check_modules(_libmariadb "libmariadb" QUIET IMPORTED_TARGET) + unset(_mysql_target) + if (_libmariadb_FOUND) + set(_mysql_target "_libmariadb") + else () + pkg_check_modules(_mariadb "mariadb" QUIET IMPORTED_TARGET) + if (NOT _mariadb_FOUND) + pkg_check_modules(_mysql "mysql" QUIET IMPORTED_TARGET) + if (_mysql_FOUND) + set(_mysql_target "_mysql") + endif () + else () + set(_mysql_target "_mariadb") + if (_mariadb_VERSION VERSION_LESS 10.4) + get_property(_include_dirs + TARGET "PkgConfig::_mariadb" + PROPERTY "INTERFACE_INCLUDE_DIRECTORIES") + # Remove "${prefix}/mariadb/.." from the interface since it breaks other + # projects. + list(FILTER _include_dirs EXCLUDE REGEX "\\.\\.") + set_property(TARGET "PkgConfig::_mariadb" + PROPERTY + "INTERFACE_INCLUDE_DIRECTORIES" "${_include_dirs}") + unset(_include_dirs) + endif () + endif () + endif () + + set(MySQL_FOUND 0) + if (_mysql_target) + set(MySQL_FOUND 1) + set(MySQL_INCLUDE_DIRS ${${_mysql_target}_INCLUDE_DIRS}) + set(MySQL_LIBRARIES ${${_mysql_target}_LINK_LIBRARIES}) + if (NOT TARGET MySQL::MySQL) + add_library(MySQL::MySQL INTERFACE IMPORTED) + target_link_libraries(MySQL::MySQL + INTERFACE "PkgConfig::${_mysql_target}") + endif () + endif () + unset(_mysql_target) + unset(_MySQL_use_pkgconfig) + if (TARGET MySQL::MySQL) + return() + endif() +endif() + + +set(_MySQL_mariadb_versions 10.2 10.3) +set(_MySQL_versions 5.0) +set(_MySQL_paths) +foreach (_MySQL_version IN LISTS _MySQL_mariadb_versions) + list(APPEND _MySQL_paths + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MariaDB ${_MySQL_version};INSTALLDIR]" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MariaDB ${_MySQL_version} (x64);INSTALLDIR]") +endforeach () +foreach (_MySQL_version IN LISTS _MySQL_versions) + list(APPEND _MySQL_paths + "C:/Program Files/MySQL/MySQL Server ${_MySQL_version}/lib/opt" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MySQL AB\\MySQL Server ${_MySQL_version};Location]" + "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\MySQL AB\\MySQL Server ${_MySQL_version};Location]") +endforeach () +unset(_MySQL_version) +unset(_MySQL_versions) +unset(_MySQL_mariadb_versions) +find_path(MySQL_INCLUDE_DIR + NAMES mysql.h + PATHS + "C:/Program Files/MySQL/include" + "C:/MySQL/include" + ${_MySQL_paths} + PATH_SUFFIXES include include/mysql + DOC "Location of mysql.h") +mark_as_advanced(MySQL_INCLUDE_DIR) +find_library(MySQL_LIBRARY + NAMES libmariadb mysql libmysql mysqlclient + PATHS + "C:/Program Files/MySQL/lib" + "C:/MySQL/lib/debug" + ${_MySQL_paths} + PATH_SUFFIXES lib lib/opt + DOC "Location of the mysql library") + +if (MySQL_LIBRARY MATCHES "libmariadb.lib$") + set(MYSQL_DLL libmariadb.dll) +elseif (MYSQL_LIBRARY MATCHES "libmysql.lib$") + set(MYSQL_DLL libmysql.dll) +endif() +if(MYSQL_DLL) + find_file(MYSQL_DLL_PATH + NAMES ${MYSQL_DLL} + PATHS + PATH_SUFFIXES ../../bin + ) + set(MYSQL_DLL_PATH_VAR MYSQL_DLL_PATH) +ENDIF() +mark_as_advanced(MySQL_LIBRARY) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MySQL + REQUIRED_VARS MySQL_INCLUDE_DIR MySQL_LIBRARY ${MYSQL_DLL_PATH_VAR}) + +if (MySQL_FOUND) + set(MySQL_INCLUDE_DIRS "${MySQL_INCLUDE_DIR}") + set(MySQL_LIBRARIES "${MySQL_LIBRARY}") + if (NOT TARGET MySQL::MySQL) + if (MYSQL_DLL_PATH) + add_library(MySQL::MySQL SHARED IMPORTED) + set_target_properties(MySQL::MySQL PROPERTIES + IMPORTED_IMPLIB "${MySQL_LIBRARY}" + IMPORTED_LOCATION "${MYSQL_DLL_PATH}" + INTERFACE_INCLUDE_DIRECTORIES "${MySQL_INCLUDE_DIR}") + else() + add_library(MySQL::MySQL UNKNOWN IMPORTED) + set_target_properties(MySQL::MySQL PROPERTIES + IMPORTED_LOCATION "${MySQL_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${MySQL_INCLUDE_DIR}") + endif() + endif() +endif() +unset(_MySQL_use_pkgconfig) diff --git a/cmake/compile_flags.cmake b/cmake/compile_flags.cmake new file mode 100644 index 00000000..c4647933 --- /dev/null +++ b/cmake/compile_flags.cmake @@ -0,0 +1,25 @@ +if((CMAKE_C_COMPILER_ID STREQUAL GNU) OR (CMAKE_C_COMPILER_ID MATCHES "Clang")) + add_compile_options( + -Wall + -Wextra + -Wpointer-arith + -Wbad-function-cast + -Wstrict-prototypes + -Wnested-externs + -Wno-format-zero-length + -Wundef + -Wstrict-prototypes + -Wmissing-prototypes + -Wmissing-declarations + -Wredundant-decls + -Wcast-align + -Wvla) +endif() + +if(CMAKE_COMPILE_WARNING_AS_ERROR AND (CMAKE_VERSION VERSION_LESS "3.24")) + if(MSVC) + add_compile_options(-WX) + elseif(CMAKE_COMPILER_IS_GNUC) + add_compile_options(-Werror) + endif() +endif() diff --git a/cmake/githash.cmake b/cmake/githash.cmake new file mode 100644 index 00000000..ee11c113 --- /dev/null +++ b/cmake/githash.cmake @@ -0,0 +1,26 @@ +# Sets variable with the given name to the git hash of the last commit, or empty +# string on any error +function(GitHash varname) + set("${varname}" + "" + PARENT_SCOPE) + if(NOT EXISTS ${PROJECT_SOURCE_DIR}/.git) + return() + endif() + find_package(Git QUIET) + if(NOT Git_FOUND) + return() + endif() + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + OUTPUT_VARIABLE _git_hash + RESULT_VARIABLE result) + if(NOT (result EQUAL 0)) + return() + endif() + string(REGEX REPLACE "\n$" "" _git_hash "${_git_hash}") + set(${varname} + ${_git_hash} + PARENT_SCOPE) +endfunction() diff --git a/cmake/sanitizer.cmake b/cmake/sanitizer.cmake new file mode 100644 index 00000000..cadcd9c3 --- /dev/null +++ b/cmake/sanitizer.cmake @@ -0,0 +1,38 @@ +option(WITH_ASAN OFF "Enable Address sanitizer") + +if(WITH_ASAN) + add_compile_options(-fsanitize=address) + if(MSVC AND (NOT CMAKE_C_COMPILER_ID MATCHES "Clang")) + add_link_options(-INCREMENTAL:NO) + else() + add_link_options(-fsanitize=address) + endif() +endif() + +if(MSVC) + return() +endif() + +option(WITH_TSAN OFF "Enable thread sanitizer") +if(WITH_TSAN) + add_compile_options(-fsanitize=thread) + add_link_options(-fsanitize=thread) +endif() + +option(WITH_UBSAN OFF "Enable undefined behavior sanitizer") +if(WITH_UBSAN) + add_compile_options(-fsanitize=undefined) + add_link_options(-fsanitize=undefined) +endif() +option(WITH_MSAN OFF "Emable memory sanitizer") +if(WITH_MSAN) + add_compile_options(-fsanitize=memory) + add_link_options(-fsanitize=memory) +endif() + +if(CMAKE_COMPILER_IS_GNUCC + AND (WITH_ASAN + OR WITH_UBSAN + OR WITH_TSAN)) + add_compile_options(-fno-omit-frame-pointer -g) +endif() diff --git a/cmake/systemchecks.cmake b/cmake/systemchecks.cmake new file mode 100644 index 00000000..d318f6bf --- /dev/null +++ b/cmake/systemchecks.cmake @@ -0,0 +1,45 @@ +# System checks +include(CheckIncludeFile) +include(CheckFunctionExists) +include(CheckTypeSize) +include(CheckSymbolExists) +include(CheckFunctionExists) +check_function_exists(clock_gettime HAVE_CLOCK_GETTIME) +check_function_exists(alarm HAVE_ALARM) +check_function_exists(directio HAVE_DIRECTIO) +check_include_file(errno.h HAVE_ERRNO_H) +check_include_file(fcntl.h HAVE_FCNTL_H) +check_function_exists(fdatasync HAVE_FDATASYNC) +if(CMAKE_COMPILER_IS_GNUCC) + set(HAVE_FUNC_ATTRIBUTE_FORMAT 1) + set(HAVE_FUNC_ATTRIBUTE_UNUSED 1) +endif() +check_function_exists(isatty HAVE_ISATTY) +find_library(aio HAVE_LIBAIO) +check_include_file(libgen.h HAVE_LIBGEN_H) +check_include_file(limits.h HAVE_LIMITS_H) +check_include_file(math.h HAVE_MATH_H) +check_symbol_exists(SHM_HUGETLB sys/shm.h HAVE_LARGE_PAGES) +check_function_exists(posix_memalign HAVE_POSIX_MEMALIGN) +check_include_file(pthread.h HAVE_PTHREAD_H) +check_symbol_exists(pthread_cancel pthread.h HAVE_PTHREAD_CANCEL) +check_symbol_exists(pthread_yield pthread.h HAVE_PTHREAD_YIELD) +check_include_file(sched.h HAVE_SCHED_H) +check_function_exists(setvbuf HAVE_SETVBUF) +check_include_file(signal.h HAVE_SIGNAL_H) +check_function_exists(strerror_r HAVE_STRERROR_R) +check_include_file(strings.h HAVE_STRINGS_H) +check_include_file(string.h HAVE_STRING_H) +check_include_file(sys/ipc.h HAVE_SYS_IPC_H) +check_include_file(sys/mman.h HAVE_SYS_MMAN_H) +check_include_file(sys/shm.h HAVE_SYS_SHM_H) +check_include_file(sys/stat.h HAVE_SYS_STAT_H) +check_include_file(thread.h HAVE_THREAD_H) +check_function_exists(thr_setconcurrency HAVE_THR_SETCONCURRENCY) +check_include_file(unistd.h HAVE_UNISTD_H) +check_function_exists(fdatasync HAVE_FDATASYNC) +check_function_exists(memalign HAVE_MEMALIGN) +set(SIZEOF_BOOL 1) +check_type_size(size_t SIZEOF_SIZE_T LANGUAGE C) +check_function_exists(mmap HAVE_MMAP) +check_function_exists(valloc HAVE_VALLOC) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..088a13cf --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,203 @@ +add_executable( + sysbench + db_driver.c + db_driver.h + sb_barrier.c + sb_barrier.h + sb_ck_pr.h + sb_counter.c + sb_counter.h + sb_global.h + sb_histogram.c + sb_histogram.h + sb_list.h + sb_logger.c + sb_logger.h + sb_lua.c + sb_lua.h + sb_options.c + sb_options.h + sb_rand.c + sb_rand.h + sb_thread.c + sb_thread.h + sb_timer.c + sb_timer.h + sb_util.c + sb_util.h + sysbench.c + sysbench.h + tests/sb_cpu.h + tests/cpu/sb_cpu.c + tests/sb_fileio.h + tests/fileio/sb_fileio.c + tests/fileio/crc32.c + tests/fileio/crc32.h + tests/fileio/crc32tbl.h + tests/sb_memory.h + tests/memory/sb_memory.c + tests/sb_mutex.h + tests/mutex/sb_mutex.c + tests/sb_threads.h + tests/threads/sb_threads.c + xoroshiro128plus.h + config.h.in + ${CMAKE_CURRENT_BINARY_DIR}/config.h) +set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/config.h + PROPERTIES GENERATED 1) + +target_include_directories(sysbench PRIVATE ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(sysbench PRIVATE -DHAVE_CONFIG_H) +target_compile_features(sysbench PRIVATE c_std_99) +add_subdirectory(lua/internal) + +set_target_properties(sysbench PROPERTIES ENABLE_EXPORTS 1) +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) +target_link_libraries(sysbench PRIVATE Threads::Threads) +set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) +include(systemchecks) +set(PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(PACKAGE ${CMAKE_PROJECT_NAME}) +include(GNUInstallDirs) +set(DATADIR + ${CMAKE_INSTALL_FULL_DATADIR}/${CMAKE_PROJECT_NAME} + CACHE STRING "full path to pkgdatadir") +set(LIBDIR + ${CMAKE_INSTALL_FULL_LIBDIR}/${CMAKE_PROJECT_NAME} + CACHE STRING "full path to pkglibdir") + +# Use LuaJit library, either as system library or build from bundled source +set(WITH_SYSTEM_LUAJIT + OFF + CACHE BOOL "Use system-provided LuaJIT headers and library") +if(WITH_SYSTEM_LUAJIT) + # Find system library + find_package(LuaJit REQUIRED) + set(SB_WITH_LUAJIT "system") +else() + # Build from bundled source + include(BuildLuaJit) + set(SB_WITH_LUAJIT "bundled") +endif() +target_link_libraries(sysbench PRIVATE luajit::libluajit) + + +set(WITH_MYSQL + ON + CACHE BOOL "Enable mysql support for database tests") +if(WITH_MYSQL) + find_package(MySQL REQUIRED) + target_link_libraries(sysbench PRIVATE MySQL::MySQL) + set(CMAKE_REQUIRED_INCLUDES ${MySQL_INCLUDE_DIRS}) + include(CheckCSourceCompiles) + check_c_source_compiles( + " + #include + int main() + { + return (int)MYSQL_OPT_SSL_MODE; + }" + HAVE_MYSQL_OPT_SSL_MODE) +endif() + +if(WITH_MYSQL) + target_compile_definitions(sysbench PRIVATE -DUSE_MYSQL) + target_sources(sysbench PRIVATE drivers/mysql/drv_mysql.c) +endif() + +# Postgres driver +set(WITH_PGSQL + OFF + CACHE BOOL "Whether to use postgres driver.") +if(WITH_PGSQL) + # workaround CMake issue # 17223 (non-essential header lookup) + set(PostgreSQL_TYPE_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + find_package(PostgreSQL REQUIRED) + target_sources(sysbench PRIVATE drivers/pgsql/drv_pgsql.c) + target_compile_definitions(sysbench PRIVATE -DUSE_PGSQL) + target_link_libraries(sysbench PRIVATE PostgreSQL::PostgreSQL) +endif() + +set(WITH_SYSTEM_CONCURRENCY_KIT + OFF + CACHE BOOL "Use concurrency kit library provided by system") +if(WITH_SYSTEM_CONCURRENCY_KIT) + find_package(ConcurrencyKit REQUIRED) +else() + include(BuildConcurrencyKit) +endif() +add_dependencies(sysbench ConcurrencyKit) +target_link_libraries(sysbench PRIVATE ConcurrencyKit) + +# Find git hash, if built from git +include(githash) +githash(SB_GIT_SHA) +if(SB_GIT_SHA) + set(SB_GIT_SHA "-${SB_GIT_SHA}") +endif() + +configure_file(config.h.in config.h @ONLY) + +# Installation related stuff +add_subdirectory(lua) + + +# Install tests +if(WITH_MYSQL) + set(USE_MYSQL 1) +endif() +if(WITH_PGSQL) + set(USE_MYSQL 1) +endif() + +configure_file(${PROJECT_SOURCE_DIR}/tests/include/config.sh.in + ${PROJECT_BINARY_DIR}/tests/include/config.sh @ONLY) +install( + FILES ${PROJECT_BINARY_DIR}/tests/include/config.sh + DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME}/tests/include) + +install( + DIRECTORY ${PROJECT_SOURCE_DIR}/tests + DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME} + USE_SOURCE_PERMISSIONS FILES_MATCHING + PATTERN "*.t" + PATTERN "*.sh" + PATTERN "*.md" + PATTERN "*.lua") + +if(${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR}) + # make test is broken on out-of-source + add_custom_target( + test + COMMAND sh -c ./test_run.sh + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests) +endif() + +install( + TARGETS sysbench + EXPORT sysbenchTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +install( + EXPORT sysbenchTargets + FILE sysbenchTargets.cmake + NAMESPACE sysbench:: + DESTINATION share/sysbench/cmake) + +include(CMakePackageConfigHelpers) +# generate the config file that includes the exports +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/sysbenchConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/sysbenchConfig.cmake + INSTALL_DESTINATION share/sysbench/cmake + NO_SET_AND_CHECK_MACRO NO_CHECK_REQUIRED_COMPONENTS_MACRO) +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/sysbenchConfigVersion.cmake + COMPATIBILITY SameMajorVersion) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/sysbenchConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/sysbenchConfigVersion.cmake + DESTINATION share/sysbench/cmake) +include(CPack) diff --git a/src/config.h.in b/src/config.h.in new file mode 100644 index 00000000..0be3f25f --- /dev/null +++ b/src/config.h.in @@ -0,0 +1,67 @@ +#pragma once + +#cmakedefine PACKAGE_VERSION "@PACKAGE_VERSION@" +#cmakedefine PACKAGE "@PACKAGE@" +#define PACKAGE_NAME PACKAGE +#define DATADIR "@DATADIR@" +#define LIBDIR "@LIBDIR@" +#define SB_WITH_LUAJIT "@SB_WITH_LUAJIT@" +#define SB_GIT_SHA "@SB_GIT_SHA@" +#define STDC_HEADERS + + +#cmakedefine HAVE_ALARM +#cmakedefine HAVE_CLOCK_GETTIME +#cmakedefine HAVE_DIRECTIO +#cmakedefine HAVE_ERRNO_H +#cmakedefine HAVE_FCNTL_H +#cmakedefine HAVE_FDATASYNC +#cmakedefine HAVE_FUNC_ATTRIBUTE_FORMAT +#cmakedefine HAVE_FUNC_ATTRIBUTE_UNUSED +#define HAVE_ISATTY 1 +#cmakedefine HAVE_LARGE_PAGES +#cmakedefine HAVE_LIBAIO +#cmakedefine HAVE_LIBGEN_H +#cmakedefine HAVE_LIMITS_H +#cmakedefine HAVE_MATH_H +#cmakedefine HAVE_MMAP +#cmakedefine HAVE_MYSQL_OPT_SSL_MODE +#cmakedefine HAVE_OLD_GETEVENTS +#cmakedefine HAVE_POSIX_MEMALIGN +#cmakedefine HAVE_PTHREAD_CANCEL +#cmakedefine HAVE_PTHREAD_H +#cmakedefine HAVE_PTHREAD_YIELD +#cmakedefine HAVE_SCHED_H +#cmakedefine HAVE_SETVBUF +#cmakedefine HAVE_SIGNAL_H +#cmakedefine HAVE_STRERROR_R +#cmakedefine HAVE_STRINGS_H +#cmakedefine HAVE_STRING_H +#cmakedefine HAVE_SYS_IPC_H +#cmakedefine HAVE_SYS_MMAN_H +#cmakedefine HAVE_SYS_SHM_H +#cmakedefine HAVE_SYS_STAT_H +#cmakedefine HAVE_SYS_TIME_H +#cmakedefine HAVE_THREAD_H +#cmakedefine HAVE_THR_SETCONCURRENCY +#cmakedefine HAVE_UNISTD_H +#cmakedefine HAVE_MEMALIGN +#cmakedefine SIZEOF_SIZE_T @SIZEOF_SIZE_T@ +#cmakedefine SIZEOF_BOOL @SIZEOF_BOOL@ +#cmakedefine HAVE_VALLOC + +/* + Thread_local is standardized C99 but only if __STDC_NO_THREADS__ + is not defined. MSVC defines __STDC_NO_THREADS__ atm +*/ +#ifndef thread_local +# if __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__ +# define thread_local _Thread_local +/* note that ICC (linux) and Clang are covered by __GNUC__ */ +# elif defined __GNUC__ || defined __SUNPRO_C || defined __xlC__ +# define thread_local __thread +# else +# error "Cannot define thread_local" +# endif +#endif +#define TLS thread_local diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt new file mode 100644 index 00000000..d48d333b --- /dev/null +++ b/src/lua/CMakeLists.txt @@ -0,0 +1,17 @@ +install( + FILES bulk_insert.lua + empty-test.lua + oltp_common.lua + oltp_delete.lua + oltp_insert.lua + oltp_point_select.lua + oltp_read_only.lua + oltp_read_write.lua + oltp_update_index.lua + oltp_update_non_index.lua + oltp_write_only.lua + prime-test.lua + select_random_points.lua + select_random_ranges.lua + DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME} + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ) diff --git a/src/lua/internal/CMakeLists.txt b/src/lua/internal/CMakeLists.txt new file mode 100644 index 00000000..c17a1504 --- /dev/null +++ b/src/lua/internal/CMakeLists.txt @@ -0,0 +1,28 @@ +# Create a C header file with the contents of a Lua script +function(create_lua_h src varname dest) + file(READ ${src} LUA_SRC) + string(REPLACE "\\" "\\\\" LUA_SRC "${LUA_SRC}") + string(REPLACE "\"" "\\\"" LUA_SRC "${LUA_SRC}") + string(REPLACE "\n" "\\n\"\n\"" LUA_SRC "${LUA_SRC}") + set(filecontent "unsigned char ${varname}[] =\n\"${LUA_SRC}\";\nsize_t ${varname}_len=sizeof(${varname})-1;\n") + file(WRITE ${dest}.tmp "${filecontent}") + configure_file(${dest}.tmp ${dest} COPYONLY) + file(REMOVE ${dest}.tmp) +endfunction() + +set( + LUA_INTERNAL_FILES + sysbench.cmdline.lua + sysbench.histogram.lua + sysbench.lua + sysbench.rand.lua + sysbench.sql.lua +) + +# Rerun cmake if any of these files changes. +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${LUA_INTERNAL_FILES}) + +foreach(f ${LUA_INTERNAL_FILES}) + string(REPLACE "." "_" name ${f}) + create_lua_h(${f} ${name} ${CMAKE_CURRENT_BINARY_DIR}/${f}.h) +endforeach() diff --git a/src/sb_timer.h b/src/sb_timer.h index 43d0d149..62293226 100644 --- a/src/sb_timer.h +++ b/src/sb_timer.h @@ -27,7 +27,7 @@ # include # include #else -# if HAVE_SYS_TIME_H +# ifdef HAVE_SYS_TIME_H # include # else # include diff --git a/src/sysbenchConfig.cmake.in b/src/sysbenchConfig.cmake.in new file mode 100644 index 00000000..f32bd71a --- /dev/null +++ b/src/sysbenchConfig.cmake.in @@ -0,0 +1,15 @@ +@PACKAGE_INIT@ + +include ( "${CMAKE_CURRENT_LIST_DIR}/sysbenchTargets.cmake" ) + +get_target_property(loc sysbench::sysbench IMPORTED_LOCATION) +if (NOT loc) + foreach(c RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) + get_target_property(loc_config sysbench::sysbench IMPORTED_LOCATION_${c}) + if (loc_config) + set_target_properties(sysbench::sysbench PROPERTIES + IMPORTED_LOCATION ${loc_config}) + return() + endif() + endforeach() +endif() From aeb156114b16d9582f93b748cac174c5e4f609ee Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 03/45] CMake port - Test CMake build on CI (Ubuntu) Add just basic build and unit test, nothing fancy --- .github/workflows/ubuntu.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/ubuntu.yml diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml new file mode 100644 index 00000000..848ddaca --- /dev/null +++ b/.github/workflows/ubuntu.yml @@ -0,0 +1,23 @@ +name: ubuntu-CMake +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + name: Build on ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: configure + run: cmake . -DWITH_PGSQL=1 + - name: Build + run: cmake --build . + - name: MySQL version + run: mysql_config --version + - name: Sysbench version + run: ./src/sysbench --version + - name: test + run: cmake --build . --target test From d32fec967e77d12a310ba27dd48eabce8e1d2e5c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 04/45] CMake port - build and test on macOS This build actually runs some 30 seconds OLTP smoke tests against MySQL 8.0 --- .github/workflows/macos.yml | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/macos.yml diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 00000000..830f4066 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,50 @@ +name: macOS-CMake +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + os: [macos-latest] + runs-on: ${{ matrix.os }} + name: Build on ${{ matrix.os }} + steps: + - name: actions-setup-mysql + uses: shogo82148/actions-setup-mysql@v1.17.0 + with: + mysql-version: 'mysql-8.0' + my-cnf: | + innodb_log_file_size=256MB + innodb_buffer_pool_size=512MB + max_allowed_packet=16MB + max_connections=500 + - name: show variables + run: mysql -uroot --host=127.0.0.1 -e "show variables" + - name: install client drivers + run : | + HOMEBREW_NO_AUTO_UPDATE=1 brew install mysql-client libpq + brew link --force mysql-client + - name: checkout + uses: actions/checkout@v3 + - name: configure + run: cmake -B build_dir -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DWITH_PGSQL=1 + - name: build + run: cmake --build build_dir --config RelWithDebInfo -j4 + - name: test_install + run: | + cmake --install build_dir --prefix install_dir --config RelWithDebInfo + echo "$GITHUB_WORKSPACE/install_dir/bin" >> $GITHUB_PATH + - name: which sysbench + run: which sysbench + - name: sysbench version + run: sysbench --version + - name: create database sbtest + run: mysql -uroot --host=127.0.0.1 -e "create database sbtest" + - name: sysbench oltp_read_write prepare + run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 --table-size=100000 prepare + - name: sysbench oltp_read_write run + run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 --time=30 --table-size=100000 --threads=4 --report-interval=1 run + - name: sysbench oltp_read_write run ssl + run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 --time=30 --mysql-ssl=preferred --table-size=100000 --threads=4 --report-interval=1 run + - name: sysbench oltp_read_write cleanup + run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 cleanup From f1e129722dd18cfeef5491bd0e409a6b7d43762d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 05/45] Fix GCC 11.3 warning (stringop-truncation) __builtin_strncpy specified bound 4096 equals destination size [-Werror=stringop-truncation] strncpy wants output buffer to have place for terminating 0 --- src/sb_logger.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sb_logger.c b/src/sb_logger.c index 7fe1c30d..25d8b3c5 100644 --- a/src/sb_logger.c +++ b/src/sb_logger.c @@ -378,7 +378,7 @@ void log_errno(log_msg_priority_t priority, const char *fmt, ...) tmp = errbuf; #endif /* STRERROR_R_CHAR_P */ #else /* !HAVE_STRERROR_P */ - strncpy(errbuf, strerror(old_errno), sizeof(errbuf)); + strncpy(errbuf, strerror(old_errno), sizeof(errbuf) - 1); tmp = errbuf; #endif /* HAVE_STRERROR_P */ @@ -447,7 +447,7 @@ int text_handler_process(log_msg_t *msg) printf("(last message repeated %u times)\n", text_cnt); text_cnt = 0; - strncpy(text_buf, text_msg->text, TEXT_BUFFER_SIZE); + strncpy(text_buf, text_msg->text, TEXT_BUFFER_SIZE - 1); } pthread_mutex_unlock(&text_mutex); } From d2401288b126be095123647df85ec9d6ef8480b1 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 06/45] Fix GCC 11.3 warning (-Werror=maybe-uninitialized) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sysbench/src/tests/memory/sb_memory.c:199:18: error: ‘buffer’ may be used uninitialized in this function [-Werror=maybe-uninitialized] 199 | buffers[i] = buffer; --- src/tests/memory/sb_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/memory/sb_memory.c b/src/tests/memory/sb_memory.c index 27ecae84..6f829c29 100644 --- a/src/tests/memory/sb_memory.c +++ b/src/tests/memory/sb_memory.c @@ -112,7 +112,7 @@ int memory_init(void) { unsigned int i; char *s; - size_t *buffer; + size_t *buffer = NULL; memory_block_size = sb_get_value_size("memory-block-size"); if (memory_block_size < SIZEOF_SIZE_T || From fd4241c182cf01bfef665c3aad5f44a3d3cfbb3d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 07/45] Fix AppleClang 14 warning - use C11 _Static_assert for compile time assert. This fixes warning about unused typedef - SB_ATTRIBUTE_UNUSED does not help --- src/sb_util.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sb_util.h b/src/sb_util.h index 6fe2e8f4..67bee767 100644 --- a/src/sb_util.h +++ b/src/sb_util.h @@ -96,10 +96,14 @@ (char *)(SB_MEMBER_TYPE(type, member) *){ ptr } - offsetof(type, member))) /* Compile-time assertion */ +#if __STDC_VERSION__ >= 201112L +#define SB_COMPILE_TIME_ASSERT(expr) _Static_assert(expr, #expr) +#else #define SB_COMPILE_TIME_ASSERT(expr) \ do { \ typedef char cta[(expr) ? 1 : -1] SB_ATTRIBUTE_UNUSED; \ } while(0) +#endif #ifdef HAVE_ISATTY # define SB_ISATTY() isatty(0) From 1e21765e33984f1c0135b4295b79710cbe19f181 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 08/45] CMake port - handle compile warning as error on CI --- .github/workflows/macos.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 830f4066..1942b900 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -27,7 +27,7 @@ jobs: - name: checkout uses: actions/checkout@v3 - name: configure - run: cmake -B build_dir -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DWITH_PGSQL=1 + run: cmake -B build_dir -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DWITH_PGSQL=1 -DCMAKE_COMPILE_WARNING_AS_ERROR=1 - name: build run: cmake --build build_dir --config RelWithDebInfo -j4 - name: test_install diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 848ddaca..5a1f4b2b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: configure - run: cmake . -DWITH_PGSQL=1 + run: cmake . -DWITH_PGSQL=1 -DCMAKE_COMPILE_WARNING_AS_ERROR=1 - name: Build run: cmake --build . - name: MySQL version From 3559b2f1c9e532f4816dac266e09cfac345f9099 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 09/45] Windows port - MSVC/MinGW compile flags --- cmake/compile_flags.cmake | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/cmake/compile_flags.cmake b/cmake/compile_flags.cmake index c4647933..80bf99a6 100644 --- a/cmake/compile_flags.cmake +++ b/cmake/compile_flags.cmake @@ -1,4 +1,28 @@ -if((CMAKE_C_COMPILER_ID STREQUAL GNU) OR (CMAKE_C_COMPILER_ID MATCHES "Clang")) +if(WIN32) + add_definitions( + -DWIN32_MEAN_AND_LEAN + -DNOGDI + -DNOMINMAX + -DNOCOMM + -DNOUSER + -DNOCRYPT # minimize windows.h + -D_CRT_NONSTDC_NO_WARNINGS + -D_CRT_SECURE_NO_WARNINGS # Remove warnings about using "insecure" functions + ) +endif() + +if(MSVC) + # MSVC warning settings + set(MSVC_WARNING_LEVEL "3" CACHE STRING "MSVC warning level (1-4)") + add_compile_options( + /W${MSVC_WARNING_LEVEL} + /wd4324 # ignore C4324 '': structure was padded due to alignment specifier + /wd4200 # ignore C4200 nonstandard extension used: zero-sized array in struct/union + ) +endif() + +if(((CMAKE_C_COMPILER_ID STREQUAL GNU) OR (CMAKE_C_COMPILER_ID MATCHES "Clang")) + AND NOT MSVC) add_compile_options( -Wall -Wextra @@ -14,6 +38,10 @@ if((CMAKE_C_COMPILER_ID STREQUAL GNU) OR (CMAKE_C_COMPILER_ID MATCHES "Clang")) -Wredundant-decls -Wcast-align -Wvla) + if(MINGW) + # mingw gcc is buggy wrt format (does not know %zd, etc) + add_compile_options(-Wno-format) + endif() endif() if(CMAKE_COMPILE_WARNING_AS_ERROR AND (CMAKE_VERSION VERSION_LESS "3.24")) @@ -23,3 +51,5 @@ if(CMAKE_COMPILE_WARNING_AS_ERROR AND (CMAKE_VERSION VERSION_LESS "3.24")) add_compile_options(-Werror) endif() endif() + +set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT ProgramDatabase) From 9c9198b81c8c3c7828ccf22225c49fb7175e58b7 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 10/45] Windows port - add pthreads library from Lockless Inc The original URL is https://locklessinc.com/downloads/winpthreads.h mingw64 winpthreads is based on this library. Compared to other pthread implementations, this one does support pthread cancellation, which turns out to be important for sysbench. --- cmake/systemchecks.cmake | 12 +- src/CMakeLists.txt | 15 +- src/win/pthread/CMakeLists.txt | 7 + src/win/pthread/pthread.h | 1576 ++++++++++++++++++++++++++++++++ src/win/pthread/pthread_data.c | 50 + 5 files changed, 1652 insertions(+), 8 deletions(-) create mode 100644 src/win/pthread/CMakeLists.txt create mode 100644 src/win/pthread/pthread.h create mode 100644 src/win/pthread/pthread_data.c diff --git a/cmake/systemchecks.cmake b/cmake/systemchecks.cmake index d318f6bf..fa5c06a4 100644 --- a/cmake/systemchecks.cmake +++ b/cmake/systemchecks.cmake @@ -21,9 +21,15 @@ check_include_file(limits.h HAVE_LIMITS_H) check_include_file(math.h HAVE_MATH_H) check_symbol_exists(SHM_HUGETLB sys/shm.h HAVE_LARGE_PAGES) check_function_exists(posix_memalign HAVE_POSIX_MEMALIGN) -check_include_file(pthread.h HAVE_PTHREAD_H) -check_symbol_exists(pthread_cancel pthread.h HAVE_PTHREAD_CANCEL) -check_symbol_exists(pthread_yield pthread.h HAVE_PTHREAD_YIELD) +if(WIN32) + set(HAVE_PTHREAD_CANCEL 1) + set(HAVE_PTHREAD_H 1) + set(HAVE_PTHREAD_YIELD 1) +else() + check_include_file(pthread.h HAVE_PTHREAD_H) + check_symbol_exists(pthread_cancel pthread.h HAVE_PTHREAD_CANCEL) + check_symbol_exists(pthread_yield pthread.h HAVE_PTHREAD_YIELD) +endif() check_include_file(sched.h HAVE_SCHED_H) check_function_exists(setvbuf HAVE_SETVBUF) check_include_file(signal.h HAVE_SIGNAL_H) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 088a13cf..b053b5c4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -53,11 +53,16 @@ target_compile_features(sysbench PRIVATE c_std_99) add_subdirectory(lua/internal) set_target_properties(sysbench PROPERTIES ENABLE_EXPORTS 1) -set(CMAKE_THREAD_PREFER_PTHREAD TRUE) -set(THREADS_PREFER_PTHREAD_FLAG TRUE) -find_package(Threads REQUIRED) -target_link_libraries(sysbench PRIVATE Threads::Threads) -set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) +if(WIN32) + # Use pthreads port from Lockless.inc + add_subdirectory(win/pthread) +else() + set(CMAKE_THREAD_PREFER_PTHREAD TRUE) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) + find_package(Threads REQUIRED) + target_link_libraries(sysbench PRIVATE Threads::Threads) + set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) +endif() include(systemchecks) set(PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) set(PACKAGE ${CMAKE_PROJECT_NAME}) diff --git a/src/win/pthread/CMakeLists.txt b/src/win/pthread/CMakeLists.txt new file mode 100644 index 00000000..1718e79c --- /dev/null +++ b/src/win/pthread/CMakeLists.txt @@ -0,0 +1,7 @@ +target_sources(sysbench + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/pthread_data.c + ${CMAKE_CURRENT_SOURCE_DIR}/pthread.h) +target_include_directories(sysbench + PRIVATE SYSTEM + ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/win/pthread/pthread.h b/src/win/pthread/pthread.h new file mode 100644 index 00000000..6f2132d0 --- /dev/null +++ b/src/win/pthread/pthread.h @@ -0,0 +1,1576 @@ +/* + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * You may want to use the MingW64 winpthreads library instead. + * It is based on this, but adds error checking. + */ + +/* + * Version 1.0.1 Released 2 Feb 2012 + * Fixes pthread_barrier_destroy() to wait for threads to exit the barrier. + */ + +#ifndef WIN_PTHREADS +#define WIN_PTHREADS +#if defined __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wbad-function-cast" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wincompatible-pointer-types" +#pragma GCC diagnostic ignored "-Wpointer-sign" +#elif defined _MSC_VER +#pragma warning(push) +/* disable signed/unsigned mismatch (C4018) and conversion(C4244) warnings */ +#pragma warning(disable : 4018 4244) +#endif + + +#include +#include +#include +#include +#include +#include + +#define PTHREAD_CANCEL_DISABLE 0 +#define PTHREAD_CANCEL_ENABLE 0x01 + +#define PTHREAD_CANCEL_DEFERRED 0 +#define PTHREAD_CANCEL_ASYNCHRONOUS 0x02 + +#define PTHREAD_CREATE_JOINABLE 0 +#define PTHREAD_CREATE_DETACHED 0x04 + +#define PTHREAD_EXPLICT_SCHED 0 +#define PTHREAD_INHERIT_SCHED 0x08 + +#define PTHREAD_SCOPE_PROCESS 0 +#define PTHREAD_SCOPE_SYSTEM 0x10 + +#define PTHREAD_DEFAULT_ATTR (PTHREAD_CANCEL_ENABLE) + +#define PTHREAD_CANCELED ((void *)(intptr_t) 0xDEADBEEF) + +#define PTHREAD_ONCE_INIT 0 +#define PTHREAD_MUTEX_INITIALIZER {(void*)-1,-1,0,0,0,0} +#define PTHREAD_RWLOCK_INITIALIZER {0} +#define PTHREAD_COND_INITIALIZER {0} +#define PTHREAD_BARRIER_INITIALIZER \ + {0,0,PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER} +#define PTHREAD_SPINLOCK_INITIALIZER 0 + +#define PTHREAD_DESTRUCTOR_ITERATIONS 256 +#define PTHREAD_KEYS_MAX (1<<20) + +#define PTHREAD_MUTEX_NORMAL 0 +#define PTHREAD_MUTEX_ERRORCHECK 1 +#define PTHREAD_MUTEX_RECURSIVE 2 +#define PTHREAD_MUTEX_DEFAULT 3 +#define PTHREAD_MUTEX_SHARED 4 +#define PTHREAD_MUTEX_PRIVATE 0 +#define PTHREAD_PRIO_NONE 0 +#define PTHREAD_PRIO_INHERIT 8 +#define PTHREAD_PRIO_PROTECT 16 +#define PTHREAD_PRIO_MULT 32 +#define PTHREAD_PROCESS_SHARED 0 +#define PTHREAD_PROCESS_PRIVATE 1 + +#define PTHREAD_BARRIER_SERIAL_THREAD 1 + +#include /* timespec */ + +typedef struct _pthread_cleanup _pthread_cleanup; +struct _pthread_cleanup +{ + void (*func)(void *); + void *arg; + _pthread_cleanup *next; +}; + +struct _pthread_v +{ + void *ret_arg; + void *(* func)(void *); + _pthread_cleanup *clean; + HANDLE h; + int cancelled; + unsigned p_state; + int keymax; + void **keyval; + + jmp_buf jb; +}; + +typedef struct _pthread_v *pthread_t; + +typedef struct pthread_barrier_t pthread_barrier_t; +struct pthread_barrier_t +{ + int count; + int total; + CRITICAL_SECTION m; + CONDITION_VARIABLE cv; +}; + +typedef struct pthread_attr_t pthread_attr_t; +struct pthread_attr_t +{ + unsigned p_state; + void *stack; + size_t s_size; +}; + +typedef long pthread_once_t; +typedef unsigned pthread_mutexattr_t; +typedef SRWLOCK pthread_rwlock_t; +typedef CRITICAL_SECTION pthread_mutex_t; +typedef unsigned pthread_key_t; +typedef void *pthread_barrierattr_t; +typedef long pthread_spinlock_t; +typedef int pthread_condattr_t; +typedef CONDITION_VARIABLE pthread_cond_t; +typedef int pthread_rwlockattr_t; + +extern volatile long _pthread_cancelling; + +extern int _pthread_concur; + +/* Will default to zero as needed */ +extern pthread_once_t _pthread_tls_once; +extern DWORD _pthread_tls; + +/* Note initializer is zero, so this works */ +extern pthread_rwlock_t _pthread_key_lock; +extern long _pthread_key_max; +extern long _pthread_key_sch; +extern void (**_pthread_key_dest)(void *); + + +#define pthread_cleanup_push(F, A)\ +{\ + const _pthread_cleanup _pthread_cup = {(F), (A), pthread_self()->clean};\ + _ReadWriteBarrier();\ + pthread_self()->clean = (_pthread_cleanup *) &_pthread_cup;\ + _ReadWriteBarrier() + +/* Note that if async cancelling is used, then there is a race here */ +#define pthread_cleanup_pop(E)\ + (pthread_self()->clean = _pthread_cup.next, (E?_pthread_cup.func(_pthread_cup.arg):0));} + +static inline void _pthread_once_cleanup(void *o) +{ + *(pthread_once_t *)o = 0; +} + +static inline pthread_t pthread_self(void); +static inline int pthread_once(pthread_once_t *o, void (*func)(void)) +{ + long state = *o; + + _ReadWriteBarrier(); + + while (state != 1) + { + if (!state) + { + if (!_InterlockedCompareExchange(o, 2, 0)) + { + /* Success */ + pthread_cleanup_push(_pthread_once_cleanup, o); + func(); + pthread_cleanup_pop(0); + + /* Mark as done */ + *o = 1; + + return 0; + } + } + + YieldProcessor(); + + _ReadWriteBarrier(); + + state = *o; + } + + /* Done */ + return 0; +} + +static inline int _pthread_once_raw(pthread_once_t *o, void (*func)(void)) +{ + long state = *o; + + _ReadWriteBarrier(); + + while (state != 1) + { + if (!state) + { + if (!_InterlockedCompareExchange(o, 2, 0)) + { + /* Success */ + func(); + + /* Mark as done */ + *o = 1; + + return 0; + } + } + + YieldProcessor(); + + _ReadWriteBarrier(); + + state = *o; + } + + /* Done */ + return 0; +} + +static inline int pthread_mutex_lock(pthread_mutex_t *m) +{ + EnterCriticalSection(m); + return 0; +} + +static inline int pthread_mutex_unlock(pthread_mutex_t *m) +{ + LeaveCriticalSection(m); + return 0; +} + +static inline int pthread_mutex_trylock(pthread_mutex_t *m) +{ + return TryEnterCriticalSection(m) ? 0 : EBUSY; +} + +static inline int pthread_mutex_init(pthread_mutex_t *m, pthread_mutexattr_t *a) +{ + (void) a; + InitializeCriticalSection(m); + + return 0; +} + +static inline int pthread_mutex_destroy(pthread_mutex_t *m) +{ + DeleteCriticalSection(m); + return 0; +} + +#define pthread_mutex_getprioceiling(M, P) ENOTSUP +#define pthread_mutex_setprioceiling(M, P) ENOTSUP + +static inline int pthread_equal(pthread_t t1, pthread_t t2) +{ + return t1 == t2; +} + +static inline void pthread_testcancel(void); + +static inline int pthread_rwlock_init(pthread_rwlock_t *l, pthread_rwlockattr_t *a) +{ + (void) a; + InitializeSRWLock(l); + + return 0; +} + +static inline int pthread_rwlock_destroy(pthread_rwlock_t *l) +{ + (void) *l; + return 0; +} + +static inline int pthread_rwlock_rdlock(pthread_rwlock_t *l) +{ + pthread_testcancel(); + AcquireSRWLockShared(l); + + return 0; +} + +static inline int pthread_rwlock_wrlock(pthread_rwlock_t *l) +{ + pthread_testcancel(); + AcquireSRWLockExclusive(l); + + return 0; +} + +static inline void pthread_tls_init(void) +{ + _pthread_tls = TlsAlloc(); + + /* Cannot continue if out of indexes */ + if (_pthread_tls == TLS_OUT_OF_INDEXES) abort(); +} + +static inline int pthread_rwlock_unlock(pthread_rwlock_t *l); +static inline void _pthread_cleanup_dest(pthread_t t) +{ + int i, j; + + for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; j++) + { + int flag = 0; + + for (i = 0; i < t->keymax; i++) + { + void *val = t->keyval[i]; + + if (val) + { + pthread_rwlock_rdlock(&_pthread_key_lock); + if ((uintptr_t) _pthread_key_dest[i] > 1) + { + /* Call destructor */ + t->keyval[i] = NULL; + _pthread_key_dest[i](val); + flag = 1; + } + pthread_rwlock_unlock(&_pthread_key_lock); + } + } + + /* Nothing to do? */ + if (!flag) return; + } +} + +static inline pthread_t pthread_self(void) +{ + pthread_t t; + + _pthread_once_raw(&_pthread_tls_once, pthread_tls_init); + + t = TlsGetValue(_pthread_tls); + + /* Main thread? */ + if (!t) + { + t = malloc(sizeof(struct _pthread_v)); + + /* If cannot initialize main thread, then the only thing we can do is abort */ + if (!t) abort(); + + t->ret_arg = NULL; + t->func = NULL; + t->clean = NULL; + t->cancelled = 0; + t->p_state = PTHREAD_DEFAULT_ATTR; + t->keymax = 0; + t->keyval = NULL; + t->h = GetCurrentThread(); + + /* Save for later */ + TlsSetValue(_pthread_tls, t); + + if (setjmp(t->jb)) + { + /* Make sure we free ourselves if we are detached */ + if (!t->h) free(t); + + /* Time to die */ + _endthreadex(0); + } + } + + return t; +} + +static inline int pthread_rwlock_unlock(pthread_rwlock_t *l) +{ + void *state = *(void **)l; + + if (state == (void *) 1) + { + /* Known to be an exclusive lock */ + ReleaseSRWLockExclusive(l); + } + else + { + /* A shared unlock will work */ + ReleaseSRWLockShared(l); + } + + return 0; +} + + +static inline int pthread_rwlock_tryrdlock(pthread_rwlock_t *l) +{ + if (TryAcquireSRWLockShared(l)) return 0; + + return EBUSY; +} + +static inline int pthread_rwlock_trywrlock(pthread_rwlock_t *l) +{ + if (TryAcquireSRWLockExclusive(l)) return 0; + + return EBUSY; +} + +static inline unsigned long long _pthread_time_in_ms(void) +{ + struct __timeb64 tb; + + _ftime64(&tb); + + return tb.time * 1000 + tb.millitm; +} + +static inline unsigned long long _pthread_time_in_ms_from_timespec(const struct timespec *ts) +{ + unsigned long long t = (unsigned long long)ts->tv_sec * 1000; + t += ts->tv_nsec / 1000000; + + return t; +} + +static inline unsigned long long _pthread_rel_time_in_ms(const struct timespec *ts) +{ + unsigned long long t1 = _pthread_time_in_ms_from_timespec(ts); + unsigned long long t2 = _pthread_time_in_ms(); + + /* Prevent underflow */ + if (t1 < t2) return 0; + return t1 - t2; +} + +static inline int pthread_rwlock_timedrdlock(pthread_rwlock_t *l, const struct timespec *ts) +{ + unsigned long long ct = _pthread_time_in_ms(); + unsigned long long t = _pthread_time_in_ms_from_timespec(ts); + + pthread_testcancel(); + + /* Use a busy-loop */ + while (1) + { + /* Try to grab lock */ + if (!pthread_rwlock_tryrdlock(l)) return 0; + + /* Get current time */ + ct = _pthread_time_in_ms(); + + /* Have we waited long enough? */ + if (ct > t) return ETIMEDOUT; + } +} + +static inline int pthread_rwlock_timedwrlock(pthread_rwlock_t *l, const struct timespec *ts) +{ + unsigned long long ct = _pthread_time_in_ms(); + unsigned long long t = _pthread_time_in_ms_from_timespec(ts); + + pthread_testcancel(); + + /* Use a busy-loop */ + while (1) + { + /* Try to grab lock */ + if (!pthread_rwlock_trywrlock(l)) return 0; + + /* Get current time */ + ct = _pthread_time_in_ms(); + + /* Have we waited long enough? */ + if (ct > t) return ETIMEDOUT; + } +} + +static inline int pthread_get_concurrency(int *val) +{ + *val = _pthread_concur; + return 0; +} + +static inline int pthread_set_concurrency(int val) +{ + _pthread_concur = val; + return 0; +} + +#define pthread_getschedparam(T, P, S) ENOTSUP +#define pthread_setschedparam(T, P, S) ENOTSUP +#define pthread_getcpuclockid(T, C) ENOTSUP + +static inline int pthread_exit(void *res) +{ + pthread_t t = pthread_self(); + + t->ret_arg = res; + + _pthread_cleanup_dest(t); + + longjmp(t->jb, 1); +} + + +static inline void _pthread_invoke_cancel(void) +{ + _pthread_cleanup *pcup; + + _InterlockedDecrement(&_pthread_cancelling); + + /* Call cancel queue */ + for (pcup = pthread_self()->clean; pcup; pcup = pcup->next) + { + pcup->func(pcup->arg); + } + + pthread_exit(PTHREAD_CANCELED); +} + +static inline void pthread_testcancel(void) +{ + if (_pthread_cancelling) + { + pthread_t t = pthread_self(); + + if (t->cancelled && (t->p_state & PTHREAD_CANCEL_ENABLE)) + { + _pthread_invoke_cancel(); + } + } +} + + +static inline int pthread_cancel(pthread_t t) +{ + if (t->p_state & PTHREAD_CANCEL_ASYNCHRONOUS) + { + /* Dangerous asynchronous cancelling */ + CONTEXT ctxt; + + /* Already done? */ + if (t->cancelled) return ESRCH; + + ctxt.ContextFlags = CONTEXT_CONTROL; + + SuspendThread(t->h); + GetThreadContext(t->h, &ctxt); +#if defined (_M_ARM64) || defined (_M_ARM64EC) + ctxt.Pc = (uintptr_t) _pthread_invoke_cancel; +#elif defined _M_X64 + ctxt.Rip = (uintptr_t)_pthread_invoke_cancel; +#elif defined _M_IX86 + ctxt.Eip = (uintptr_t) _pthread_invoke_cancel; +#else + #error Unsupported architecture +#endif + SetThreadContext(t->h, &ctxt); + + /* Also try deferred Cancelling */ + t->cancelled = 1; + + /* Notify everyone to look */ + _InterlockedIncrement(&_pthread_cancelling); + + ResumeThread(t->h); + } + else + { + /* Safe deferred Cancelling */ + t->cancelled = 1; + + /* Notify everyone to look */ + _InterlockedIncrement(&_pthread_cancelling); + } + + return 0; +} + +static inline unsigned _pthread_get_state(pthread_attr_t *attr, unsigned flag) +{ + return attr->p_state & flag; +} + +static inline int _pthread_set_state(pthread_attr_t *attr, unsigned flag, unsigned val) +{ + if (~flag & val) return EINVAL; + attr->p_state &= ~flag; + attr->p_state |= val; + + return 0; +} + +static inline int pthread_attr_init(pthread_attr_t *attr) +{ + attr->p_state = PTHREAD_DEFAULT_ATTR; + attr->stack = NULL; + attr->s_size = 0; + return 0; +} + +static inline int pthread_attr_destroy(pthread_attr_t *attr) +{ + /* No need to do anything */ + return 0; +} + + +static inline int pthread_attr_setdetachstate(pthread_attr_t *a, int flag) +{ + return _pthread_set_state(a, PTHREAD_CREATE_DETACHED, flag); +} + +static inline int pthread_attr_getdetachstate(pthread_attr_t *a, int *flag) +{ + *flag = _pthread_get_state(a, PTHREAD_CREATE_DETACHED); + return 0; +} + +static inline int pthread_attr_setinheritsched(pthread_attr_t *a, int flag) +{ + return _pthread_set_state(a, PTHREAD_INHERIT_SCHED, flag); +} + +static inline int pthread_attr_getinheritsched(pthread_attr_t *a, int *flag) +{ + *flag = _pthread_get_state(a, PTHREAD_INHERIT_SCHED); + return 0; +} + +static inline int pthread_attr_setscope(pthread_attr_t *a, int flag) +{ + return _pthread_set_state(a, PTHREAD_SCOPE_SYSTEM, flag); +} + +static inline int pthread_attr_getscope(pthread_attr_t *a, int *flag) +{ + *flag = _pthread_get_state(a, PTHREAD_SCOPE_SYSTEM); + return 0; +} + +static inline int pthread_attr_getstackaddr(pthread_attr_t *attr, void **stack) +{ + *stack = attr->stack; + return 0; +} + +static inline int pthread_attr_setstackaddr(pthread_attr_t *attr, void *stack) +{ + attr->stack = stack; + return 0; +} + +static inline int pthread_attr_getstacksize(pthread_attr_t *attr, size_t *size) +{ + *size = attr->s_size; + return 0; +} + +static inline int pthread_attr_setstacksize(pthread_attr_t *attr, size_t size) +{ + attr->s_size = size; + return 0; +} + +#define pthread_attr_getguardsize(A, S) ENOTSUP +#define pthread_attr_setgaurdsize(A, S) ENOTSUP +#define pthread_attr_getschedparam(A, S) ENOTSUP +#define pthread_attr_setschedparam(A, S) ENOTSUP +#define pthread_attr_getschedpolicy(A, S) ENOTSUP +#define pthread_attr_setschedpolicy(A, S) ENOTSUP + + +static inline int pthread_setcancelstate(int state, int *oldstate) +{ + pthread_t t = pthread_self(); + + if ((state & PTHREAD_CANCEL_ENABLE) != state) return EINVAL; + if (oldstate) *oldstate = t->p_state & PTHREAD_CANCEL_ENABLE; + t->p_state &= ~PTHREAD_CANCEL_ENABLE; + t->p_state |= state; + + return 0; +} + +static inline int pthread_setcanceltype(int type, int *oldtype) +{ + pthread_t t = pthread_self(); + + if ((type & PTHREAD_CANCEL_ASYNCHRONOUS) != type) return EINVAL; + if (oldtype) *oldtype = t->p_state & PTHREAD_CANCEL_ASYNCHRONOUS; + t->p_state &= ~PTHREAD_CANCEL_ASYNCHRONOUS; + t->p_state |= type; + + return 0; +} + +static inline unsigned int WINAPI pthread_create_wrapper(void *args) +{ + struct _pthread_v *tv = args; + + _pthread_once_raw(&_pthread_tls_once, pthread_tls_init); + + TlsSetValue(_pthread_tls, tv); + + if (!setjmp(tv->jb)) + { + /* Call function and save return value */ + tv->ret_arg = tv->func(tv->ret_arg); + + /* Clean up destructors */ + _pthread_cleanup_dest(tv); + } + + /* If we exit too early, then we can race with create */ + while (tv->h == (HANDLE) -1) + { + YieldProcessor(); + _ReadWriteBarrier(); + } + + /* Make sure we free ourselves if we are detached */ + if (!tv->h) free(tv); + + return 0; +} + +static inline int pthread_create(pthread_t *th, const pthread_attr_t *attr, void *(* func)(void *), void *arg) +{ + struct _pthread_v *tv = malloc(sizeof(struct _pthread_v)); + unsigned ssize = 0; + + if (!tv) return 1; + + *th = tv; + + /* Save data in pthread_t */ + tv->ret_arg = arg; + tv->func = func; + tv->clean = NULL; + tv->cancelled = 0; + tv->p_state = PTHREAD_DEFAULT_ATTR; + tv->keymax = 0; + tv->keyval = NULL; + tv->h = (HANDLE) -1; + + if (attr) + { + tv->p_state = attr->p_state; + ssize = (unsigned int)attr->s_size; + } + + /* Make sure tv->h has value of -1 */ + _ReadWriteBarrier(); + + tv->h = (HANDLE) _beginthreadex(NULL, ssize, pthread_create_wrapper, tv, 0, NULL); + + /* Failed */ + if (!tv->h) return 1; + + if (tv->p_state & PTHREAD_CREATE_DETACHED) + { + CloseHandle(tv->h); + _ReadWriteBarrier(); + tv->h = 0; + } + + return 0; +} + +static inline int pthread_join(pthread_t t, void **res) +{ + struct _pthread_v *tv = t; + + pthread_testcancel(); + + WaitForSingleObject(tv->h, INFINITE); + CloseHandle(tv->h); + + /* Obtain return value */ + if (res) *res = tv->ret_arg; + + free(tv); + + return 0; +} + +static inline int pthread_detach(pthread_t t) +{ + struct _pthread_v *tv = t; + + /* + * This can't race with thread exit because + * our call would be undefined if called on a dead thread. + */ + + CloseHandle(tv->h); + _ReadWriteBarrier(); + tv->h = 0; + + return 0; +} + +static inline int pthread_mutexattr_init(pthread_mutexattr_t *a) +{ + *a = 0; + return 0; +} + +static inline int pthread_mutexattr_destroy(pthread_mutexattr_t *a) +{ + (void) a; + return 0; +} + +static inline int pthread_mutexattr_gettype(pthread_mutexattr_t *a, int *type) +{ + *type = *a & 3; + + return 0; +} + +static inline int pthread_mutexattr_settype(pthread_mutexattr_t *a, int type) +{ + if ((unsigned) type > 3) return EINVAL; + *a &= ~3; + *a |= type; + + return 0; +} + +static inline int pthread_mutexattr_getpshared(pthread_mutexattr_t *a, int *type) +{ + *type = *a & 4; + + return 0; +} + +static inline int pthread_mutexattr_setpshared(pthread_mutexattr_t * a, int type) +{ + if ((type & 4) != type) return EINVAL; + + *a &= ~4; + *a |= type; + + return 0; +} + +static inline int pthread_mutexattr_getprotocol(pthread_mutexattr_t *a, int *type) +{ + *type = *a & (8 + 16); + + return 0; +} + +static inline int pthread_mutexattr_setprotocol(pthread_mutexattr_t *a, int type) +{ + if ((type & (8 + 16)) != 8 + 16) return EINVAL; + + *a &= ~(8 + 16); + *a |= type; + + return 0; +} + +static inline int pthread_mutexattr_getprioceiling(pthread_mutexattr_t *a, int * prio) +{ + *prio = *a / PTHREAD_PRIO_MULT; + return 0; +} + +static inline int pthread_mutexattr_setprioceiling(pthread_mutexattr_t *a, int prio) +{ + *a &= (PTHREAD_PRIO_MULT - 1); + *a += prio * PTHREAD_PRIO_MULT; + + return 0; +} + + +#define _PTHREAD_BARRIER_FLAG (1<<30) + +static inline int pthread_barrier_destroy(pthread_barrier_t *b) +{ + EnterCriticalSection(&b->m); + + while (b->total > _PTHREAD_BARRIER_FLAG) + { + /* Wait until everyone exits the barrier */ + SleepConditionVariableCS(&b->cv, &b->m, INFINITE); + } + + LeaveCriticalSection(&b->m); + + DeleteCriticalSection(&b->m); + + return 0; +} + +static inline int pthread_barrier_init(pthread_barrier_t *b, void *attr, int count) +{ + /* Ignore attr */ + (void) attr; + + b->count = count; + b->total = 0; + + InitializeCriticalSection(&b->m); + InitializeConditionVariable(&b->cv); + + return 0; +} + +static inline int pthread_barrier_wait(pthread_barrier_t *b) +{ + EnterCriticalSection(&b->m); + + while (b->total > _PTHREAD_BARRIER_FLAG) + { + /* Wait until everyone exits the barrier */ + SleepConditionVariableCS(&b->cv, &b->m, INFINITE); + } + + /* Are we the first to enter? */ + if (b->total == _PTHREAD_BARRIER_FLAG) b->total = 0; + + b->total++; + + if (b->total == b->count) + { + b->total += _PTHREAD_BARRIER_FLAG - 1; + WakeAllConditionVariable(&b->cv); + + LeaveCriticalSection(&b->m); + + return 1; + } + else + { + while (b->total < _PTHREAD_BARRIER_FLAG) + { + /* Wait until enough threads enter the barrier */ + SleepConditionVariableCS(&b->cv, &b->m, INFINITE); + } + + b->total--; + + /* Get entering threads to wake up */ + if (b->total == _PTHREAD_BARRIER_FLAG) WakeAllConditionVariable(&b->cv); + + LeaveCriticalSection(&b->m); + + return 0; + } +} + +static inline int pthread_barrierattr_init(void **attr) +{ + *attr = NULL; + return 0; +} + +static inline int pthread_barrierattr_destroy(void **attr) +{ + /* Ignore attr */ + (void) attr; + + return 0; +} + +static inline int pthread_barrierattr_setpshared(void **attr, int s) +{ + (void)attr; + (void)s; + return 0; +} + +static inline int pthread_barrierattr_getpshared(void **attr, int *s) +{ + *s = 0; + (void)attr; + return 0; +} + +static inline int pthread_key_create(pthread_key_t *key, void (* dest)(void *)) +{ + int i; + long nmax; + void (**d)(void *); + + if (!key) return EINVAL; + + pthread_rwlock_wrlock(&_pthread_key_lock); + + for (i = _pthread_key_sch; i < _pthread_key_max; i++) + { + if (!_pthread_key_dest[i]) + { + *key = i; + if (dest) + { + _pthread_key_dest[i] = dest; + } + else + { + _pthread_key_dest[i] = (void(*)(void *))1; + } + pthread_rwlock_unlock(&_pthread_key_lock); + + return 0; + } + } + + for (i = 0; i < _pthread_key_sch; i++) + { + if (!_pthread_key_dest[i]) + { + *key = i; + if (dest) + { + _pthread_key_dest[i] = dest; + } + else + { + _pthread_key_dest[i] = (void(*)(void *))1; + } + pthread_rwlock_unlock(&_pthread_key_lock); + + return 0; + } + } + + if (!_pthread_key_max) _pthread_key_max = 1; + if (_pthread_key_max == PTHREAD_KEYS_MAX) + { + pthread_rwlock_unlock(&_pthread_key_lock); + + return ENOMEM; + } + + nmax = _pthread_key_max * 2; + if (nmax > PTHREAD_KEYS_MAX) nmax = PTHREAD_KEYS_MAX; + + /* No spare room anywhere */ + d = realloc(_pthread_key_dest, nmax * sizeof(*d)); + if (!d) + { + pthread_rwlock_unlock(&_pthread_key_lock); + + return ENOMEM; + } + + /* Clear new region */ + memset((void *) &d[_pthread_key_max], 0, (nmax-_pthread_key_max)*sizeof(void *)); + + /* Use new region */ + _pthread_key_dest = d; + _pthread_key_sch = _pthread_key_max + 1; + *key = _pthread_key_max; + _pthread_key_max = nmax; + + if (dest) + { + _pthread_key_dest[*key] = dest; + } + else + { + _pthread_key_dest[*key] = (void(*)(void *))1; + } + + pthread_rwlock_unlock(&_pthread_key_lock); + + return 0; +} + +static inline int pthread_key_delete(pthread_key_t key) +{ + if (key > _pthread_key_max) return EINVAL; + if (!_pthread_key_dest) return EINVAL; + + pthread_rwlock_wrlock(&_pthread_key_lock); + _pthread_key_dest[key] = NULL; + + /* Start next search from our location */ + if (_pthread_key_sch > key) _pthread_key_sch = key; + + pthread_rwlock_unlock(&_pthread_key_lock); + + return 0; +} + +static inline void *pthread_getspecific(pthread_key_t key) +{ + pthread_t t = pthread_self(); + + if (key >= t->keymax) return NULL; + + return t->keyval[key]; + +} + +static inline int pthread_setspecific(pthread_key_t key, const void *value) +{ + pthread_t t = pthread_self(); + + if (key > t->keymax) + { + int keymax = (key + 1) * 2; + void **kv = realloc(t->keyval, keymax * sizeof(void *)); + + if (!kv) return ENOMEM; + + /* Clear new region */ + memset(&kv[t->keymax], 0, (keymax - t->keymax)*sizeof(void*)); + + t->keyval = kv; + t->keymax = keymax; + } + + t->keyval[key] = (void *) value; + + return 0; +} + + +static inline int pthread_spin_init(pthread_spinlock_t *l, int pshared) +{ + (void) pshared; + + *l = 0; + return 0; +} + +static inline int pthread_spin_destroy(pthread_spinlock_t *l) +{ + (void) l; + return 0; +} + +/* No-fair spinlock due to lack of knowledge of thread number */ +static inline int pthread_spin_lock(pthread_spinlock_t *l) +{ + while (_InterlockedExchange(l, EBUSY)) + { + /* Don't lock the bus whilst waiting */ + while (*l) + { + YieldProcessor(); + + /* Compiler barrier. Prevent caching of *l */ + _ReadWriteBarrier(); + } + } + + return 0; +} + +static inline int pthread_spin_trylock(pthread_spinlock_t *l) +{ + return _InterlockedExchange(l, EBUSY); +} + +static inline int pthread_spin_unlock(pthread_spinlock_t *l) +{ + /* Compiler barrier. The store below acts with release symmantics */ + _ReadWriteBarrier(); + + *l = 0; + + return 0; +} + +static inline int pthread_cond_init(pthread_cond_t *c, pthread_condattr_t *a) +{ + (void) a; + + InitializeConditionVariable(c); + return 0; +} + +static inline int pthread_cond_signal(pthread_cond_t *c) +{ + WakeConditionVariable(c); + return 0; +} + +static inline int pthread_cond_broadcast(pthread_cond_t *c) +{ + WakeAllConditionVariable(c); + return 0; +} + +static inline int pthread_cond_wait(pthread_cond_t *c, pthread_mutex_t *m) +{ + pthread_testcancel(); + SleepConditionVariableCS(c, m, INFINITE); + return 0; +} + +static inline int pthread_cond_destroy(pthread_cond_t *c) +{ + (void) c; + return 0; +} + +static inline int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, struct timespec *t) +{ + unsigned long long tm = _pthread_rel_time_in_ms(t); + + pthread_testcancel(); + + if (!SleepConditionVariableCS(c, m, tm)) return ETIMEDOUT; + + /* We can have a spurious wakeup after the timeout */ + if (!_pthread_rel_time_in_ms(t)) return ETIMEDOUT; + + return 0; +} + +static inline int pthread_condattr_destroy(pthread_condattr_t *a) +{ + (void) a; + return 0; +} + +#define pthread_condattr_getclock(A, C) ENOTSUP +#define pthread_condattr_setclock(A, C) ENOTSUP + +static inline int pthread_condattr_init(pthread_condattr_t *a) +{ + *a = 0; + return 0; +} + +static inline int pthread_condattr_getpshared(pthread_condattr_t *a, int *s) +{ + *s = *a; + return 0; +} + +static inline int pthread_condattr_setpshared(pthread_condattr_t *a, int s) +{ + *a = s; + return 0; +} + +static inline int pthread_rwlockattr_destroy(pthread_rwlockattr_t *a) +{ + (void) a; + return 0; +} + +static inline int pthread_rwlockattr_init(pthread_rwlockattr_t *a) +{ + *a = 0; + return 0; +} + +static inline int pthread_rwlockattr_getpshared(pthread_rwlockattr_t *a, int *s) +{ + *s = *a; + return 0; +} + +static inline int pthread_rwlockattr_setpshared(pthread_rwlockattr_t *a, int s) +{ + *a = s; + return 0; +} + + +/* No fork() in windows - so ignore this */ +#define pthread_atfork(F1,F2,F3) 0 + +/* Windows has rudimentary signals support */ +#define pthread_kill(T, S) 0 +#define pthread_sigmask(H, S1, S2) 0 + +static inline int pthread_yield(void) +{ + SwitchToThread(); + return 0; +} + +#ifdef WRAP_CANCELLATION_POINTS +/* Wrap cancellation points */ +#define accept(...) (pthread_testcancel(), accept(__VA_ARGS__)) +#define aio_suspend(...) (pthread_testcancel(), aio_suspend(__VA_ARGS__)) +#define clock_nanosleep(...) (pthread_testcancel(), clock_nanosleep(__VA_ARGS__)) +#define close(...) (pthread_testcancel(), close(__VA_ARGS__)) +#define connect(...) (pthread_testcancel(), connect(__VA_ARGS__)) +#define creat(...) (pthread_testcancel(), creat(__VA_ARGS__)) +#define fcntl(...) (pthread_testcancel(), fcntl(__VA_ARGS__)) +#define fdatasync(...) (pthread_testcancel(), fdatasync(__VA_ARGS__)) +#define fsync(...) (pthread_testcancel(), fsync(__VA_ARGS__)) +#define getmsg(...) (pthread_testcancel(), getmsg(__VA_ARGS__)) +#define getpmsg(...) (pthread_testcancel(), getpmsg(__VA_ARGS__)) +#define lockf(...) (pthread_testcancel(), lockf(__VA_ARGS__)) +#define mg_receive(...) (pthread_testcancel(), mg_receive(__VA_ARGS__)) +#define mg_send(...) (pthread_testcancel(), mg_send(__VA_ARGS__)) +#define mg_timedreceive(...) (pthread_testcancel(), mg_timedreceive(__VA_ARGS__)) +#define mg_timessend(...) (pthread_testcancel(), mg_timedsend(__VA_ARGS__)) +#define msgrcv(...) (pthread_testcancel(), msgrecv(__VA_ARGS__)) +#define msgsnd(...) (pthread_testcancel(), msgsnd(__VA_ARGS__)) +#define msync(...) (pthread_testcancel(), msync(__VA_ARGS__)) +#define nanosleep(...) (pthread_testcancel(), nanosleep(__VA_ARGS__)) +#define open(...) (pthread_testcancel(), open(__VA_ARGS__)) +#define pause(...) (pthread_testcancel(), pause(__VA_ARGS__)) +#define poll(...) (pthread_testcancel(), poll(__VA_ARGS__)) +#define pread(...) (pthread_testcancel(), pread(__VA_ARGS__)) +#define pselect(...) (pthread_testcancel(), pselect(__VA_ARGS__)) +#define putmsg(...) (pthread_testcancel(), putmsg(__VA_ARGS__)) +#define putpmsg(...) (pthread_testcancel(), putpmsg(__VA_ARGS__)) +#define pwrite(...) (pthread_testcancel(), pwrite(__VA_ARGS__)) +#define read(...) (pthread_testcancel(), read(__VA_ARGS__)) +#define readv(...) (pthread_testcancel(), readv(__VA_ARGS__)) +#define recv(...) (pthread_testcancel(), recv(__VA_ARGS__)) +#define recvfrom(...) (pthread_testcancel(), recvfrom(__VA_ARGS__)) +#define recvmsg(...) (pthread_testcancel(), recvmsg(__VA_ARGS__)) +#define select(...) (pthread_testcancel(), select(__VA_ARGS__)) +#define sem_timedwait(...) (pthread_testcancel(), sem_timedwait(__VA_ARGS__)) +#define sem_wait(...) (pthread_testcancel(), sem_wait(__VA_ARGS__)) +#define send(...) (pthread_testcancel(), send(__VA_ARGS__)) +#define sendmsg(...) (pthread_testcancel(), sendmsg(__VA_ARGS__)) +#define sendto(...) (pthread_testcancel(), sendto(__VA_ARGS__)) +#define sigpause(...) (pthread_testcancel(), sigpause(__VA_ARGS__)) +#define sigsuspend(...) (pthread_testcancel(), sigsuspend(__VA_ARGS__)) +#define sigwait(...) (pthread_testcancel(), sigwait(__VA_ARGS__)) +#define sigwaitinfo(...) (pthread_testcancel(), sigwaitinfo(__VA_ARGS__)) +#define sleep(...) (pthread_testcancel(), sleep(__VA_ARGS__)) +//#define Sleep(...) (pthread_testcancel(), Sleep(__VA_ARGS__)) +#define system(...) (pthread_testcancel(), system(__VA_ARGS__)) + + +#define access(...) (pthread_testcancel(), access(__VA_ARGS__)) +#define asctime(...) (pthread_testcancel(), asctime(__VA_ARGS__)) +#define asctime_r(...) (pthread_testcancel(), asctime_r(__VA_ARGS__)) +#define catclose(...) (pthread_testcancel(), catclose(__VA_ARGS__)) +#define catgets(...) (pthread_testcancel(), catgets(__VA_ARGS__)) +#define catopen(...) (pthread_testcancel(), catopen(__VA_ARGS__)) +#define closedir(...) (pthread_testcancel(), closedir(__VA_ARGS__)) +#define closelog(...) (pthread_testcancel(), closelog(__VA_ARGS__)) +#define ctermid(...) (pthread_testcancel(), ctermid(__VA_ARGS__)) +#define ctime(...) (pthread_testcancel(), ctime(__VA_ARGS__)) +#define ctime_r(...) (pthread_testcancel(), ctime_r(__VA_ARGS__)) +#define dbm_close(...) (pthread_testcancel(), dbm_close(__VA_ARGS__)) +#define dbm_delete(...) (pthread_testcancel(), dbm_delete(__VA_ARGS__)) +#define dbm_fetch(...) (pthread_testcancel(), dbm_fetch(__VA_ARGS__)) +#define dbm_nextkey(...) (pthread_testcancel(), dbm_nextkey(__VA_ARGS__)) +#define dbm_open(...) (pthread_testcancel(), dbm_open(__VA_ARGS__)) +#define dbm_store(...) (pthread_testcancel(), dbm_store(__VA_ARGS__)) +#define dlclose(...) (pthread_testcancel(), dlclose(__VA_ARGS__)) +#define dlopen(...) (pthread_testcancel(), dlopen(__VA_ARGS__)) +#define endgrent(...) (pthread_testcancel(), endgrent(__VA_ARGS__)) +#define endhostent(...) (pthread_testcancel(), endhostent(__VA_ARGS__)) +#define endnetent(...) (pthread_testcancel(), endnetent(__VA_ARGS__)) +#define endprotoent(...) (pthread_testcancel(), endprotoend(__VA_ARGS__)) +#define endpwent(...) (pthread_testcancel(), endpwent(__VA_ARGS__)) +#define endservent(...) (pthread_testcancel(), endservent(__VA_ARGS__)) +#define endutxent(...) (pthread_testcancel(), endutxent(__VA_ARGS__)) +#define fclose(...) (pthread_testcancel(), fclose(__VA_ARGS__)) +#define fflush(...) (pthread_testcancel(), fflush(__VA_ARGS__)) +#define fgetc(...) (pthread_testcancel(), fgetc(__VA_ARGS__)) +#define fgetpos(...) (pthread_testcancel(), fgetpos(__VA_ARGS__)) +#define fgets(...) (pthread_testcancel(), fgets(__VA_ARGS__)) +#define fgetwc(...) (pthread_testcancel(), fgetwc(__VA_ARGS__)) +#define fgetws(...) (pthread_testcancel(), fgetws(__VA_ARGS__)) +#define fmtmsg(...) (pthread_testcancel(), fmtmsg(__VA_ARGS__)) +#define fopen(...) (pthread_testcancel(), fopen(__VA_ARGS__)) +#define fpathconf(...) (pthread_testcancel(), fpathconf(__VA_ARGS__)) +#define fprintf(...) (pthread_testcancel(), fprintf(__VA_ARGS__)) +#define fputc(...) (pthread_testcancel(), fputc(__VA_ARGS__)) +#define fputs(...) (pthread_testcancel(), fputs(__VA_ARGS__)) +#define fputwc(...) (pthread_testcancel(), fputwc(__VA_ARGS__)) +#define fputws(...) (pthread_testcancel(), fputws(__VA_ARGS__)) +#define fread(...) (pthread_testcancel(), fread(__VA_ARGS__)) +#define freopen(...) (pthread_testcancel(), freopen(__VA_ARGS__)) +#define fscanf(...) (pthread_testcancel(), fscanf(__VA_ARGS__)) +#define fseek(...) (pthread_testcancel(), fseek(__VA_ARGS__)) +#define fseeko(...) (pthread_testcancel(), fseeko(__VA_ARGS__)) +#define fsetpos(...) (pthread_testcancel(), fsetpos(__VA_ARGS__)) +#define fstat(...) (pthread_testcancel(), fstat(__VA_ARGS__)) +#define ftell(...) (pthread_testcancel(), ftell(__VA_ARGS__)) +#define ftello(...) (pthread_testcancel(), ftello(__VA_ARGS__)) +#define ftw(...) (pthread_testcancel(), ftw(__VA_ARGS__)) +#define fwprintf(...) (pthread_testcancel(), fwprintf(__VA_ARGS__)) +#define fwrite(...) (pthread_testcancel(), fwrite(__VA_ARGS__)) +#define fwscanf(...) (pthread_testcancel(), fwscanf(__VA_ARGS__)) +#define getaddrinfo(...) (pthread_testcancel(), getaddrinfo(__VA_ARGS__)) +#define getc(...) (pthread_testcancel(), getc(__VA_ARGS__)) +#define getc_unlocked(...) (pthread_testcancel(), getc_unlocked(__VA_ARGS__)) +#define getchar(...) (pthread_testcancel(), getchar(__VA_ARGS__)) +#define getchar_unlocked(...) (pthread_testcancel(), getchar_unlocked(__VA_ARGS__)) +#define getcwd(...) (pthread_testcancel(), getcwd(__VA_ARGS__)) +#define getdate(...) (pthread_testcancel(), getdate(__VA_ARGS__)) +#define getgrent(...) (pthread_testcancel(), getgrent(__VA_ARGS__)) +#define getgrgid(...) (pthread_testcancel(), getgrgid(__VA_ARGS__)) +#define getgrgid_r(...) (pthread_testcancel(), getgrgid_r(__VA_ARGS__)) +#define gergrnam(...) (pthread_testcancel(), getgrnam(__VA_ARGS__)) +#define getgrnam_r(...) (pthread_testcancel(), getgrnam_r(__VA_ARGS__)) +#define gethostbyaddr(...) (pthread_testcancel(), gethostbyaddr(__VA_ARGS__)) +#define gethostbyname(...) (pthread_testcancel(), gethostbyname(__VA_ARGS__)) +#define gethostent(...) (pthread_testcancel(), gethostent(__VA_ARGS__)) +#define gethostid(...) (pthread_testcancel(), gethostid(__VA_ARGS__)) +#define gethostname(...) (pthread_testcancel(), gethostname(__VA_ARGS__)) +#define getlogin(...) (pthread_testcancel(), getlogin(__VA_ARGS__)) +#define getlogin_r(...) (pthread_testcancel(), getlogin_r(__VA_ARGS__)) +#define getnameinfo(...) (pthread_testcancel(), getnameinfo(__VA_ARGS__)) +#define getnetbyaddr(...) (pthread_testcancel(), getnetbyaddr(__VA_ARGS__)) +#define getnetbyname(...) (pthread_testcancel(), getnetbyname(__VA_ARGS__)) +#define getnetent(...) (pthread_testcancel(), getnetent(__VA_ARGS__)) +#define getopt(...) (pthread_testcancel(), getopt(__VA_ARGS__)) +#define getprotobyname(...) (pthread_testcancel(), getprotobyname(__VA_ARGS__)) +#define getprotobynumber(...) (pthread_testcancel(), getprotobynumber(__VA_ARGS__)) +#define getprotoent(...) (pthread_testcancel(), getprotoent(__VA_ARGS__)) +#define getpwent(...) (pthread_testcancel(), getpwent(__VA_ARGS__)) +#define getpwnam(...) (pthread_testcancel(), getpwnam(__VA_ARGS__)) +#define getpwnam_r(...) (pthread_testcancel(), getpwnam_r(__VA_ARGS__)) +#define getpwuid(...) (pthread_testcancel(), getpwuid(__VA_ARGS__)) +#define getpwuid_r(...) (pthread_testcancel(), getpwuid_r(__VA_ARGS__)) +#define gets(...) (pthread_testcancel(), gets(__VA_ARGS__)) +#define getservbyname(...) (pthread_testcancel(), getservbyname(__VA_ARGS__)) +#define getservbyport(...) (pthread_testcancel(), getservbyport(__VA_ARGS__)) +#define getservent(...) (pthread_testcancel(), getservent(__VA_ARGS__)) +#define getutxent(...) (pthread_testcancel(), getutxent(__VA_ARGS__)) +#define getutxid(...) (pthread_testcancel(), getutxid(__VA_ARGS__)) +#define getutxline(...) (pthread_testcancel(), getutxline(__VA_ARGS__)) +#undef getwc +#define getwc(...) (pthread_testcancel(), getwc(__VA_ARGS__)) +#undef getwchar +#define getwchar(...) (pthread_testcancel(), getwchar(__VA_ARGS__)) +#define getwd(...) (pthread_testcancel(), getwd(__VA_ARGS__)) +#define glob(...) (pthread_testcancel(), glob(__VA_ARGS__)) +#define iconv_close(...) (pthread_testcancel(), iconv_close(__VA_ARGS__)) +#define iconv_open(...) (pthread_testcancel(), iconv_open(__VA_ARGS__)) +#define ioctl(...) (pthread_testcancel(), ioctl(__VA_ARGS__)) +#define link(...) (pthread_testcancel(), link(__VA_ARGS__)) +#define localtime(...) (pthread_testcancel(), localtime(__VA_ARGS__)) +#define localtime_r(...) (pthread_testcancel(), localtime_r(__VA_ARGS__)) +#define lseek(...) (pthread_testcancel(), lseek(__VA_ARGS__)) +#define lstat(...) (pthread_testcancel(), lstat(__VA_ARGS__)) +#define mkstemp(...) (pthread_testcancel(), mkstemp(__VA_ARGS__)) +#define nftw(...) (pthread_testcancel(), nftw(__VA_ARGS__)) +#define opendir(...) (pthread_testcancel(), opendir(__VA_ARGS__)) +#define openlog(...) (pthread_testcancel(), openlog(__VA_ARGS__)) +#define pathconf(...) (pthread_testcancel(), pathconf(__VA_ARGS__)) +#define pclose(...) (pthread_testcancel(), pclose(__VA_ARGS__)) +#define perror(...) (pthread_testcancel(), perror(__VA_ARGS__)) +#define popen(...) (pthread_testcancel(), popen(__VA_ARGS__)) +#define posix_fadvise(...) (pthread_testcancel(), posix_fadvise(__VA_ARGS__)) +#define posix_fallocate(...) (pthread_testcancel(), posix_fallocate(__VA_ARGS__)) +#define posix_madvise(...) (pthread_testcancel(), posix_madvise(__VA_ARGS__)) +#define posix_openpt(...) (pthread_testcancel(), posix_openpt(__VA_ARGS__)) +#define posix_spawn(...) (pthread_testcancel(), posix_spawn(__VA_ARGS__)) +#define posix_spawnp(...) (pthread_testcancel(), posix_spawnp(__VA_ARGS__)) +#define posix_trace_clear(...) (pthread_testcancel(), posix_trace_clear(__VA_ARGS__)) +#define posix_trace_close(...) (pthread_testcancel(), posix_trace_close(__VA_ARGS__)) +#define posix_trace_create(...) (pthread_testcancel(), posix_trace_create(__VA_ARGS__)) +#define posix_trace_create_withlog(...) (pthread_testcancel(), posix_trace_create_withlog(__VA_ARGS__)) +#define posix_trace_eventtypelist_getne(...) (pthread_testcancel(), posix_trace_eventtypelist_getne(__VA_ARGS__)) +#define posix_trace_eventtypelist_rewin(...) (pthread_testcancel(), posix_trace_eventtypelist_rewin(__VA_ARGS__)) +#define posix_trace_flush(...) (pthread_testcancel(), posix_trace_flush(__VA_ARGS__)) +#define posix_trace_get_attr(...) (pthread_testcancel(), posix_trace_get_attr(__VA_ARGS__)) +#define posix_trace_get_filter(...) (pthread_testcancel(), posix_trace_get_filter(__VA_ARGS__)) +#define posix_trace_get_status(...) (pthread_testcancel(), posix_trace_get_status(__VA_ARGS__)) +#define posix_trace_getnext_event(...) (pthread_testcancel(), posix_trace_getnext_event(__VA_ARGS__)) +#define posix_trace_open(...) (pthread_testcancel(), posix_trace_open(__VA_ARGS__)) +#define posix_trace_rewind(...) (pthread_testcancel(), posix_trace_rewind(__VA_ARGS__)) +#define posix_trace_setfilter(...) (pthread_testcancel(), posix_trace_setfilter(__VA_ARGS__)) +#define posix_trace_shutdown(...) (pthread_testcancel(), posix_trace_shutdown(__VA_ARGS__)) +#define posix_trace_timedgetnext_event(...) (pthread_testcancel(), posix_trace_timedgetnext_event(__VA_ARGS__)) +#define posix_typed_mem_open(...) (pthread_testcancel(), posix_typed_mem_open(__VA_ARGS__)) +#define printf(...) (pthread_testcancel(), printf(__VA_ARGS__)) +#define putc(...) (pthread_testcancel(), putc(__VA_ARGS__)) +#define putc_unlocked(...) (pthread_testcancel(), putc_unlocked(__VA_ARGS__)) +#define putchar(...) (pthread_testcancel(), putchar(__VA_ARGS__)) +#define putchar_unlocked(...) (pthread_testcancel(), putchar_unlocked(__VA_ARGS__)) +#define puts(...) (pthread_testcancel(), puts(__VA_ARGS__)) +#define pututxline(...) (pthread_testcancel(), pututxline(__VA_ARGS__)) +#undef putwc +#define putwc(...) (pthread_testcancel(), putwc(__VA_ARGS__)) +#undef putwchar +#define putwchar(...) (pthread_testcancel(), putwchar(__VA_ARGS__)) +#define readdir(...) (pthread_testcancel(), readdir(__VA_ARSG__)) +#define readdir_r(...) (pthread_testcancel(), readdir_r(__VA_ARGS__)) +#define remove(...) (pthread_testcancel(), remove(__VA_ARGS__)) +#define rename(...) (pthread_testcancel(), rename(__VA_ARGS__)) +#define rewind(...) (pthread_testcancel(), rewind(__VA_ARGS__)) +#define rewinddir(...) (pthread_testcancel(), rewinddir(__VA_ARGS__)) +#define scanf(...) (pthread_testcancel(), scanf(__VA_ARGS__)) +#define seekdir(...) (pthread_testcancel(), seekdir(__VA_ARGS__)) +#define semop(...) (pthread_testcancel(), semop(__VA_ARGS__)) +#define setgrent(...) (pthread_testcancel(), setgrent(__VA_ARGS__)) +#define sethostent(...) (pthread_testcancel(), sethostemt(__VA_ARGS__)) +#define setnetent(...) (pthread_testcancel(), setnetent(__VA_ARGS__)) +#define setprotoent(...) (pthread_testcancel(), setprotoent(__VA_ARGS__)) +#define setpwent(...) (pthread_testcancel(), setpwent(__VA_ARGS__)) +#define setservent(...) (pthread_testcancel(), setservent(__VA_ARGS__)) +#define setutxent(...) (pthread_testcancel(), setutxent(__VA_ARGS__)) +#define stat(...) (pthread_testcancel(), stat(__VA_ARGS__)) +#define strerror(...) (pthread_testcancel(), strerror(__VA_ARGS__)) +#define strerror_r(...) (pthread_testcancel(), strerror_r(__VA_ARGS__)) +#define strftime(...) (pthread_testcancel(), strftime(__VA_ARGS__)) +#define symlink(...) (pthread_testcancel(), symlink(__VA_ARGS__)) +#define sync(...) (pthread_testcancel(), sync(__VA_ARGS__)) +#define syslog(...) (pthread_testcancel(), syslog(__VA_ARGS__)) +#define tmpfile(...) (pthread_testcancel(), tmpfile(__VA_ARGS__)) +#define tmpnam(...) (pthread_testcancel(), tmpnam(__VA_ARGS__)) +#define ttyname(...) (pthread_testcancel(), ttyname(__VA_ARGS__)) +#define ttyname_r(...) (pthread_testcancel(), ttyname_r(__VA_ARGS__)) +#define tzset(...) (pthread_testcancel(), tzset(__VA_ARGS__)) +#define ungetc(...) (pthread_testcancel(), ungetc(__VA_ARGS__)) +#define ungetwc(...) (pthread_testcancel(), ungetwc(__VA_ARGS__)) +#define unlink(...) (pthread_testcancel(), unlink(__VA_ARGS__)) +#define vfprintf(...) (pthread_testcancel(), vfprintf(__VA_ARGS__)) +#define vfwprintf(...) (pthread_testcancel(), vfwprintf(__VA_ARGS__)) +#define vprintf(...) (pthread_testcancel(), vprintf(__VA_ARGS__)) +#define vwprintf(...) (pthread_testcancel(), vwprintf(__VA_ARGS__)) +#define wcsftime(...) (pthread_testcancel(), wcsftime(__VA_ARGS__)) +#define wordexp(...) (pthread_testcancel(), wordexp(__VA_ARGS__)) +#define wprintf(...) (pthread_testcancel(), wprintf(__VA_ARGS__)) +#define wscanf(...) (pthread_testcancel(), wscanf(__VA_ARGS__)) +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined _MSC_VER +#pragma warning(pop) +#endif +#endif /* WIN_PTHREADS */ diff --git a/src/win/pthread/pthread_data.c b/src/win/pthread/pthread_data.c new file mode 100644 index 00000000..542eb92e --- /dev/null +++ b/src/win/pthread/pthread_data.c @@ -0,0 +1,50 @@ +/* + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* This file contains global data referenced in pthread.h */ +#include "pthread.h" + +volatile long _pthread_cancelling; +int _pthread_concur; + +/* Will default to zero as needed */ +pthread_once_t _pthread_tls_once; +DWORD _pthread_tls; + +/* Note initializer is zero, so this works */ +pthread_rwlock_t _pthread_key_lock; +long _pthread_key_max; +long _pthread_key_sch; +void (**_pthread_key_dest)(void *); From 3dcd6af8d2e23f7d1496b56896c3b0c90ea73116 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 11/45] Windows port - port several POSIX functions and definitions Functions used in sysbench were ported : alarm() ,clock_gettime(), basename(), pread(), pwrite(), fdatasync() It also contains an open() portability wrapper that supports O_SYNC and O_DIRECT --- src/CMakeLists.txt | 1 + src/config.h.in | 6 + src/win/CMakeLists.txt | 4 + src/win/sb_win_posix.c | 315 +++++++++++++++++++++++++++++++++++++++++ src/win/sb_win_posix.h | 105 ++++++++++++++ 5 files changed, 431 insertions(+) create mode 100644 src/win/CMakeLists.txt create mode 100644 src/win/sb_win_posix.c create mode 100644 src/win/sb_win_posix.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b053b5c4..30dd2779 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -56,6 +56,7 @@ set_target_properties(sysbench PROPERTIES ENABLE_EXPORTS 1) if(WIN32) # Use pthreads port from Lockless.inc add_subdirectory(win/pthread) + add_subdirectory(win) else() set(CMAKE_THREAD_PREFER_PTHREAD TRUE) set(THREADS_PREFER_PTHREAD_FLAG TRUE) diff --git a/src/config.h.in b/src/config.h.in index 0be3f25f..5d5921a8 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -65,3 +65,9 @@ # endif #endif #define TLS thread_local + +#ifdef _WIN32 +#define SB_WIN_POSIX_NAMES +/* Windows fixups for ssize_t, strcasecmp etc */ +#include "sb_win_posix.h" +#endif diff --git a/src/win/CMakeLists.txt b/src/win/CMakeLists.txt new file mode 100644 index 00000000..a94786b0 --- /dev/null +++ b/src/win/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(sysbench PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/sb_win_posix.h + ${CMAKE_CURRENT_SOURCE_DIR}/sb_win_posix.c) +target_include_directories(sysbench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/win/sb_win_posix.c b/src/win/sb_win_posix.c new file mode 100644 index 00000000..6085d029 --- /dev/null +++ b/src/win/sb_win_posix.c @@ -0,0 +1,315 @@ +/* + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sb_win_posix.h" + +/* + alarm(2) on Windows, implemented with timer queue timers +*/ +static HANDLE timer_handle; +static sb_win_signal_t sigalrm_handler; +static time_t timer_deadline; + +static void CALLBACK timer_func(void* param, BOOLEAN fired) +{ + (void)param; + (void)fired; + sigalrm_handler(SIGALRM); +} + +unsigned int sb_win_alarm(unsigned int seconds) +{ + unsigned int ret = 0; + if (timer_handle) + { + (void)DeleteTimerQueueTimer(NULL, timer_handle, INVALID_HANDLE_VALUE); + timer_handle = NULL; + ret = (unsigned int)(timer_deadline - time(NULL)); + } + if (!seconds) + { + return ret; + } + + timer_deadline = time(NULL) + seconds; + if (!CreateTimerQueueTimer(&timer_handle, NULL, timer_func, NULL, + seconds * 1000, 0, WT_EXECUTEONLYONCE)) + { + abort(); /* alarm does can't return errors, have to abort */ + } + return ret; +} + +/* + signal(), capable of handling SIGALRM. + + CRT signal() does runtime checks, and disallows unknown + signals, so we have to implement our own. +*/ +sb_win_signal_t sb_win_signal(int sig, sb_win_signal_t func) +{ + if (sig == SIGALRM) + { + sb_win_signal_t old = sigalrm_handler; + sigalrm_handler = func; + return old; + } + return signal(sig, func); +} + +static inline HANDLE get_handle(int fd) +{ + intptr_t h = _get_osfhandle(fd); + return (HANDLE)h; +} + + +/* pread() and pwrite()*/ +ssize_t sb_win_pread(int fd, void* buf, size_t count, unsigned long long offset) +{ + HANDLE hFile = get_handle(fd); + if (hFile == INVALID_HANDLE_VALUE) + { + return -1; + } + + OVERLAPPED overlapped; + memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = (DWORD)offset; + overlapped.OffsetHigh = (DWORD)(offset >> 32); + + DWORD bytes_read; + if (!ReadFile(hFile, buf, (DWORD)count, &bytes_read, &overlapped)) + { + return -1; + } + + return (intptr_t)bytes_read; +} + +ssize_t sb_win_pwrite(int fd, const void* buf, size_t count, unsigned long long offset) +{ + HANDLE hFile = get_handle(fd); + if (hFile == INVALID_HANDLE_VALUE) + { + return -1; + } + OVERLAPPED overlapped; + memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = (DWORD)offset; + overlapped.OffsetHigh = (DWORD)(offset >> 32); + + DWORD bytes_written; + if (!WriteFile(hFile, buf, (DWORD)count, &bytes_written, &overlapped)) + { + return -1; + } + return (intptr_t)bytes_written; +} + +/* Get cached query performance frequency value. */ +static inline long long get_qpf(void) +{ + static LARGE_INTEGER f; + static int initialized; + if (initialized) + { + return f.QuadPart; + } + QueryPerformanceFrequency(&f); + initialized = 1; + return f.QuadPart; +} + +/* clock_gettime() */ +int sb_win_clock_gettime(int clock_id, struct timespec* tv) +{ + (void)clock_id; + long long freq = get_qpf(); + LARGE_INTEGER counter; + QueryPerformanceCounter(&counter); + long long now = counter.QuadPart; + tv->tv_sec = (time_t)(now / freq); + tv->tv_nsec = (long)((now % freq) * 1000000000LL / freq); + return 0; +} + +const char* sb_win_basename(const char* path) +{ + const char* p1 = strrchr(path, '\\'); + const char* p2 = strrchr(path, '/'); + if (p1 > p2) + return p1 + 1; + if (p1 < p2) + return p2 + 1; + return path; +} + + +/* + Windows' "fdatasync", NtFlushBuffersFileEx. + Needs to be loaded dynamically, since not in public headers. + Available since Windows 10 1607. +*/ +typedef NTSTATUS(WINAPI *NtFlushBuffersFileEx_func)( + HANDLE FileHandle, ULONG Flags, PVOID Parameters, ULONG ParametersSize, + PIO_STATUS_BLOCK IoStatusBlock); +static NtFlushBuffersFileEx_func sb_NtFlushBuffersFileEx; + + +#ifndef FLUSH_FLAGS_FILE_DATA_SYNC_ONLY +#define FLUSH_FLAGS_FILE_DATA_SYNC_ONLY 0x00000004 +#endif + +static BOOL CALLBACK load_NtFlushBuffersFileEx( + PINIT_ONCE initOnce, PVOID Parameter, PVOID *Context) +{ + (void) initOnce; + (void) Parameter; + (void) Context; + sb_NtFlushBuffersFileEx = (NtFlushBuffersFileEx_func) (void *) + GetProcAddress( GetModuleHandle("ntdll"), "NtFlushBuffersFileEx"); + return TRUE; +} + +int sb_win_fdatasync(int fd) +{ + static INIT_ONCE init_once= INIT_ONCE_STATIC_INIT; + static BOOL disable_datasync; + InitOnceExecuteOnce(&init_once, load_NtFlushBuffersFileEx, NULL, NULL); + + HANDLE h = get_handle(fd); + if (h == NULL || h == INVALID_HANDLE_VALUE) + return -1; + + if (!disable_datasync && sb_NtFlushBuffersFileEx) + { + IO_STATUS_BLOCK iosb; + memset(&iosb, 0, sizeof(iosb)); + NTSTATUS status = sb_NtFlushBuffersFileEx( h, FLUSH_FLAGS_FILE_DATA_SYNC_ONLY, + NULL, 0, &iosb); + if (!status) + { + return 0; + } + } + + /* Fallback to full sync. */ + if (FlushFileBuffers(h)) + { + if (!disable_datasync) + disable_datasync = TRUE; + return 0; + } + errno = EINVAL; + return -1; +} + +/* + Variation of open() that additionally understands O_SYNC and O_DIRECT +*/ +int sb_win_open(const char *path, int oflag) +{ + DWORD access_flag; + switch (oflag & (O_RDONLY | O_WRONLY | O_RDWR)) + { + case O_RDONLY: + access_flag = GENERIC_READ; + break; + case O_WRONLY: + access_flag = GENERIC_WRITE; + break; + case O_RDWR: + access_flag = GENERIC_READ | GENERIC_WRITE; + break; + default: + errno = EINVAL; + return -1; + } + + DWORD disposition; + switch (oflag & (O_CREAT | O_EXCL | O_TRUNC)) + { + case 0: + case O_EXCL: + disposition = OPEN_EXISTING; + break; + case O_CREAT: + disposition = OPEN_ALWAYS; + break; + case O_CREAT | O_EXCL: + case O_CREAT | O_TRUNC | O_EXCL: + disposition = CREATE_NEW; + break; + case O_TRUNC: + case O_TRUNC | O_EXCL: + disposition = TRUNCATE_EXISTING; + break; + case O_CREAT | O_TRUNC: + disposition = CREATE_ALWAYS; + break; + default: + errno = EINVAL; + return -1; + } + + DWORD attr = FILE_ATTRIBUTE_NORMAL; + if (oflag & O_DIRECT) + { + attr |= FILE_FLAG_NO_BUFFERING; + } + if (oflag & O_SYNC) + { + attr |= FILE_FLAG_WRITE_THROUGH; + } + + HANDLE h = CreateFile(path, access_flag, FILE_SHARE_READ, NULL, + disposition, attr, NULL); + if (h == INVALID_HANDLE_VALUE) + { + DWORD last_error = GetLastError(); + switch (last_error) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + errno = ENOENT; + break; + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + errno = EACCES; + break; + case ERROR_FILE_EXISTS: + errno = EEXIST; + break; + default: + errno = EINVAL; + break; + } + return -1; + } + int fd = _open_osfhandle((intptr_t)h, 0); + if (fd < 0) + CloseHandle(h); + return fd; +} diff --git a/src/win/sb_win_posix.h b/src/win/sb_win_posix.h new file mode 100644 index 00000000..d8231185 --- /dev/null +++ b/src/win/sb_win_posix.h @@ -0,0 +1,105 @@ +/* + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +/* + This file contains ports of various, mostly POSIX functions and macros +*/ +#pragma once + +#include +#include +#include + +#ifndef __GNUC__ +#define ssize_t intptr_t +#endif + +/* alarm(), and signal() that understands SIGALRM */ +#ifndef HAVE_ALARM +#define HAVE_ALARM +#endif + +#ifndef SIGALRM +#define SIGALRM 14 +#endif +unsigned int sb_win_alarm(unsigned int seconds); +typedef void (* sb_win_signal_t)(int); +sb_win_signal_t sb_win_signal(int sig, sb_win_signal_t); + +/* pread()/write() */ +int sb_win_open(const char *name, int oflag); +ssize_t sb_win_pread(int fd, void* buf, size_t count, unsigned long long offset); +ssize_t sb_win_pwrite(int fd, const void* buf, size_t count, unsigned long long offset); +int sb_win_fdatasync(int fd); + +/* clock_gettime() */ +#ifndef HAVE_CLOCK_GETTIME +int sb_win_clock_gettime(int id,struct timespec* ts); +#endif + +/* basename() */ +#ifndef HAVE_LIBGEN_H +const char* sb_win_basename(const char* path); +#endif + +#ifndef O_SYNC +#define O_SYNC 0x80000 +#elif O_SYNC == 0 +#error Dummy O_SYNC +#endif + +#ifndef O_DIRECT +#define O_DIRECT 0x100000 +#elif O_DIRECT == 0 +#error Dummy O_DIRECT +#endif + +#ifdef SB_WIN_POSIX_NAMES +#include +#include +#include +#include +#ifndef S_IRUSR +#define S_IRUSR 0x0100 +#endif +#ifndef S_IWUSR +#define S_IWUSR 0x0080 +#endif +#define alarm(sec) sb_win_alarm(sec) +#define signal(sig, func) sb_win_signal(sig, func) +#define pread(fd, buf, count, offset) sb_win_pread(fd, buf, count, offset) +#define pwrite(fd, buf, count, offset) sb_win_pwrite(fd, buf, count, offset) +#define fsync(x) _commit(x) +#define stat _stat64 +#define fstat _fstat64 + +#define strcasecmp _stricmp +#define random() rand() +#define srandom(seed) srand(seed) +#ifndef HAVE_CLOCK_GETTIME +#define HAVE_CLOCK_GETTIME +#define CLOCK_MONOTONIC 0 +#define clock_gettime(id, ts) sb_win_clock_gettime(id,ts) +#endif +#ifndef HAVE_LIBGEN_H +#define basename(path) sb_win_basename(path) +#endif +#ifndef HAVE_FDATASYNC +#define HAVE_FDATASYNC +#define fdatasync(fd) sb_win_fdatasync(fd) +#endif + +#endif From 5ade153f2b7c37fa76c929a732967ddd5d92cbc3 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 12/45] Windows port - declaration of TLS variables for MSVC --- src/config.h.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config.h.in b/src/config.h.in index 5d5921a8..75d230a5 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -57,6 +57,8 @@ #ifndef thread_local # if __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__ # define thread_local _Thread_local +# elif defined _WIN32 && defined _MSC_VER +# define thread_local __declspec(thread) /* note that ICC (linux) and Clang are covered by __GNUC__ */ # elif defined __GNUC__ || defined __SUNPRO_C || defined __xlC__ # define thread_local __thread From 1fbfaa0b1b7771f77a531e6accf52408cfdb683c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 13/45] Windows port - partial port of Concurrency Kit Port some bits of ConcurrencyKit used by sysbench to MSVC compiler. Use native compiler intrinsics for ck_pr_{inc/dec/store} family functions. Copy ck_ring code from concurrency kit, as it is not possible to use the original header. --- cmake/BuildConcurrencyKit.cmake | 4 + cmake/FindConcurrencyKit.cmake | 12 ++ src/CMakeLists.txt | 2 +- src/win/ck/ck_cc.h | 1 + src/win/ck/ck_md.h | 1 + src/win/ck/ck_pr.h | 1 + src/win/ck/ck_ring.h | 1 + src/win/ck/ck_spinlock.h | 1 + src/win/ck/win_ck.h | 350 ++++++++++++++++++++++++++++++++ 9 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 src/win/ck/ck_cc.h create mode 100644 src/win/ck/ck_md.h create mode 100644 src/win/ck/ck_pr.h create mode 100644 src/win/ck/ck_ring.h create mode 100644 src/win/ck/ck_spinlock.h create mode 100644 src/win/ck/win_ck.h diff --git a/cmake/BuildConcurrencyKit.cmake b/cmake/BuildConcurrencyKit.cmake index a35a7c84..253fbae5 100644 --- a/cmake/BuildConcurrencyKit.cmake +++ b/cmake/BuildConcurrencyKit.cmake @@ -1,3 +1,7 @@ +if(WIN32) + message(FATAL_ERROR "concurrency kit can't be built on Windows") +endif() + # Compile concurrency kit from source Currently, we do not even need to link the # library we only use functionality from headers only diff --git a/cmake/FindConcurrencyKit.cmake b/cmake/FindConcurrencyKit.cmake index 723ad6d4..5db1e948 100644 --- a/cmake/FindConcurrencyKit.cmake +++ b/cmake/FindConcurrencyKit.cmake @@ -1,3 +1,15 @@ +if(WIN32) + # On Windows, we just use own header-only library + if(NOT TARGET ConcurrencyKit) + add_library(ConcurrencyKit INTERFACE) + target_include_directories( + ConcurrencyKit INTERFACE ${PROJECT_SOURCE_DIR}/src/win/ck + ${PROJECT_SOURCE_DIR}/src/win/pthread) + endif() + set(ConcurrencyKit_FOUND TRUE) + return() +endif() + find_package(PkgConfig) if(PKG_CONFIG_FOUND) pkg_check_modules(PC_CK QUIET ck) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 30dd2779..3e5e412b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -130,7 +130,7 @@ endif() set(WITH_SYSTEM_CONCURRENCY_KIT OFF CACHE BOOL "Use concurrency kit library provided by system") -if(WITH_SYSTEM_CONCURRENCY_KIT) +if(WITH_SYSTEM_CONCURRENCY_KIT OR WIN32) find_package(ConcurrencyKit REQUIRED) else() include(BuildConcurrencyKit) diff --git a/src/win/ck/ck_cc.h b/src/win/ck/ck_cc.h new file mode 100644 index 00000000..a3dd71b2 --- /dev/null +++ b/src/win/ck/ck_cc.h @@ -0,0 +1 @@ +#include "win_ck.h" diff --git a/src/win/ck/ck_md.h b/src/win/ck/ck_md.h new file mode 100644 index 00000000..a3dd71b2 --- /dev/null +++ b/src/win/ck/ck_md.h @@ -0,0 +1 @@ +#include "win_ck.h" diff --git a/src/win/ck/ck_pr.h b/src/win/ck/ck_pr.h new file mode 100644 index 00000000..a3dd71b2 --- /dev/null +++ b/src/win/ck/ck_pr.h @@ -0,0 +1 @@ +#include "win_ck.h" diff --git a/src/win/ck/ck_ring.h b/src/win/ck/ck_ring.h new file mode 100644 index 00000000..a3dd71b2 --- /dev/null +++ b/src/win/ck/ck_ring.h @@ -0,0 +1 @@ +#include "win_ck.h" diff --git a/src/win/ck/ck_spinlock.h b/src/win/ck/ck_spinlock.h new file mode 100644 index 00000000..a3dd71b2 --- /dev/null +++ b/src/win/ck/ck_spinlock.h @@ -0,0 +1 @@ +#include "win_ck.h" diff --git a/src/win/ck/win_ck.h b/src/win/ck/win_ck.h new file mode 100644 index 00000000..668ad69a --- /dev/null +++ b/src/win/ck/win_ck.h @@ -0,0 +1,350 @@ +/* + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +/* +* This is Windows port for the bits of Concurrency Kit, that are used by sysbench. +* Mostly implemented using compiler intrinsics. +* +* ring queue was lifted literally from ck_ring.h (unfortunately, it is not possible +* to include the original file due to dependencies on GCC style compiler). +* +* ck_ring.h includes the following copyright notice +*/ + +/* +* +* Copyright 2009-2015 Samy Al Bahra. +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +* SUCH DAMAGE. +*/ + +#pragma once + +#include +#include +#include "pthread.h" /* spinlock*/ +#include /* bool */ +#include /* uint32_t */ +#if defined __GNUC__ || defined __clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wbad-function-cast" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wincompatible-pointer-types" +#pragma GCC diagnostic ignored "-Wpointer-sign" +#endif + +#define CK_MD_CACHELINE 64 +#if defined (_MSC_VER) +#define CK_CC_CACHELINE __declspec(align(CK_MD_CACHELINE)) +#elif defined (__GNUC__) +#define CK_CC_CACHELINE __attribute__((aligned(CK_MD_CACHELINE))) +#endif +#define CK_CC_UNLIKELY(x) x +#define CK_CC_LIKELY(x) x +#define CK_CC_INLINE inline +#define CK_CC_FORCE_INLINE __forceinline +#define CK_CC_RESTRICT __restrict + + +#define ck_spinlock_t pthread_spinlock_t +#define ck_spinlock_init(x) pthread_spin_init(x, PTHREAD_PROCESS_PRIVATE) +#define ck_spinlock_lock pthread_spin_lock +#define ck_spinlock_unlock pthread_spin_unlock + +static inline void ck_pr_barrier(void) +{ + _ReadWriteBarrier(); +} +static inline void ck_pr_fence_load(void) +{ + MemoryBarrier(); +} +static inline void ck_pr_dec_int(int* target) +{ + _InterlockedDecrement(target); +} + +static inline void ck_pr_dec_uint(unsigned int* target) +{ + _InterlockedDecrement(target); +} + +static inline void ck_pr_inc_int(int* target) +{ + _InterlockedIncrement(target); +} +static inline void ck_pr_store_int(int* target, int value) +{ + *target = value; + _ReadWriteBarrier(); +} +static inline void ck_pr_store_uint(unsigned int* target, unsigned int value) +{ + *target = value; + _ReadWriteBarrier(); +} +static inline uint32_t ck_pr_faa_uint(unsigned int* target, unsigned int delta) +{ + return InterlockedExchangeAdd(target, delta); +} +static inline uint32_t ck_pr_faa_32(uint32_t * target, uint32_t delta) +{ + return InterlockedExchangeAdd(target, delta); +} +static inline uint64_t ck_pr_faa_64(uint64_t* target, uint64_t delta) +{ + return InterlockedExchangeAdd64(target, delta); +} +static inline uint64_t ck_pr_load_64(uint64_t* target) +{ +#ifdef _WIN64 + _ReadBarrier(); + return *target; +#else + return InterlockedXor64(target, 0); +#endif +} +static inline void ck_pr_store_64(uint64_t* target, uint64_t value) +{ + *target = value; + _ReadWriteBarrier(); +} + +static inline void ck_pr_fence_store(void) +{ + MemoryBarrier(); +} +static inline uint64_t ck_pr_fas_64(uint64_t* target, uint64_t value) +{ + return InterlockedExchange64(target, value); +} +static inline uint64_t ck_pr_inc_64(uint64_t* target) +{ + return InterlockedIncrement64(target); +} + +static inline uint8_t ck_pr_load_8(const uint8_t* target) +{ +#ifdef __MINGW64__ + /* Mingw is missing native Windows atomic, use gcc one */ + return __atomic_load_n(target, __ATOMIC_SEQ_CST); +#else + return InterlockedXor8((volatile char *)target,0); +#endif +} + +static inline void ck_pr_store_8(uint8_t* target, uint8_t value) +{ +#ifdef __MINGW64__ + return __atomic_store_n(target, value, __ATOMIC_SEQ_CST); +#else + InterlockedExchange8(target, value); +#endif +} + +static inline uint32_t ck_pr_load_32(uint32_t *target) +{ + _ReadBarrier(); + return *target; +} +static inline void ck_pr_store_32(uint32_t* target, uint32_t value) +{ + *target = value; + _ReadWriteBarrier(); +} +static inline int ck_pr_load_int(const int* target) +{ + _ReadBarrier(); + return *target; +} +static inline unsigned int ck_pr_load_uint(const unsigned int* target) +{ + _ReadBarrier(); + return *target; +} +static inline void ck_pr_fence_store_atomic(void) +{ + MemoryBarrier(); +} + +static inline bool +ck_pr_cas_uint_value(unsigned int* target, unsigned int old_value, unsigned int new_value, unsigned int* original_value) +{ + *original_value = InterlockedCompareExchange(target, new_value, old_value); + return *original_value == old_value; +} +/* + * Concurrent ring buffer. + */ + +struct ck_ring { + unsigned int c_head; + char pad[CK_MD_CACHELINE - sizeof(unsigned int)]; + unsigned int p_tail; + unsigned int p_head; + char _pad[CK_MD_CACHELINE - sizeof(unsigned int) * 2]; + unsigned int size; + unsigned int mask; +}; +typedef struct ck_ring ck_ring_t; + +struct ck_ring_buffer { + void* value; +}; +typedef struct ck_ring_buffer ck_ring_buffer_t; + +CK_CC_INLINE static unsigned int +ck_ring_size(const struct ck_ring* ring) +{ + unsigned int c, p; + + c = ck_pr_load_uint(&ring->c_head); + p = ck_pr_load_uint(&ring->p_tail); + return (p - c) & ring->mask; +} +CK_CC_INLINE static unsigned int +ck_ring_capacity(const struct ck_ring* ring) +{ + return ring->size; +} + +CK_CC_INLINE static void +ck_ring_init(struct ck_ring* ring, unsigned int size) +{ + + ring->size = size; + ring->mask = size - 1; + ring->p_tail = 0; + ring->p_head = 0; + ring->c_head = 0; + return; +} + +/* + * The _ck_ring_* namespace is internal only and must not used externally. + */ +CK_CC_INLINE static bool +_ck_ring_enqueue_sp(struct ck_ring* ring, + void* CK_CC_RESTRICT buffer, + const void* CK_CC_RESTRICT entry, + unsigned int ts, + unsigned int* size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer, delta; + + consumer = ck_pr_load_uint(&ring->c_head); + producer = ring->p_tail; + delta = producer + 1; + if (size != NULL) + *size = (producer - consumer) & mask; + + if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask))) + return false; + + buffer = (char*)buffer + ts * (producer & mask); + memcpy(buffer, entry, ts); + + /* + * Make sure to update slot value before indicating + * that the slot is available for consumption. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->p_tail, delta); + return true; +} +CK_CC_INLINE static bool +ck_ring_enqueue_spmc(struct ck_ring* ring, + struct ck_ring_buffer* buffer, + const void* entry) +{ + + return _ck_ring_enqueue_sp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +_ck_ring_dequeue_mc(struct ck_ring* ring, + const void* buffer, + void* data, + unsigned int ts) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ck_pr_load_uint(&ring->c_head); + + do { + const char* target; + + /* + * Producer counter must represent state relative to + * our latest consumer snapshot. + */ + ck_pr_fence_load(); + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + ck_pr_fence_load(); + + target = (const char*)buffer + ts * (consumer & mask); + memcpy(data, target, ts); + + /* Serialize load with respect to head update. */ + ck_pr_fence_store_atomic(); + } while (ck_pr_cas_uint_value(&ring->c_head, + consumer, + consumer + 1, + &consumer) == false); + + return true; +} + +CK_CC_INLINE static bool +ck_ring_dequeue_spmc(struct ck_ring* ring, + const struct ck_ring_buffer* buffer, + void* data) +{ + return _ck_ring_dequeue_mc(ring, buffer, (void**)data, sizeof(void*)); +} +#define CK_F_PR_LOAD_64 1 +#if defined __GNUC__ || defined __clang__ +#pragma GCC diagnostic pop +#endif + From fae8c83ef37eb61bb9042636a5e01bcc190e40a7 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 14/45] Windows port - build or find LuaJit library on Windows Use instructions from Luajit documentation to build the library. Allow FindLuajit.cmake to find luajit if it is built by vcpkg, if system luajit library is requested. Also fix BuildLuajit.cmake to work with Ninja --- cmake/BuildLuaJit.cmake | 102 ++++++++++++++++++++++++++++------------ cmake/FindLuaJit.cmake | 45 ++++++++++++++---- 2 files changed, 110 insertions(+), 37 deletions(-) diff --git a/cmake/BuildLuaJit.cmake b/cmake/BuildLuaJit.cmake index d10586ef..f56eb7f8 100644 --- a/cmake/BuildLuaJit.cmake +++ b/cmake/BuildLuaJit.cmake @@ -1,41 +1,85 @@ # Build the bundled luajit library and create imported library. # luajit::libluajit. # -# This follows the instructions on the luajit page +# This follows the instructions on the luajit page Both Unix and MSVC build are +# done in-source, with "make" rsp batch file On *Nix, luajit::libluajit is a +# static library,on Windows its a DLL (this has to be so) include(ExternalProject) set(build_dir ${PROJECT_BINARY_DIR}/build/third_party/luajit) set(install_dir ${build_dir}/install) set(url ${PROJECT_SOURCE_DIR}/third_party/luajit/luajit) - -# Note : luajit's Makefile is non-portable (GNU)it could fail on BSD, unless -# gmake is there´ -find_program(MAKE_EXECUTABLE NAMES gmake make REQUIRED) -# Makefile requires MACOSX_DEPLOYMENT_TARGET on Apple -if(CMAKE_OSX_DEPLOYMENT_TARGET) - set(MACOSX_DEPLOYMENT_TARGET - "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") +if(NOT WIN32) + # Note : luajit's Makefile is non-portable (GNU)it could fail on BSD, unless + # gmake is there´ + find_program(MAKE_EXECUTABLE NAMES gmake make REQUIRED) + # Makefile requires MACOSX_DEPLOYMENT_TARGET on Apple + if(CMAKE_OSX_DEPLOYMENT_TARGET) + set(MACOSX_DEPLOYMENT_TARGET + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() + set(libluajit_static_lib ${install_dir}/lib/libluajit-5.1.a) + ExternalProject_Add( + buildLuaJit + URL ${url} + SOURCE_DIR ${build_dir} + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_ALWAYS FALSE + BUILD_COMMAND ${MAKE_EXECUTABLE} install PREFIX=${install_dir} ${MACOSX_DEPLOYMENT_TARGET} + BUILD_BYPRODUCTS ${libluajit_static_lib}) + add_library(luajit::libluajit STATIC IMPORTED) + make_directory(${install_dir}/include/luajit-2.1) + set_target_properties( + luajit::libluajit + PROPERTIES IMPORTED_LOCATION + ${libluajit_static_lib} + INTERFACE_INCLUDE_DIRECTORIES + ${install_dir}/include/luajit-2.1 + INTERFACE_LINK_LIBRARIES "m;${CMAKE_DL_LIBS}") +else() + if (MINGW) + # GNU compatible make required + find_program(MAKE_EXECUTABLE NAMES gmake REQUIRED) + set(buildCommand ${MAKE_EXECUTABLE}) + set(luajit_implib ${build_dir}/src/libluajit-5.1.dll.a) + else() + #MSVC + set(buildCommand ${CMAKE_COMMAND} -E chdir src cmd.exe /c msvcbuild.bat debug) + set(luajit_implib ${build_dir}/src/lua51.lib) + endif() + set(luajit_dll ${build_dir}/src/lua51.dll) + ExternalProject_Add( + buildLuaJit + URL ${url} + SOURCE_DIR ${build_dir} + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "" + BUILD_COMMAND ${buildCommand} + BUILD_BYPRODUCTS ${luajit_implib} ${luajit_dll} + BUILD_ALWAYS FALSE) + add_library(luajit::libluajit SHARED IMPORTED) + set_target_properties( + luajit::libluajit + PROPERTIES IMPORTED_IMPLIB + ${luajit_implib} + IMPORTED_LOCATION + ${luajit_dll} + INTERFACE_INCLUDE_DIRECTORIES + ${PROJECT_SOURCE_DIR}/third_party/luajit/luajit/src) endif() -set(lubluajit_static_lib ${install_dir}/lib/libluajit-5.1.a) -ExternalProject_Add( - buildLuaJit - URL ${url} - SOURCE_DIR ${build_dir} - CONFIGURE_COMMAND "" - INSTALL_COMMAND "" - BUILD_IN_SOURCE 1 - BUILD_ALWAYS FALSE - BUILD_COMMAND ${MAKE_EXECUTABLE} install PREFIX=${install_dir} ${MACOSX_DEPLOYMENT_TARGET} - BUILD_BYPRODUCTS ${libluajit_static_lib}) -add_library(luajit::libluajit STATIC IMPORTED) -make_directory(${install_dir}/include/luajit-2.1) -set_target_properties( - luajit::libluajit - PROPERTIES IMPORTED_LOCATION - ${lubluajit_static_lib} - INTERFACE_INCLUDE_DIRECTORIES - ${install_dir}/include/luajit-2.1 - INTERFACE_LINK_LIBRARIES "m;${CMAKE_DL_LIBS}") + add_dependencies(luajit::libluajit buildLuaJit) +if(WIN32) + # Lua files below are used by sysbench --luajit-cmd + # The location must be bin\lua\jit + install( + DIRECTORY ${build_dir}/src/jit + DESTINATION ${CMAKE_INSTALL_BINDIR}/lua + USE_SOURCE_PERMISSIONS FILES_MATCHING + PATTERN "*.lua") +endif() diff --git a/cmake/FindLuaJit.cmake b/cmake/FindLuaJit.cmake index 41e4dff3..4e950e1e 100644 --- a/cmake/FindLuaJit.cmake +++ b/cmake/FindLuaJit.cmake @@ -19,8 +19,17 @@ FIND_PATH(LUA_INCLUDE_DIR luajit.h /opt ) - -FIND_LIBRARY(LUA_LIBRARY +# Test if running on vcpkg toolchain +if(WIN32 OR (DEFINED VCPKG_TARGET_TRIPLET AND DEFINED VCPKG_APPLOCAL_DEPS)) + # On vcpkg luajit is 'lua51' and normal lua is 'lua' + FIND_LIBRARY(LUA_LIBRARY + NAMES lua51 + HINTS + $ENV{LUA_DIR} + PATH_SUFFIXES lib + ) +else() + FIND_LIBRARY(LUA_LIBRARY NAMES luajit-5.1 HINTS @@ -34,20 +43,40 @@ FIND_LIBRARY(LUA_LIBRARY /opt/csw /opt ) +endif() + +IF(WIN32) + FIND_FILE(LUA_LIBRARY_DLL + NAMES lua51.dll + PATH_SUFFIXES ../../bin + ) + set(LUA_LIBRARY_DLL_VAR LUA_LIBRARY_DLL) +else() + set(LUA_LIBRARY_DLL_VAR) +endif() INCLUDE(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set LuaJit_FOUND to TRUE if # all listed variables exist FIND_PACKAGE_HANDLE_STANDARD_ARGS(LuaJit - REQUIRED_VARS LUA_LIBRARY LUA_INCLUDE_DIR) + REQUIRED_VARS LUA_LIBRARY LUA_INCLUDE_DIR ${LUA_LIBRARY_DLL_VAR}) -MARK_AS_ADVANCED(LUA_INCLUDE_DIR LUA_LIBRARY) +MARK_AS_ADVANCED(LUA_INCLUDE_DIR LUA_LIBRARY ${LUA_LIBRARY_DLL_VAR}) if (LuaJit_FOUND AND NOT TARGET luajit::libluajit) - add_library(luajit::libluajit UNKNOWN IMPORTED) - set_target_properties(luajit::libluajit PROPERTIES - IMPORTED_LOCATION "${LUA_LIBRARY}" - INTERFACE_LINK_LIBRARIES "m;${CMAKE_DL_LIBS}" # Linuxism? + if(WIN32) + add_library(luajit::libluajit SHARED IMPORTED) + get_filename_component(LUA_LIBRARY_DLL "${LUA_LIBRARY_DLL}" REALPATH) + set_target_properties(luajit::libluajit PROPERTIES + IMPORTED_LOCATION "${LUA_LIBRARY_DLL}" + IMPORTED_IMPLIB "${LUA_LIBRARY}") + else() + add_library(luajit::libluajit UNKNOWN IMPORTED) + set_target_properties(luajit::libluajit PROPERTIES + IMPORTED_LOCATION "${LUA_LIBRARY}" + INTERFACE_LINK_LIBRARIES "m;${CMAKE_DL_LIBS}" # Linuxism? + ) + endif() set_target_properties(luajit::libluajit PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LUA_INCLUDE_DIR}") endif() From 5a4c2f70e848da6af187a3b504e967f7d7de6140 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 15/45] Windows port - CMake option WITH_LIBMARIADB, build with libmariadb from source As Windows does not have system libraries for MySQL or MariaDB client, for the sake of simplicity, allow to build libmariadb as external project during sysbench build. That is, if WITH_LIBMARIADB is defined, libmariadb will be built from source https://github.com/mariadb-corporation/mariadb-connector-c, courtesy of CMake's ExternalProject_Add libmariadb is chosen because it is easy to build standalone (as opposed to libmysqlclient) and works well with named pipes, which libmysqlclient has problems with. WITH_LIBMARIADB defaults to ON on Windows, as to OFF elsewhere. It is built as shared library on Windows, as static elsewhere( to avoid linker warning about mix of C runtimes) --- cmake/BuildLibmariadb.cmake | 142 ++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 19 ++++- 2 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 cmake/BuildLibmariadb.cmake diff --git a/cmake/BuildLibmariadb.cmake b/cmake/BuildLibmariadb.cmake new file mode 100644 index 00000000..66f04927 --- /dev/null +++ b/cmake/BuildLibmariadb.cmake @@ -0,0 +1,142 @@ +# Build libmariadb from source On *Nix, this is done when mariadb/mysql client +# system library is missing, and USE_MYSQL is not set to OFF +# +# On Windows, we build it by default, "thanks" to subpar quality of vcpkg's +# libmariadb/libmysql libraries + +include(ExternalProject) +set(src_dir ${PROJECT_BINARY_DIR}/build/third_party/src/libmariadb) +set(build_dir ${PROJECT_BINARY_DIR}/build/third_party/libmariadb) +set(install_dir ${build_dir}/install) + +if(WIN32) + set(mariadbclient_LIBRARY_NAME libmariadb.dll) +else() + set(mariadbclient_LIBRARY_NAME libmariadbclient.a) +endif() + +set(mariadbclient_IMPORTED_LOCATION + ${install_dir}/lib/mariadb/${mariadbclient_LIBRARY_NAME}) + +set(_EXTRA_CMAKE_ARGS) + +if(WIN32) + if(MINGW) + set(import_suffix .dll.a) + else() + set(import_suffix .lib) + endif() + list(APPEND _EXTRA_CMAKE_ARGS -DCLIENT_PLUGIN_PVIO_NPIPE=STATIC) + set(mariadbclient_IMPORTED_IMPLIB + ${install_dir}/lib/mariadb/${CMAKE_SHARED_LIBRARY_PREFIX}libmariadb${import_suffix} + ) +endif() + +if(MSVC) + list(APPEND _EXTRA_CMAKE_ARGS -DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE=ON + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO=ON + -DCMAKE_POLICY_DEFAULT_CMP0069=NEW) +elseif(WIN32) + # gcc/clang is not a native env for the mariadb client library can compile + # only with some extra flags + set(_CFLAGS + "-DWIN32_MEAN_AND_LEAN -DNOGDI -DNOMINMAX -D_CRT_NONSTDC_NO_WARNINGS \ + -D_CRT_SECURE_NO_WARNINGS -D_WINSOCKAPI_ -DNOCRYPT -DNOCOMM\ + ") + if(CMAKE_C_COMPILER_ID STREQUAL "Clang") + string(APPEND _CFLAGS " -Wno-error") + endif() + if(CMAKE_COMPILER_IS_GNUCC) + string(APPEND _CFLAGS + " -Dstrerror_r(errno,buf,len)=strerror_s(buf,len,errno)") + endif() + list(APPEND _EXTRA_CMAKE_ARGS "-DCMAKE_C_FLAGS_INIT=${_CFLAGS}") +endif() + +set(LIBMARIADB_GIT_TAG + v3.3.4 + CACHE + STRING + "Git tag of mariadb client library. Set to empty string to get most recent revision" +) +if(LIBMARIADB_GIT_TAG) + set(_GIT_TAG GIT_TAG ${LIBMARIADB_GIT_TAG}) +endif() +find_package(ZLIB) +if(ZLIB_FOUND) + list(APPEND _EXTRA_CMAKE_ARGS -DWITH_EXTERNAL_ZLIB=ON) +endif() + +# pass down toolchain related variables (important for vcpkg) +foreach(varname CMAKE_TOOLCHAIN_FILE VCPKG_TARGET_TRIPLET VCPKG_HOST_TRIPLET) + if(${varname}) + list(APPEND _EXTRA_CMAKE_ARGS "-D${varname}=${${varname}}") + endif() +endforeach() + +if(CMAKE_PREFIX_PATH) + # Create a list with an alternate separator e.g. pipe symbol + string(REPLACE ";" "|" CMAKE_PREFIX_PATH_ALT_SEP "${CMAKE_PREFIX_PATH}") + list(APPEND _EXTRA_CMAKE_ARGS + "-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH_ALT_SEP}") + list(APPEND _LIST_SEPARATOR LIST_SEPARATOR |) +else() + set(_LIST_SEPARATOR) +endif() + +ExternalProject_Add( + buildLibmariadb + GIT_REPOSITORY https://github.com/mariadb-corporation/mariadb-connector-c + ${_GIT_TAG} + SOURCE_DIR ${src_dir} + BINARY_DIR ${build_dir} + UPDATE_COMMAND "" + PATCH_COMMAND "" + BUILD_ALWAYS FALSE + ${_LIST_SEPARATOR} + CMAKE_ARGS -G${CMAKE_GENERATOR} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCLIENT_PLUGIN_CACHING_SHA2_PASSWORD=STATIC + -DCLIENT_PLUGIN_DIALOG=OFF + -DCLIENT_PLUGIN_CLIENT_ED25519=OFF + -DCLIENT_PLUGIN_SHA256_PASSWORD=OFF + -DCLIENT_PLUGIN_MYSQL_CLEAR_PASSWORD=OFF + -DCLIENT_PLUGIN_ZSTD=OFF + -DSKIP_TESTS=1 + -DCMAKE_DISABLE_FIND_PACKAGE_CURL=1 + -DCMAKE_INSTALL_PREFIX=${install_dir} + ${_EXTRA_CMAKE_ARGS} + -Wno-dev + --no-warn-unused-cli + BUILD_BYPRODUCTS ${mariadbclient_IMPORTED_LOCATION} + ${mariadbclient_IMPORTED_IMPLIB}) +make_directory(${install_dir}/include/mariadb) +if(WIN32) + add_library(mariadbclient SHARED IMPORTED GLOBAL) + set_target_properties( + mariadbclient + PROPERTIES IMPORTED_LOCATION ${mariadbclient_IMPORTED_LOCATION} + IMPORTED_IMPLIB ${mariadbclient_IMPORTED_IMPLIB} + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/mariadb) +else() + find_package(OpenSSL) + if(NOT OpenSSL_FOUND) + message( + FATAL_ERROR + "Could not find OpenSSL, which is required to build libmariadb from source" + ) + endif() + set(mariadbclient_INTERFACE_LINK_LIBRARIES m ${CMAKE_DL_LIBS} OpenSSL::SSL) + if(ZLIB_FOUND) + list(APPEND mariadbclient_INTERFACE_LINK_LIBRARIES ZLIB::ZLIB) + endif() + add_library(mariadbclient STATIC IMPORTED GLOBAL) + set_target_properties( + mariadbclient + PROPERTIES IMPORTED_LOCATION ${mariadbclient_IMPORTED_LOCATION} + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/mariadb + INTERFACE_LINK_LIBRARIES + "${mariadbclient_INTERFACE_LINK_LIBRARIES}") +endif() +add_dependencies(mariadbclient buildLibmariadb) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3e5e412b..9f6e4080 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -90,11 +90,26 @@ else() endif() target_link_libraries(sysbench PRIVATE luajit::libluajit) +# On Windows, we prefer libmariadb built from source (as external project) to +# provide better named pipe support On *nix, we prefer system +# libmysql/libmariadb +if(WIN32) + set(_WITH_LIBMARIADB_DEFAULT ON) +else() + set(_WITH_LIBMARIADB_DEFAULT OFF) +endif() +set(WITH_LIBMARIADB + ${_WITH_LIBMARIADB_DEFAULT} + CACHE BOOL "Build libmariadb from source (as external project)") +if(WITH_LIBMARIADB) + include(BuildLibmariadb) + target_link_libraries(sysbench PRIVATE mariadbclient) +endif() set(WITH_MYSQL ON CACHE BOOL "Enable mysql support for database tests") -if(WITH_MYSQL) +if(WITH_MYSQL AND (NOT WITH_LIBMARIADB)) find_package(MySQL REQUIRED) target_link_libraries(sysbench PRIVATE MySQL::MySQL) set(CMAKE_REQUIRED_INCLUDES ${MySQL_INCLUDE_DIRS}) @@ -109,7 +124,7 @@ if(WITH_MYSQL) HAVE_MYSQL_OPT_SSL_MODE) endif() -if(WITH_MYSQL) +if(WITH_MYSQL OR WITH_LIBMARIADB) target_compile_definitions(sysbench PRIVATE -DUSE_MYSQL) target_sources(sysbench PRIVATE drivers/mysql/drv_mysql.c) endif() From dfee295e7ea3567ce3709f199c479eef5450c77e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 16/45] Windows port - fix for CK_CC_CACHELINE with MSVC. MSVC does not allow to use declspec(align) after the variable definition. So CK_CC_CACHELINE is moved before variable declaration. --- src/db_driver.c | 2 +- src/sb_counter.c | 2 +- src/sb_histogram.c | 2 +- src/sb_lua.c | 6 +++--- src/sb_rand.c | 4 ++-- src/sysbench.c | 14 +++++++------- src/sysbench.h | 20 ++++++++++---------- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/db_driver.c b/src/db_driver.c index 97310a66..c495f8cb 100644 --- a/src/db_driver.c +++ b/src/db_driver.c @@ -46,7 +46,7 @@ #define ROWS_BEFORE_COMMIT 1000 /* Global variables */ -db_globals_t db_globals CK_CC_CACHELINE; +CK_CC_CACHELINE db_globals_t db_globals; static sb_list_t drivers; /* list of available DB drivers */ diff --git a/src/sb_counter.c b/src/sb_counter.c index 355329fc..333a0183 100644 --- a/src/sb_counter.c +++ b/src/sb_counter.c @@ -25,7 +25,7 @@ #include "sb_util.h" #include "sb_ck_pr.h" -sb_counters_t *sb_counters CK_CC_CACHELINE; +CK_CC_CACHELINE sb_counters_t *sb_counters ; static sb_counters_t last_intermediate_counters; static sb_counters_t last_cumulative_counters; diff --git a/src/sb_histogram.c b/src/sb_histogram.c index 88c9ad6f..e18185e3 100644 --- a/src/sb_histogram.c +++ b/src/sb_histogram.c @@ -48,7 +48,7 @@ #define SB_HISTOGRAM_NSLOTS 128 /* Global latency histogram */ -sb_histogram_t sb_latency_histogram CK_CC_CACHELINE; +CK_CC_CACHELINE sb_histogram_t sb_latency_histogram; int sb_histogram_init(sb_histogram_t *h, size_t size, diff --git a/src/sb_lua.c b/src/sb_lua.c index 4ec038bc..2dc55f35 100644 --- a/src/sb_lua.c +++ b/src/sb_lua.c @@ -96,11 +96,11 @@ int sb_lua_set_test_args(sb_arg_t *, size_t); /* Lua interpreter states */ -static lua_State **states CK_CC_CACHELINE; +static CK_CC_CACHELINE lua_State **states; -static sb_test_t sbtest CK_CC_CACHELINE; +static CK_CC_CACHELINE sb_test_t sbtest; -static TLS sb_lua_ctxt_t tls_lua_ctxt CK_CC_CACHELINE; +static CK_CC_CACHELINE TLS sb_lua_ctxt_t tls_lua_ctxt; /* List of pre-loaded internal scripts */ static internal_script_t internal_scripts[] = { diff --git a/src/sb_rand.c b/src/sb_rand.c index 563b566e..4a7cabc9 100644 --- a/src/sb_rand.c +++ b/src/sb_rand.c @@ -56,7 +56,7 @@ #include "sb_ck_pr.h" -TLS sb_rng_state_t sb_rng_state CK_CC_CACHELINE; +TLS CK_CC_CACHELINE sb_rng_state_t sb_rng_state; /* Exported variables */ int sb_rand_seed; /* optional seed set on the command line */ @@ -106,7 +106,7 @@ static double zipf_s; static double zipf_hIntegralX1; /* Unique sequence generator state */ -static uint32_t rand_unique_index CK_CC_CACHELINE; +static CK_CC_CACHELINE uint32_t rand_unique_index; static uint32_t rand_unique_offset; extern inline uint64_t sb_rand_uniform_uint64(void); diff --git a/src/sysbench.c b/src/sysbench.c index fb83bef4..4714ac35 100644 --- a/src/sysbench.c +++ b/src/sysbench.c @@ -142,11 +142,11 @@ static sb_barrier_t report_barrier; /* structures to handle queue of events, needed for tx_rate mode */ static pthread_mutex_t queue_mutex; static pthread_cond_t queue_cond; -static uint64_t queue_array[MAX_QUEUE_LEN] CK_CC_CACHELINE; -static ck_ring_buffer_t queue_ring_buffer[MAX_QUEUE_LEN] CK_CC_CACHELINE; -static ck_ring_t queue_ring CK_CC_CACHELINE; +static CK_CC_CACHELINE uint64_t queue_array[MAX_QUEUE_LEN]; +static CK_CC_CACHELINE ck_ring_buffer_t queue_ring_buffer[MAX_QUEUE_LEN]; +static CK_CC_CACHELINE ck_ring_t queue_ring ; -static int report_thread_created CK_CC_CACHELINE; +static CK_CC_CACHELINE int report_thread_created; static int checkpoints_thread_created; static int eventgen_thread_created; @@ -157,11 +157,11 @@ static sb_timer_t *timers; static sb_timer_t *timers_copy; /* Global execution timer */ -sb_timer_t sb_exec_timer CK_CC_CACHELINE; +CK_CC_CACHELINE sb_timer_t sb_exec_timer; /* timers for intermediate/checkpoint reports */ -sb_timer_t sb_intermediate_timer CK_CC_CACHELINE; -sb_timer_t sb_checkpoint_timer CK_CC_CACHELINE; +CK_CC_CACHELINE sb_timer_t sb_intermediate_timer; +CK_CC_CACHELINE sb_timer_t sb_checkpoint_timer; TLS int sb_tls_thread_id; diff --git a/src/sysbench.h b/src/sysbench.h index 2fb2ad89..acc39536 100644 --- a/src/sysbench.h +++ b/src/sysbench.h @@ -179,16 +179,16 @@ typedef struct sb_test typedef struct { - int error CK_CC_CACHELINE; /* global error flag */ + CK_CC_CACHELINE int error ;/* global error flag */ int argc; /* command line arguments count */ char **argv; /* command line arguments */ unsigned int tx_rate; /* target transaction rate */ uint64_t max_events; /* maximum number of events to execute */ uint64_t max_time_ns; /* total execution time limit */ - pthread_mutex_t exec_mutex CK_CC_CACHELINE; /* execution mutex */ + CK_CC_CACHELINE pthread_mutex_t exec_mutex ; /* execution mutex */ const char *testname; /* test name or script path to execute */ const char *cmdname; /* command passed from command line */ - unsigned int threads CK_CC_CACHELINE; /* number of threads to use */ + CK_CC_CACHELINE unsigned int threads ; /* number of threads to use */ unsigned int threads_running; /* number of threads currently active */ unsigned int report_interval; /* intermediate reports interval */ unsigned int percentile; /* percentile rank for latency stats */ @@ -199,22 +199,22 @@ typedef struct unsigned char debug; /* debug flag */ unsigned int timeout; /* forced shutdown timeout */ unsigned char validate; /* validation flag */ - unsigned char verbosity CK_CC_CACHELINE; /* log verbosity */ - int concurrency CK_CC_CACHELINE; /* number of concurrent requests + CK_CC_CACHELINE unsigned char verbosity; /* log verbosity */ + CK_CC_CACHELINE int concurrency; /* number of concurrent requests when tx-rate is used */ - int force_shutdown CK_CC_CACHELINE; /* whether we must force test + CK_CC_CACHELINE int force_shutdown; /* whether we must force test shutdown */ int forced_shutdown_in_progress; int warmup_time; /* warmup time */ - uint64_t nevents CK_CC_CACHELINE; /* event counter */ + CK_CC_CACHELINE uint64_t nevents; /* event counter */ const char *luajit_cmd; /* LuaJIT command */ } sb_globals_t; -extern sb_globals_t sb_globals CK_CC_CACHELINE; -extern pthread_mutex_t event_queue_mutex CK_CC_CACHELINE; +extern CK_CC_CACHELINE sb_globals_t sb_globals; +extern CK_CC_CACHELINE pthread_mutex_t event_queue_mutex; /* Global execution timer */ -extern sb_timer_t sb_exec_timer CK_CC_CACHELINE; +extern CK_CC_CACHELINE sb_timer_t sb_exec_timer; /* timers for checkpoint reports */ extern sb_timer_t sb_intermediate_timer; From 4b20112e8e2d5bef13a23753531b63eeec0119d8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 17/45] Windows port - MSVC preprocessor fixes - Do not use #ifdef within macro - Use do{}while(0) instead of unportable GCC braces in xfree() macro --- src/db_driver.c | 12 ++++++------ src/drivers/pgsql/drv_pgsql.c | 2 +- src/sb_lua.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/db_driver.c b/src/db_driver.c index c495f8cb..f273c88f 100644 --- a/src/db_driver.c +++ b/src/db_driver.c @@ -70,17 +70,17 @@ static void db_reset_stats(void); static int db_free_results_int(db_conn_t *con); /* DB layer arguments */ +#ifdef USE_MYSQL +#define DEFAULT_DB_DRIVER "mysql" +#else +#define DEFAULT_DB_DRIVER NULL +#endif static sb_arg_t db_args[] = { SB_OPT("db-driver", "specifies database driver to use " "('help' to get list of available drivers)", -#ifdef USE_MYSQL - "mysql", -#else - NULL, -#endif - STRING), + DEFAULT_DB_DRIVER, STRING), SB_OPT("db-ps-mode", "prepared statements usage mode {auto, disable}", "auto", STRING), SB_OPT("db-debug", "print database-specific debug information", "off", BOOL), diff --git a/src/drivers/pgsql/drv_pgsql.c b/src/drivers/pgsql/drv_pgsql.c index 3f2b93ac..8ce9bff4 100644 --- a/src/drivers/pgsql/drv_pgsql.c +++ b/src/drivers/pgsql/drv_pgsql.c @@ -33,7 +33,7 @@ #include "db_driver.h" #include "sb_rand.h" -#define xfree(ptr) ({ if (ptr) free((void *)ptr); ptr = NULL; }) +#define xfree(ptr) do{ if (ptr) free((void *)ptr); ptr = NULL; }while(0) /* Maximum length of text representation of bind parameters */ #define MAX_PARAM_LENGTH 256UL diff --git a/src/sb_lua.c b/src/sb_lua.c index 2dc55f35..d5ad3980 100644 --- a/src/sb_lua.c +++ b/src/sb_lua.c @@ -61,7 +61,7 @@ #define REPORT_INTERMEDIATE_HOOK "report_intermediate" #define REPORT_CUMULATIVE_HOOK "report_cumulative" -#define xfree(ptr) ({ if ((ptr) != NULL) free((void *) ptr); ptr = NULL; }) +#define xfree(ptr) do{ if ((ptr) != NULL) free((void *) ptr); ptr = NULL; }while(0) /* Interpreter context */ From bb8dbd6e956755641945030cbd5988d93babf894 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 18/45] Windows port - aligned memory allocation Use _aligned_malloc() for aligned allocation, and _aligned_free() for deallocation. Add sb_free_memaligned() instead of free() where appropriate. --- src/db_driver.c | 4 ++-- src/sb_counter.c | 2 +- src/sb_util.c | 17 +++++++++++++---- src/sb_util.h | 3 +++ src/sysbench.c | 4 ++-- src/tests/fileio/sb_fileio.c | 9 ++------- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/db_driver.c b/src/db_driver.c index f273c88f..b3985b3e 100644 --- a/src/db_driver.c +++ b/src/db_driver.c @@ -799,8 +799,8 @@ void db_done(void) if (db_globals.debug) { - free(exec_timers); - free(fetch_timers); + sb_free_memaligned(exec_timers); + sb_free_memaligned(fetch_timers); exec_timers = fetch_timers = NULL; } diff --git a/src/sb_counter.c b/src/sb_counter.c index 333a0183..8fd5f010 100644 --- a/src/sb_counter.c +++ b/src/sb_counter.c @@ -45,7 +45,7 @@ void sb_counters_done(void) { if (sb_counters != NULL) { - free(sb_counters); + sb_free_memaligned(sb_counters); sb_counters = NULL; } } diff --git a/src/sb_util.c b/src/sb_util.c index 90a07f36..2dc87c2b 100644 --- a/src/sb_util.c +++ b/src/sb_util.c @@ -40,8 +40,9 @@ void *sb_memalign(size_t size, size_t alignment) { void *buf; - -#ifdef HAVE_POSIX_MEMALIGN +#if defined(_WIN32) + buf = _aligned_malloc(size, alignment); +#elif defined(HAVE_POSIX_MEMALIGN) int ret= posix_memalign(&buf, alignment, size); if (ret != 0) buf = NULL; @@ -50,13 +51,21 @@ void *sb_memalign(size_t size, size_t alignment) #elif defined(HAVE_VALLOC) /* Allocate on page boundary */ (void) alignment; /* unused */ - buffer = valloc(size); + buf = valloc(size); #else # error Cannot find an aligned allocation library function! #endif - return buf; } +/* Free memory allocated with sb_memalign() */ +void sb_free_memaligned(void* p) +{ +#if defined(WIN32) + _aligned_free(p); +#else + free(p); +#endif +} /* Get OS page size */ diff --git a/src/sb_util.h b/src/sb_util.h index 67bee767..b4c2a292 100644 --- a/src/sb_util.h +++ b/src/sb_util.h @@ -117,6 +117,9 @@ */ void *sb_memalign(size_t size, size_t alignment); +/* Free memory allocated with sb_memalign() */ +void sb_free_memaligned(void* p); + /* Get OS page size */ size_t sb_getpagesize(void); diff --git a/src/sysbench.c b/src/sysbench.c index 4714ac35..5c723f60 100644 --- a/src/sysbench.c +++ b/src/sysbench.c @@ -1598,8 +1598,8 @@ int main(int argc, char *argv[]) sb_thread_done(); - free(timers); - free(timers_copy); + sb_free_memaligned(timers); + sb_free_memaligned(timers_copy); free(sb_globals.argv); diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index 5a003769..c4229d0c 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -268,7 +268,6 @@ static int file_mmap_done(void); /* Portability wrappers */ static size_t sb_get_allocation_granularity(void); -static void sb_free_memaligned(void *buf); static FILE_DESCRIPTOR sb_open(const char *); static int sb_create(const char *); @@ -1730,7 +1729,7 @@ void check_seq_req(sb_file_request_t *prev_req, sb_file_request_t *r) } } - +#ifdef HAVE_MMAP /* Alignment requirement for mmap(). The same as page size, except on Windows (on Windows it has to be 64KB, even if pagesize is only 4 or 8KB) @@ -1739,11 +1738,7 @@ size_t sb_get_allocation_granularity(void) { return sb_getpagesize(); } - -static void sb_free_memaligned(void *buf) -{ - free(buf); -} +#endif static FILE_DESCRIPTOR sb_open(const char *name) { From e14e78b87eea28d0845a4a15691527a5de1c32f4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 19/45] Windows port - implement nanosleep()/usleep() Implement cancellable sleeps. "Cancellable" property is important here, otherwise sysbench is going to hang, when used with --report-interval --- src/sb_timer.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/sb_timer.h b/src/sb_timer.h index 62293226..e40d781d 100644 --- a/src/sb_timer.h +++ b/src/sb_timer.h @@ -23,6 +23,9 @@ # include "config.h" #endif +#ifdef _WIN32 +#include +#endif #ifdef TIME_WITH_SYS_TIME # include # include @@ -98,10 +101,20 @@ typedef struct static inline int sb_nanosleep(uint64_t ns) { +#ifdef _WIN32 + pthread_testcancel(); + Sleep((DWORD)(ns / NS_PER_MS)); + return 0; +#else struct timespec ts = { ns / NS_PER_SEC, ns % NS_PER_SEC }; return nanosleep(&ts, NULL); +#endif } +#ifdef _WIN32 +#define usleep(x) sb_nanosleep(1000ULL*(x)) +#endif + /* timer control functions */ /* Initialize timer */ From d902b87c7e139694cf1ffef0f1117dce9ada3e76 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 20/45] Windows port - implement sb_getpagesize() --- src/sb_util.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sb_util.c b/src/sb_util.c index 2dc87c2b..a02bfc21 100644 --- a/src/sb_util.c +++ b/src/sb_util.c @@ -71,7 +71,11 @@ void sb_free_memaligned(void* p) size_t sb_getpagesize(void) { -#ifdef _SC_PAGESIZE +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#elif defined _SC_PAGESIZE return sysconf(_SC_PAGESIZE); #else return getpagesize(); From 09d499bdaed25dc24f65e966b74c44cf34b5ea23 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 21/45] Windows port - fileio support for O_SYNC and O_DIRECT Internally, those are translated to FILE_FLAG_NO_BUFFERING and FILE_FLAG_WRITE_THROUGH --- src/tests/fileio/sb_fileio.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index c4229d0c..2f28dbf5 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -53,6 +53,16 @@ #include "sb_util.h" #include "sb_counter.h" + +#ifdef _WIN32 + +/* use portability wrapper that understands O_DIRECT and O_SYNC */ +#ifdef open +#undef open +#endif +#define open(a, b, c) sb_win_open(a, b) +#endif + /* Lengths of the checksum and the offset fields in a block */ #define FILE_CHECKSUM_LENGTH sizeof(int) #define FILE_OFFSET_LENGTH sizeof(long) From aff84358de2b224627bd5674723fa8e8092e7ec6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 22/45] Windows port - mysql driver to use named pipe on Windows, if --socket is given. --- src/drivers/mysql/drv_mysql.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index e048dbaa..f5ee6db0 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -447,6 +447,16 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con) ) == NULL; } +/* + Hostname to pass to client library, if --socket parameter is given + On Windows, --socket is interpreted as named pipe name and host must + be "." Elsewhere, it is unix domain socket, and host is "localhost" +*/ +#ifdef _WIN32 +#define LOCAL_SOCKET_MYSQL_HOST "." +#else +#define LOCAL_SOCKET_MYSQL_HOST "localhost" +#endif /* Connect to MySQL database */ @@ -497,7 +507,7 @@ int mysql_drv_connect(db_conn_t *sb_conn) } else { - db_mysql_con->host = "localhost"; + db_mysql_con->host = LOCAL_SOCKET_MYSQL_HOST; /* The sockets list may be empty. So unlike hosts/ports the loop invariant From 7a57f1c377106f2b27100c1974ed6b1b9cde0d77 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 23/45] Windows port - luajit search directories Emulate search path logic on Windows from luajit code Search paths always relative to current executable, which is great for ZIP type distribution. Do not use Unixy path with "/usr/" etc on Windows. --- src/sb_lua.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/sb_lua.c b/src/sb_lua.c index d5ad3980..a5560bc7 100644 --- a/src/sb_lua.c +++ b/src/sb_lua.c @@ -514,6 +514,40 @@ static void sb_lua_var_string(lua_State *L, const char *name, const char *s) lua_settable(L, -3); } +#ifdef _WIN32 +/* + Path relative to current executable, in the same way luajit is doing it. +*/ +static void sb_setprogdir_path(lua_State * L) +{ + char path[MAX_PATH + 1]; + char* lb; + DWORD nsize = sizeof(path); + DWORD n = GetModuleFileNameA(NULL, path, nsize); + if (n == 0 || n == nsize || (lb = strrchr(path, '\\')) == NULL) { + luaL_error(L, "unable to get ModuleFileName"); + } + else + { + *lb = '\0'; + lua_pushstring(L, path); + lua_pushliteral(L, "\\?.lua;"); + lua_pushstring(L, path); + lua_pushliteral(L, "\\lua\\?.lua;"); + /* Resolve path */ + lb = strrchr(path, '\\'); + if (lb) + { + *lb = '\0'; + lua_pushstring(L, path); + lua_pushliteral(L, "\\share\\sysbench\\?.lua;"); + } + } +} +#else +#define sb_setprogdir_path(L) (void)0 +#endif + /* Set package.path and package.cpath in a given environment. Also honor LUA_PATH/LUA_CPATH to mimic the default Lua behavior. @@ -526,6 +560,7 @@ static void sb_lua_set_paths(lua_State *L) lua_pushliteral(L, "./?.lua;"); lua_pushliteral(L, "./?/init.lua;"); + sb_setprogdir_path(L); lua_pushliteral(L, "./src/lua/?.lua;"); const char *home = getenv("HOME"); @@ -541,8 +576,10 @@ static void sb_lua_set_paths(lua_State *L) lua_pushliteral(L, "/.luarocks/share/lua/?/init.lua;"); } +#ifndef _WIN32 lua_pushliteral(L, "/usr/local/share/lua/5.1/?.lua;"); lua_pushliteral(L, "/usr/share/lua/5.1/?.lua;"); +#endif lua_pushliteral(L, DATADIR "/?.lua;"); lua_concat(L, lua_gettop(L) - top); @@ -567,9 +604,11 @@ static void sb_lua_set_paths(lua_State *L) lua_pushliteral(L, "/.luarocks/lib/lua/?" DLEXT ";"); } +#ifndef _WIN32 lua_pushliteral(L, "/usr/local/lib/lua/5.1/?" DLEXT ";"); lua_pushliteral(L, "/usr/lib/lua/5.1/?" DLEXT ";"); - lua_pushliteral(L, LIBDIR ";"); + lua_pushliteral(L, LIBDIR "/?" DLEXT ";"); +#endif lua_concat(L, lua_gettop(L) - top); From fe0c6525d1be6878724aa6b9995863707066f101 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 24/45] Windows port - export symbols from sysbench executable The list of symbols is extracted from ffi.cdef sections in internal lua files into "module definition file" sysbench.def, which is used by the linker to export symbols. Alternative to that could be WINDOWS_EXPORT_ALL_SYMBOLS in recent cmake, but that does not work with LTO/PGO, unfortunately. --- src/lua/internal/CMakeLists.txt | 16 ++++++++++++++++ src/win/extract_c_functions.ps1 | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 src/win/extract_c_functions.ps1 diff --git a/src/lua/internal/CMakeLists.txt b/src/lua/internal/CMakeLists.txt index c17a1504..0a009aee 100644 --- a/src/lua/internal/CMakeLists.txt +++ b/src/lua/internal/CMakeLists.txt @@ -26,3 +26,19 @@ foreach(f ${LUA_INTERNAL_FILES}) string(REPLACE "." "_" name ${f}) create_lua_h(${f} ${name} ${CMAKE_CURRENT_BINARY_DIR}/${f}.h) endforeach() + +if(WIN32) + # prepare list of functions to be exported from sysbench.exe + set(sysbench_def ${CMAKE_CURRENT_BINARY_DIR}/sysbench.def) + execute_process( + COMMAND + powershell.exe -noprofile -ExecutionPolicy Bypass + -File ${PROJECT_SOURCE_DIR}/src/win/extract_c_functions.ps1 + ${LUA_INTERNAL_FILES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_FILE ${sysbench_def}.tmp + COMMAND_ERROR_IS_FATAL ANY) + configure_file(${sysbench_def}.tmp ${sysbench_def} COPYONLY) + file(REMOVE ${sysbench_def}.tmp) + target_sources(sysbench PRIVATE ${sysbench_def}) +endif() diff --git a/src/win/extract_c_functions.ps1 b/src/win/extract_c_functions.ps1 new file mode 100644 index 00000000..c858a437 --- /dev/null +++ b/src/win/extract_c_functions.ps1 @@ -0,0 +1,26 @@ +# Extracts C function names from luajit's ffi.cdef section. +# output function names in "module definition file" style, +# i.e def file that can be passed to linker to export the symbols + +# This script accepts lua file path's as (unnamed) parameters. + +$functionNames=@() +$ffiSnippetPattern= "(?sm)ffi\.cdef\[\[(.*?)\]\]" +$funcDeclarationPattern = '(?ms)(?<=\n|^)\s*(?!typedef)(?:\w+\s+\**)+(\w+)\s*\([^)]*\)\s*;' + +foreach($filePath in $args) { + $fileContent = Get-Content $filePath -Raw + # Find all ffi.cdef snippets + $matches = [regex]::Matches($fileContent, $ffiSnippetPattern) + foreach ($match in $matches) { + $cSourceCode = $match.Groups[1].Value + # Extract function names from the C source code + [regex]::Matches($cSourceCode, $funcDeclarationPattern) | ForEach-Object { + $functionNames += $_.Groups[1].Value.Trim() + } + } +} +# Write module definition to stdout +"EXPORTS" +$functionNames | Sort -Unique + From 4127b30b89afd520f385752ce013926eab6e8af2 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 25/45] Windows port - shared library extension is ".dll" --- src/sb_util.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sb_util.h b/src/sb_util.h index b4c2a292..ddc7c5b6 100644 --- a/src/sb_util.h +++ b/src/sb_util.h @@ -46,7 +46,9 @@ # define SB_ATTRIBUTE_UNUSED #endif -#if defined(__MACH__) +#ifdef _WIN32 +#define DLEXT ".dll" +#elif defined(__MACH__) # define DLEXT ".dylib" #else # define DLEXT ".so" From 9909d75d69e9cecb4e979e3a5381a5947c83666e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 26/45] Windows port - use 'size_t' instead of 'long' to fill io buffer long is 4 bytes , also on 64bit Windows, so that does not work as intended --- src/tests/fileio/sb_fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index 2f28dbf5..a804bb0e 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -1805,7 +1805,7 @@ void file_fill_buffer(unsigned char *buf, unsigned int len, *(int *)(void *)(buf + i) = (int)crc32(0, (unsigned char *)buf, len - (FILE_CHECKSUM_LENGTH + FILE_OFFSET_LENGTH)); /* Store the offset */ - *(long *)(void *)(buf + i + FILE_CHECKSUM_LENGTH) = offset; + *(size_t *)(void *)(buf + i + FILE_CHECKSUM_LENGTH) = offset; } From 925ae42a73fd238dae06aad269cb4a17459576cd Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 27/45] Windows port - fix warnings about function that is declared but not defined. HAVE_MMAP was not ported so far, and sb_get_allocation_granularity() is only defined with HAVE_MMAP Also, sb_next_event() does not exist. --- src/sysbench.h | 1 - src/tests/fileio/sb_fileio.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sysbench.h b/src/sysbench.h index acc39536..9f6882b6 100644 --- a/src/sysbench.h +++ b/src/sysbench.h @@ -223,7 +223,6 @@ extern sb_timer_t sb_checkpoint_timer; extern TLS int sb_tls_thread_id; bool sb_more_events(int thread_id); -sb_event_t sb_next_event(sb_test_t *test, int thread_id); void sb_event_start(int thread_id); void sb_event_stop(int thread_id); diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index a804bb0e..7b5f2dce 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -274,10 +274,10 @@ static int file_wait(int, long); #ifdef HAVE_MMAP static int file_mmap_prepare(void); static int file_mmap_done(void); +static size_t sb_get_allocation_granularity(void); #endif /* Portability wrappers */ -static size_t sb_get_allocation_granularity(void); static FILE_DESCRIPTOR sb_open(const char *); static int sb_create(const char *); From 026529a9cfdb34984c03d652f61d9db05a904829 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 28/45] Windows port - fix warnings about potential NULL pointer derefence --- src/db_driver.c | 1 + src/tests/fileio/sb_fileio.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/db_driver.c b/src/db_driver.c index b3985b3e..dd665526 100644 --- a/src/db_driver.c +++ b/src/db_driver.c @@ -572,6 +572,7 @@ db_row_t *db_fetch_row(db_result_t *rs) if (con->driver->ops.fetch_row == NULL) { log_text(LOG_ALERT, "fetching rows is not supported by the driver"); + return NULL; } if (rs->nrows == 0 || rs->nfields == 0) diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index 7b5f2dce..bad50c2c 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -1681,6 +1681,11 @@ int parse_arguments(void) } per_thread = malloc(sizeof(*per_thread) * sb_globals.threads); + if(per_thread == NULL) + { + log_text(LOG_FATAL, "Failed to allocate per-thread data"); + return 1; + } for (i = 0; i < sb_globals.threads; i++) { per_thread[i].buffer = sb_memalign(file_request_size, sb_getpagesize()); From 615d32bdcd1558b951695f0d1f9365b1689965a8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 29/45] Windows port - fix conversion/truncation warnings. --- src/db_driver.c | 6 +++--- src/drivers/mysql/drv_mysql.c | 10 +++++----- src/drivers/pgsql/drv_pgsql.c | 8 ++++---- src/sb_counter.c | 4 ++-- src/sb_histogram.c | 13 +++++++++++++ src/sb_lua.c | 4 ++-- src/sb_options.c | 4 ++-- src/sb_rand.c | 2 +- src/sb_thread.c | 2 +- src/sb_timer.h | 2 +- src/sysbench.c | 10 +++++----- src/tests/fileio/sb_fileio.c | 20 ++++++++++---------- src/tests/memory/sb_memory.c | 7 +++---- 13 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/db_driver.c b/src/db_driver.c index dd665526..478e0e36 100644 --- a/src/db_driver.c +++ b/src/db_driver.c @@ -960,8 +960,8 @@ int db_bulk_insert_init(db_conn_t *con, const char *query, size_t query_len) con->bulk_commit_max = driver_caps.needs_commit ? ROWS_BEFORE_COMMIT : 0; con->bulk_commit_cnt = 0; strcpy(con->bulk_buffer, query); - con->bulk_ptr = query_len; - con->bulk_values = query_len; + con->bulk_ptr = (unsigned int)query_len; + con->bulk_values = (unsigned int)query_len; con->bulk_cnt = 0; return 0; @@ -1015,7 +1015,7 @@ int db_bulk_insert_next(db_conn_t *con, const char *query, size_t query_len) } else strcpy(con->bulk_buffer + con->bulk_ptr, query); - con->bulk_ptr += query_len + (con->bulk_cnt > 0); + con->bulk_ptr += (unsigned int)query_len + (con->bulk_cnt > 0); con->bulk_cnt++; diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index f5ee6db0..688b243a 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -604,7 +604,7 @@ int mysql_drv_prepare(db_stmt_t *stmt, const char *query, size_t len) stmt->ptr = (void *)mystmt; DEBUG("mysql_stmt_prepare(%p, \"%s\", %u) = %p", mystmt, query, (unsigned int) len, stmt->ptr); - if (mysql_stmt_prepare(mystmt, query, len)) + if (mysql_stmt_prepare(mystmt, query, (unsigned long)len)) { /* Check if this statement in not supported */ rc = mysql_errno(con); @@ -723,7 +723,7 @@ int mysql_drv_bind_param(db_stmt_t *stmt, db_bind_t *params, size_t len) if (stmt->bound_param == NULL) return 1; memcpy(stmt->bound_param, params, len * sizeof(db_bind_t)); - stmt->bound_param_len = len; + stmt->bound_param_len = (unsigned int)len; return 0; @@ -1064,7 +1064,7 @@ db_error_t mysql_drv_query(db_conn_t *sb_conn, const char *query, size_t len, db_mysql_con = (db_mysql_conn_t *)sb_conn->ptr; con = db_mysql_con->mysql; - int err = mysql_real_query(con, query, len); + int err = mysql_real_query(con, query, (unsigned long)len); DEBUG("mysql_real_query(%p, \"%s\", %zd) = %d", con, query, len, err); if (SB_UNLIKELY(err != 0)) @@ -1097,7 +1097,7 @@ db_error_t mysql_drv_query(db_conn_t *sb_conn, const char *query, size_t len, rs->counter = SB_CNT_READ; rs->ptr = (void *)res; - rs->nrows = mysql_num_rows(res); + rs->nrows = (uint32_t)mysql_num_rows(res); DEBUG("mysql_num_rows(%p) = %u", res, (unsigned int) rs->nrows); rs->nfields = mysql_num_fields(res); @@ -1223,7 +1223,7 @@ db_error_t mysql_drv_next_result(db_conn_t *sb_conn, db_result_t *rs) rs->counter = SB_CNT_READ; rs->ptr = (void *)res; - rs->nrows = mysql_num_rows(res); + rs->nrows = (uint32_t)mysql_num_rows(res); DEBUG("mysql_num_rows(%p) = %u", res, (unsigned int) rs->nrows); rs->nfields = mysql_num_fields(res); diff --git a/src/drivers/pgsql/drv_pgsql.c b/src/drivers/pgsql/drv_pgsql.c index 8ce9bff4..9872039b 100644 --- a/src/drivers/pgsql/drv_pgsql.c +++ b/src/drivers/pgsql/drv_pgsql.c @@ -432,7 +432,7 @@ int pgsql_drv_bind_param(db_stmt_t *stmt, db_bind_t *params, size_t len) if (stmt->bound_param == NULL) return 1; memcpy(stmt->bound_param, params, len * sizeof(db_bind_t)); - stmt->bound_param_len = len; + stmt->bound_param_len = (unsigned int)len; if (stmt->emulated) return 0; @@ -756,12 +756,12 @@ int pgsql_drv_fetch_row(db_result_t *rs, db_row_t *row) PQgetvalue() returns an empty string, not a NULL value for a NULL field. Callers of this function expect a NULL pointer in this case. */ - if (PQgetisnull(rs->ptr, rownum, i)) + if (PQgetisnull(rs->ptr, (int)rownum, i)) row->values[i].ptr = NULL; else { - row->values[i].len = PQgetlength(rs->ptr, rownum, i); - row->values[i].ptr = PQgetvalue(rs->ptr, rownum, i); + row->values[i].len = PQgetlength(rs->ptr, (int)rownum, i); + row->values[i].ptr = PQgetvalue(rs->ptr, (int)rownum, i); } } diff --git a/src/sb_counter.c b/src/sb_counter.c index 8fd5f010..9a59a786 100644 --- a/src/sb_counter.c +++ b/src/sb_counter.c @@ -52,8 +52,8 @@ void sb_counters_done(void) static void sb_counters_merge(sb_counters_t dst) { - for (size_t t = 0; t < SB_CNT_MAX; t++) - for (size_t i = 0; i < sb_globals.threads; i++) + for (unsigned int t = 0; t < SB_CNT_MAX; t++) + for (unsigned int i = 0; i < sb_globals.threads; i++) dst[t] += sb_counter_val(i, t); } diff --git a/src/sb_histogram.c b/src/sb_histogram.c index e18185e3..7253d179 100644 --- a/src/sb_histogram.c +++ b/src/sb_histogram.c @@ -40,6 +40,14 @@ #include "sb_util.h" +#ifdef _MSC_VER +/* + Allow floor() to be converted to an integer without an explicit cast + (the case causes GCC bad-cast warning) +*/ +#pragma warning(push) +#pragma warning(disable : 4244) +#endif /* Number of slots for current histogram array. TODO: replace this constant with @@ -368,3 +376,8 @@ void sb_histogram_delete(sb_histogram_t *h) sb_histogram_done(h); free(h); } + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + diff --git a/src/sb_lua.c b/src/sb_lua.c index a5560bc7..60a55007 100644 --- a/src/sb_lua.c +++ b/src/sb_lua.c @@ -230,7 +230,7 @@ static int do_export_options(lua_State *L, bool global) lua_pushnumber(L, sb_opt_to_double(opt)); break; case SB_ARG_TYPE_SIZE: - lua_pushnumber(L, sb_opt_to_size(opt)); + lua_pushinteger(L, sb_opt_to_size(opt)); break; case SB_ARG_TYPE_STRING: tmp = sb_opt_to_string(opt); @@ -1040,7 +1040,7 @@ int sb_lua_call_custom_command(const char *name) return call_custom_command(gstate); } -#define stat_to_number(name) sb_lua_var_number(L, #name, stat->name) +#define stat_to_number(name) sb_lua_var_number(L, #name, (lua_Number)stat->name) static void stat_to_lua_table(lua_State *L, sb_stat_t *stat) { diff --git a/src/sb_options.c b/src/sb_options.c index 855c3323..b7bf496f 100644 --- a/src/sb_options.c +++ b/src/sb_options.c @@ -196,9 +196,9 @@ void sb_print_options(sb_arg_t *opts) /* Count the maximum name length */ for (i = 0, maxlen = 0; opts[i].name != NULL; i++) { - len = strlen(opts[i].name); + len = (unsigned int)strlen(opts[i].name); len += (opts[i].type < SB_ARG_TYPE_MAX) ? - strlen(opt_formats[opts[i].type]) : 8 /* =UNKNOWN */; + (unsigned int)strlen(opt_formats[opts[i].type]) : 8 /* =UNKNOWN */; if (len > maxlen) maxlen = len; } diff --git a/src/sb_rand.c b/src/sb_rand.c index 4a7cabc9..fc65b376 100644 --- a/src/sb_rand.c +++ b/src/sb_rand.c @@ -236,7 +236,7 @@ uint32_t sb_rand_default(uint32_t a, uint32_t b) uint32_t sb_rand_uniform(uint32_t a, uint32_t b) { - return a + sb_rand_uniform_double() * (b - a + 1); + return (uint32_t)(a + sb_rand_uniform_double() * (b - a + 1)); } /* gaussian distribution */ diff --git a/src/sb_thread.c b/src/sb_thread.c index 220c3794..36b6a07b 100644 --- a/src/sb_thread.c +++ b/src/sb_thread.c @@ -49,7 +49,7 @@ static int thread_stack_size; int sb_thread_init(void) { - thread_stack_size = sb_get_value_size("thread-stack-size"); + thread_stack_size = (int)sb_get_value_size("thread-stack-size"); if (thread_stack_size <= 0) { log_text(LOG_FATAL, "Invalid value for thread-stack-size: %d.\n", thread_stack_size); diff --git a/src/sb_timer.h b/src/sb_timer.h index e40d781d..eec77d8f 100644 --- a/src/sb_timer.h +++ b/src/sb_timer.h @@ -50,7 +50,7 @@ /* Convert nanoseconds to seconds and vice versa */ #define NS2SEC(nsec) ((nsec) / (double) NS_PER_SEC) -#define SEC2NS(sec) ((uint64_t) (sec) * NS_PER_SEC) +#define SEC2NS(sec) (((uint64_t) (sec)) * NS_PER_SEC) /* Convert nanoseconds to milliseconds and vice versa */ #define NS2MS(nsec) ((nsec) / (double) NS_PER_MS) diff --git a/src/sysbench.c b/src/sysbench.c index 5c723f60..b9af4a92 100644 --- a/src/sysbench.c +++ b/src/sysbench.c @@ -672,7 +672,7 @@ void print_run_mode(sb_test_t *test) { log_text(LOG_NOTICE, "Initializing random number generator from current time\n"); - srandom(time(NULL)); + srandom((unsigned int)time(NULL)); } if (sb_globals.force_shutdown) @@ -849,9 +849,9 @@ static void *worker_thread(void *arg) /* Generate exponentially distributed number with a given Lambda */ -static inline double sb_rand_exp(double lambda) +static inline uint64_t sb_rand_exp(double lambda) { - return -lambda * log(1 - sb_rand_uniform_double()); + return (uint64_t)(-lambda * log(1 - sb_rand_uniform_double())); } static void *eventgen_thread_proc(void *arg) @@ -1172,7 +1172,7 @@ static int run_test(sb_test_t *test) /* Set the alarm to force shutdown */ signal(SIGALRM, sigalrm_forced_shutdown_handler); - alarm(NS2SEC(sb_globals.max_time_ns) + sb_globals.timeout); + alarm((unsigned int)NS2SEC(sb_globals.max_time_ns) + sb_globals.timeout); } #endif @@ -1340,7 +1340,7 @@ static int init(void) if (tmp == NULL) { sb_globals.force_shutdown = 1; - sb_globals.timeout = NS2SEC(sb_globals.max_time_ns) / 20; + sb_globals.timeout = (unsigned int)NS2SEC(sb_globals.max_time_ns) / 20; } else if (strcasecmp(tmp, "off")) { diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index bad50c2c..40d315c5 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -356,8 +356,8 @@ int file_prepare(void) log_text(LOG_FATAL, "Size of file '%s' is %sB, but at least %sB is expected.", file_name, - sb_print_value_size(ss1, sizeof(ss1), buf.st_size), - sb_print_value_size(ss2, sizeof(ss2), file_size)); + sb_print_value_size(ss1, sizeof(ss1), (double)buf.st_size), + sb_print_value_size(ss2, sizeof(ss2), (double)file_size)); log_text(LOG_WARNING, "Did you run 'prepare' with different --file-total-size or " "--file-num values?"); @@ -590,7 +590,7 @@ int file_execute_event(sb_event_t *sb_req, int thread_id) log_text(LOG_FATAL, "Incorrect file id in request: %u", file_req->file_id); return 1; } - if (file_req->pos + file_req->size > file_size) + if (file_req->pos + (intptr_t)file_req->size > file_size) { log_text(LOG_FATAL, "I/O request exceeds file size. " "file id: %d file size: %lld req offset: %lld req size: %lld", @@ -608,7 +608,7 @@ int file_execute_event(sb_event_t *sb_req, int thread_id) /* Store checksum and offset in a buffer when in validation mode */ if (sb_globals.validate) - file_fill_buffer(per_thread[thread_id].buffer, file_req->size, file_req->pos); + file_fill_buffer(per_thread[thread_id].buffer, (unsigned int)file_req->size, file_req->pos); if(file_pwrite(file_req->file_id, per_thread[thread_id].buffer, file_req->size, file_req->pos, thread_id) @@ -643,7 +643,7 @@ int file_execute_event(sb_event_t *sb_req, int thread_id) /* Validate block if run with validation enabled */ if (sb_globals.validate && - file_validate_buffer(per_thread[thread_id].buffer, file_req->size, file_req->pos)) + file_validate_buffer(per_thread[thread_id].buffer, (unsigned int)file_req->size, file_req->pos)) { log_text(LOG_FATAL, "Validation failed on file " FD_FMT ", block offset %lld, exiting...", @@ -692,16 +692,16 @@ void file_print_mode(void) print_file_extra_flags(); log_text(LOG_NOTICE, "%d files, %sB each", num_files, - sb_print_value_size(sizestr, sizeof(sizestr), file_size)); + sb_print_value_size(sizestr, sizeof(sizestr), (double)file_size)); log_text(LOG_NOTICE, "%sB total file size", sb_print_value_size(sizestr, sizeof(sizestr), - file_size * num_files)); + (double)(file_size * num_files))); log_text(LOG_NOTICE, "Block size %sB", - sb_print_value_size(sizestr, sizeof(sizestr), file_block_size)); + sb_print_value_size(sizestr, sizeof(sizestr), (double)file_block_size)); if (file_merged_requests > 0) log_text(LOG_NOTICE, "Merging requests up to %sB for sequential IO.", sb_print_value_size(sizestr, sizeof(sizestr), - file_request_size)); + (double)file_request_size)); switch (test_mode) { @@ -1606,7 +1606,7 @@ int parse_arguments(void) return 1; } - file_block_size = sb_get_value_size("file-block-size"); + file_block_size = (int)sb_get_value_size("file-block-size"); if (file_block_size <= 0) { log_text(LOG_FATAL, "Invalid value for file-block-size: %d.", diff --git a/src/tests/memory/sb_memory.c b/src/tests/memory/sb_memory.c index 6f829c29..be0753fe 100644 --- a/src/tests/memory/sb_memory.c +++ b/src/tests/memory/sb_memory.c @@ -285,8 +285,7 @@ int event_rnd_none(sb_event_t *req, int tid) for (ssize_t i = 0; i <= max_offset; i++) { - size_t offset = (volatile size_t) sb_rand_default(0, max_offset); - (void) offset; /* unused */ + (void)sb_rand_default(0, (uint32_t)max_offset); } return 0; @@ -299,7 +298,7 @@ int event_rnd_read(sb_event_t *req, int tid) for (ssize_t i = 0; i <= max_offset; i++) { - size_t offset = (size_t) sb_rand_default(0, max_offset); + size_t offset = (size_t) sb_rand_default(0, (uint32_t)max_offset); size_t val = SIZE_T_LOAD(buffers[tid] + offset); (void) val; /* unused */ } @@ -314,7 +313,7 @@ int event_rnd_write(sb_event_t *req, int tid) for (ssize_t i = 0; i <= max_offset; i++) { - size_t offset = (size_t) sb_rand_default(0, max_offset); + size_t offset = (size_t) sb_rand_default(0, (uint32_t)max_offset); SIZE_T_STORE(buffers[tid] + offset, i); } From 289a611f63e4ce62d3761a58923b3f66c7b4d882 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 30/45] Windows port - prevent aggressive compiler optimizations in cpu_execute_event() MSVC compiler understands that the function has no side-effects, and optimizes the function away, completely. Also, clang-16 complains about set-but-unused variable. To fix both, assign calculated prime count in cpu_execute_event() to a thread local variable, at the end of function This way, cpu_execute_event() function has _some_ side-effect and will not be optimized away. --- src/tests/cpu/sb_cpu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tests/cpu/sb_cpu.c b/src/tests/cpu/sb_cpu.c index d95f26c3..d9e5bde7 100644 --- a/src/tests/cpu/sb_cpu.c +++ b/src/tests/cpu/sb_cpu.c @@ -94,6 +94,9 @@ sb_event_t cpu_next_event(int thread_id) return req; } +/* helps to avoid compilers optimizing cpu_execute_event() away */ +static TLS unsigned long long thread_count_primes; + int cpu_execute_event(sb_event_t *r, int thread_id) { unsigned long long c; @@ -115,7 +118,7 @@ int cpu_execute_event(sb_event_t *r, int thread_id) if (l > t ) n++; } - + thread_count_primes= n; return 0; } From d1a1540c0bd4f7948593afb9abf1462fb5bb640c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 31/45] Windows port - Windows build/packaging fixes - Copy luajit and libmariadb DLLs next to sysbench executable, during build. This allows to test without packaging - Do not install Cram based test suite on Windows, it does not work - Install DLLs that sysbench depends on next to executable (luajit DLL, libmariadb, postgres client, and their dependencies such as openssl or zlib. Also install VC runtime DLLs, and ASAN runtime DLL, when built WITH_ASAN=1 - Default to ZIP for CPack. --- CMakeLists.txt | 7 ++- src/CMakeLists.txt | 118 +++++++++++++++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7a6699d..28c565a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,9 @@ -cmake_minimum_required(VERSION 3.12..3.26) +if(WIN32) + # new-ish cmake features, install(runtime_dependencies etc) + cmake_minimum_required(VERSION 3.21..3.26) +else() + cmake_minimum_required(VERSION 3.12..3.26) +endif() set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) if(APPLE) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9f6e4080..915eaca5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -153,6 +153,29 @@ endif() add_dependencies(sysbench ConcurrencyKit) target_link_libraries(sysbench PRIVATE ConcurrencyKit) +if(WIN32) + # Add location of sb_win_posix.h include file + target_include_directories(sysbench SYSTEM BEFORE + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/win) + + # Copy dlls next to exe, so that sysbench can run after compilation without + # setting PATH + add_custom_command( + TARGET sysbench + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + COMMENT "Copy luajit dll") + if(TARGET mariadbclient) + add_custom_command( + TARGET sysbench + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ + $ + COMMENT "Copy libmariadb dll") + endif() +endif() + # Find git hash, if built from git include(githash) githash(SB_GIT_SHA) @@ -165,42 +188,71 @@ configure_file(config.h.in config.h @ONLY) # Installation related stuff add_subdirectory(lua) +if(NOT WIN32) + # Install tests + if(WITH_MYSQL) + set(USE_MYSQL 1) + endif() + if(WITH_PGSQL) + set(USE_MYSQL 1) + endif() -# Install tests -if(WITH_MYSQL) - set(USE_MYSQL 1) -endif() -if(WITH_PGSQL) - set(USE_MYSQL 1) -endif() + configure_file(${PROJECT_SOURCE_DIR}/tests/include/config.sh.in + ${PROJECT_BINARY_DIR}/tests/include/config.sh @ONLY) + install( + FILES ${PROJECT_BINARY_DIR}/tests/include/config.sh + DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME}/tests/include) -configure_file(${PROJECT_SOURCE_DIR}/tests/include/config.sh.in - ${PROJECT_BINARY_DIR}/tests/include/config.sh @ONLY) -install( - FILES ${PROJECT_BINARY_DIR}/tests/include/config.sh - DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME}/tests/include) + install( + DIRECTORY ${PROJECT_SOURCE_DIR}/tests + DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME} + USE_SOURCE_PERMISSIONS FILES_MATCHING + PATTERN "*.t" + PATTERN "*.sh" + PATTERN "*.md" + PATTERN "*.lua") -install( - DIRECTORY ${PROJECT_SOURCE_DIR}/tests - DESTINATION ${CMAKE_INSTALL_DATADIR}/${CMAKE_PROJECT_NAME} - USE_SOURCE_PERMISSIONS FILES_MATCHING - PATTERN "*.t" - PATTERN "*.sh" - PATTERN "*.md" - PATTERN "*.lua") - -if(${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR}) - # make test is broken on out-of-source - add_custom_target( - test - COMMAND sh -c ./test_run.sh - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests) + if(${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR}) + # make test is broken on out-of-source + add_custom_target( + test + COMMAND sh -c ./test_run.sh + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests) + endif() endif() -install( - TARGETS sysbench - EXPORT sysbenchTargets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +if(WIN32) + if(TARGET MySQL::MySQL) + set(mysqlclientlib MySQL::MySQL) + elseif(TARGET mariadbclient) + set(mysqlclientlib mariadbclient) + endif() + cmake_path(GET CMAKE_C_COMPILER PARENT_PATH compiler_dir) + # Install .dll dependencies next to the executable + install( + TARGETS sysbench + EXPORT sysbenchTargets + RUNTIME_DEPENDENCIES + PRE_EXCLUDE_REGEXES + "api-ms-" + "ext-ms-" # magic to exclude MS .dlls + POST_EXCLUDE_REGEXES + ".*system32/.*\\.dll" + DIRECTORIES + $ + $ + $<$:$> + ${PostgreSQL_LIBRARY_DIRS} + $<$:${PostgreSQL_LIBRARY_DIR}/../bin> + $<$:$/../bin> + ${compiler_dir} + ) +else() + install( + TARGETS sysbench + EXPORT sysbenchTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() install( EXPORT sysbenchTargets @@ -221,4 +273,8 @@ write_basic_package_version_file( install(FILES ${CMAKE_CURRENT_BINARY_DIR}/sysbenchConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/sysbenchConfigVersion.cmake DESTINATION share/sysbench/cmake) +if(WIN32) + # A more reasonable choice of default package on Windows than NSIS + set(CPACK_GENERATOR ZIP) +endif() include(CPack) From eef167b6a34675f6f91f3aeafb9f72c8364facea Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 32/45] Windows port - instruct C runtime to use its maximum open file handles. LuaJit opens source files using C runtime functions, and in sysbench it does so simultaneously in different threads. To avoid "Too many open files" error, use _setmaxstdio() with maximum possible value for open handles, which is 8K --- src/sysbench.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sysbench.c b/src/sysbench.c index b9af4a92..58ef0617 100644 --- a/src/sysbench.c +++ b/src/sysbench.c @@ -1435,11 +1435,25 @@ static int init(void) } +/* OS specific initialization */ +static inline void os_init(void) +{ +#ifdef _WIN32 + /* + lua is using C runtime, and is opens multiple files simultaneously + in different threads. Allow maximum possible number of open files. + */ + _setmaxstdio(8192); +#endif +} + int main(int argc, char *argv[]) { sb_test_t *test = NULL; int rc; + os_init(); + sb_globals.argc = argc; sb_globals.argv = malloc(argc * sizeof(char *)); memcpy(sb_globals.argv, argv, argc * sizeof(char *)); From fd4a841c8482b032de31e7cfb71148df92814858 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 33/45] Windows port - use timeBeginPeriod() for better Sleep() accuracy. Without this call, Sleep(1) can take as much as 14-15ms --- src/CMakeLists.txt | 3 +++ src/sysbench.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 915eaca5..b7906403 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -154,6 +154,9 @@ add_dependencies(sysbench ConcurrencyKit) target_link_libraries(sysbench PRIVATE ConcurrencyKit) if(WIN32) + # Need winmm for timeBeginPeriod() + target_link_libraries(sysbench PRIVATE winmm) + # Add location of sb_win_posix.h include file target_include_directories(sysbench SYSTEM BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/win) diff --git a/src/sysbench.c b/src/sysbench.c index 58ef0617..e51dc6ff 100644 --- a/src/sysbench.c +++ b/src/sysbench.c @@ -1444,6 +1444,8 @@ static inline void os_init(void) in different threads. Allow maximum possible number of open files. */ _setmaxstdio(8192); + /* improve timer resolution for Sleep() */ + timeBeginPeriod(1); #endif } From 57ac23e3d4fb017275c135950ea18dd64883e4a8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 34/45] Windows port - adjust .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 37f89c13..39727d54 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,8 @@ prime stage *.snap /snap/snapcraft.yaml +/build* +.vscode +*.bak +src/libmariadb-src +*.pdb From 9426021a738d166f8b74e9f8b1491282ec465c78 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 35/45] Windows port - build and test on CI (github workflows) The build utilizes vcpkg package manager in order to build with Postgres driver. A ZIP sysbench package is built and stored as build artifact, so if desired, someone can use extract it, and produce official binary release. The tests are smoke OLTP tests that run against installed MariaDB and Postgres. As Cram unit test does not work on Windows, so this is the only test option --- .github/workflows/windows.yml | 78 +++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 00000000..21cff69f --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,78 @@ +name: Windows +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + os: [windows-latest] + runs-on: ${{ matrix.os }} + name: Build on ${{ matrix.os }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + id: vcpkg_cache + with: + path: ~\AppData\Local\vcpkg + key: ${{ runner.os }}-vcpkg-libpq-zlib + - name: vcpkg install + run: vcpkg install libpq zlib --triplet=x64-windows-release + - name: configure + run: cmake -B build_dir -DCMAKE_COMPILE_WARNING_AS_ERROR=1 -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-release -DWITH_PGSQL=1 + - name: build + run: cmake --build build_dir --config RelWithDebInfo --target package -j 4 + - name: upload package + uses: actions/upload-artifact@v3 + with: + name: package + path: build_dir/*.zip + - name: test_install + run: | + cmake --install build_dir --prefix install_dir --config RelWithDebInfo + echo "$pwd\install_dir\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + cat $env:GITHUB_PATH + - name: sysbench version + run: sysbench --version + - name: sysbench help + run: sysbench --help + - name: actions-setup-mysql + uses: shogo82148/actions-setup-mysql@v1.17.0 + env: + TMPDIR: ${{ runner.temp }} + TMP: ${{ runner.temp }} + TEMP: ${{ runner.temp }} + with: + mysql-version: 'mariadb-10.9' + my-cnf: | + innodb_log_file_size=1G + innodb_buffer_pool_size=512MB + max_allowed_packet=16MB + max_connections=500 + named-pipe=1 + socket=MySQL + - name: actions-setup-pgsql + uses: ikalnytskyi/action-setup-postgres@v4 + with: + username: sbtest + password: sbtest + database: sbtest + - name: create database sbtest + run: mysql -uroot -e "create database sbtest" + - name: sysbench oltp_update_index prepare + run: sysbench oltp_update_index --mysql-user=root --table-size=100000 prepare + - name: sysbench oltp_point_select run + run: sysbench oltp_point_select --mysql-user=root --mysql-host=. --time=30 --table-size=100000 --threads=1 --report-interval=1 --histogram run + - name: sysbench oltp_update_index run ssl + run: sysbench oltp_update_index --mysql-user=root --time=30 --mysql-ssl=on --table-size=100000 --threads=1 --report-interval=1 --histogram run + - name: sysbench oltp_update_index cleanup + run: sysbench oltp_update_index --mysql-user=root cleanup + - name: sysbench pgsql oltp_update_index prepare + run: sysbench oltp_update_index --db-driver=pgsql --pgsql-password=sbtest --table-size=100000 prepare + - name: sysbench pgsql oltp_point_select run + run: sysbench oltp_point_select --db-driver=pgsql --pgsql-password=sbtest --time=30 --table-size=100000 --threads=1 --report-interval=1 --histogram run + - name: sysbench pgsql oltp_update_index run + run: sysbench oltp_update_index --db-driver=pgsql --pgsql-password=sbtest --time=30 --table-size=100000 --threads=1 --report-interval=1 --histogram run + - name: sysbench oltp_update_index cleanup + run: sysbench oltp_update_index --db-driver=pgsql --pgsql-password=sbtest cleanup From b6dfe9838a872cc250183de01a6e84be2c45a16d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 36/45] Fix "Unknown extra flags value 5" when --file-extra-flags=sync,direct Combining flags in file-extra-flags in fileio should be allowed O_SYNC|O_DIRECT, or O_DSYNC|O_DIRECT is actually reasonable combination. --- src/tests/fileio/sb_fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/fileio/sb_fileio.c b/src/tests/fileio/sb_fileio.c index 40d315c5..190f6d29 100644 --- a/src/tests/fileio/sb_fileio.c +++ b/src/tests/fileio/sb_fileio.c @@ -885,7 +885,7 @@ static int convert_extra_flags(file_flags_t extra_flags, int *open_flags) #endif } - if (extra_flags > SB_FILE_FLAG_DIRECTIO) + if (extra_flags > (SB_FILE_FLAG_SYNC|SB_FILE_FLAG_DSYNC|SB_FILE_FLAG_DIRECTIO)) { log_text(LOG_FATAL, "Unknown extra flags value: %d", (int) extra_flags); return 1; From 4cf93382d73ab4a21a2db114c2f7a9a1a7099383 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 37/45] Fix diff in a single test On CI, the error code of the message "Unknown MySQL server host 'non-existing' " is -3, not 0. --- tests/t/api_sql_mysql.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/t/api_sql_mysql.t b/tests/t/api_sql_mysql.t index 9bb7485e..cffa308b 100644 --- a/tests/t/api_sql_mysql.t +++ b/tests/t/api_sql_mysql.t @@ -72,7 +72,7 @@ SQL Lua API + MySQL tests -- reconnects = 1 FATAL: unable to connect to MySQL server on host 'non-existing', port 3306, aborting... - FATAL: error 2005: Unknown MySQL server host 'non-existing' (0) + FATAL: error 2005: Unknown MySQL server host 'non-existing' (*) (glob) connection creation failed -- FATAL: mysql_drv_query() returned error 1048 (Column 'a' cannot be null) for query 'INSERT INTO t VALUES (NULL)' From 66dc3c0f2ec658886da8f2ed0803197ea8436abb Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 11:01:30 +0200 Subject: [PATCH 38/45] Fix clang-16 warning about old(pre-ANSI C) syntax. --- src/tests/fileio/crc32.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/tests/fileio/crc32.c b/src/tests/fileio/crc32.c index 414f4f3a..5a83f373 100644 --- a/src/tests/fileio/crc32.c +++ b/src/tests/fileio/crc32.c @@ -191,10 +191,8 @@ local void write_table(out, table) /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - unsigned len; +unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf, + unsigned len) { if (buf == Z_NULL) return 0UL; @@ -234,10 +232,8 @@ unsigned long ZEXPORT crc32(crc, buf, len) #define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4 /* ========================================================================= */ -local unsigned long crc32_little(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - unsigned len; +local unsigned long crc32_little(unsigned long crc, const unsigned char FAR *buf, + unsigned len) { register u4 c; register const u4 FAR *buf4; @@ -274,10 +270,8 @@ local unsigned long crc32_little(crc, buf, len) #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 /* ========================================================================= */ -local unsigned long crc32_big(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - unsigned len; +local unsigned long crc32_big(unsigned long crc, const unsigned char FAR *buf, + unsigned len) { register u4 c; register const u4 FAR *buf4; From 7a7da4fe2105610a257d8649a01bd5235eb83758 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 09:49:43 +0200 Subject: [PATCH 39/45] CI : Unify cmake-based build, test, and packagin on Github Actions Remove repetive copy-and-paste workflows, create a reusable one. Now the build and test, and ASAN, and building with non-default compilers runs on all supported (by CI) OSes (Windows, Linux, and macOS, as much uniform as possible. "make test" is not very portable in sysbench It does not work on Windows, does not work with either MariaDB client or server, does not work with recent poostgres Therefore on all combinations or OS(Windows, Linux, macOS)/database(MariaDB, MySQL, Postgres) some smoke tests (oltp) are also run. "make test" will run, where it is known to have worked before (non-Windows, server and client library are both recent MySQL) ASAN builds run on Windows and macOS. Due to a bug in ASAN on Ubuntu's gcc, https://github.com/google/sanitizers/issues/1010 , we do not run it on ubuntu Ninja CMake generator is run on all OSes, and "alternative" compilers (clang on Linux, mingw64 gcc on Windows) are also tested. --- .github/workflows/cmake-single-os.yml | 284 ++++++++++++++++++++++++++ .github/workflows/cmake.yml | 14 ++ .github/workflows/macos.yml | 50 ----- .github/workflows/ubuntu.yml | 23 --- .github/workflows/windows.yml | 78 ------- 5 files changed, 298 insertions(+), 151 deletions(-) create mode 100644 .github/workflows/cmake-single-os.yml create mode 100644 .github/workflows/cmake.yml delete mode 100644 .github/workflows/macos.yml delete mode 100644 .github/workflows/ubuntu.yml delete mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/cmake-single-os.yml b/.github/workflows/cmake-single-os.yml new file mode 100644 index 00000000..575b3255 --- /dev/null +++ b/.github/workflows/cmake-single-os.yml @@ -0,0 +1,284 @@ +name: CMake-reusable-workflow + +on: + workflow_call: + inputs: + os: + required: true + type: string + +jobs: + build: + runs-on: ${{ inputs.os }} + name: Build + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + + # On Windows, use vcpkg to build, as pgsql driver on build box is bad (crashes during tests) + - name: vcpkg_cache(Windows) + uses: actions/cache@v3 + if: runner.os == 'Windows' + id: vcpkg_cache + with: + path: ~\AppData\Local\vcpkg + key: ${{ runner.os }}-vcpkg-libpq-zlib + + - name: install build preprequisites (macOS) + if: runner.os == 'macOS' + run : | + HOMEBREW_NO_AUTO_UPDATE=1 brew install mysql-client libpq ninja + brew link --force mysql-client + + - name: install build prerequisites (Windows) + if: runner.os == 'Windows' + run: | + vcpkg install libpq zlib --triplet=x64-windows-release + + - name: install build prerequisites (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get install ninja-build + + - name: configure + env: + CMAKE_VCPKG_PARAMS: ${{ runner.os == 'Windows' && '-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-release' || ' ' }} + run: cmake . -DWITH_PGSQL=1 -DCMAKE_COMPILE_WARNING_AS_ERROR=1 ${{ env.CMAKE_VCPKG_PARAMS }} + + - name: build package + run: cmake --build . --config Release --target package -j + + - name: setup mysql for unit tests + if: runner.os != 'Windows' + uses: shogo82148/actions-setup-mysql@v1 + with: + my-cnf: | + innodb_redo_log_capacity=4G + innodb_buffer_pool_size=4G + skip-log-bin + + - name: unit test + if: runner.os != 'Windows' + run: | + mysql -uroot -e "create database sbtest" + SBTEST_MYSQL_ARGS=" --mysql-user=root --mysql-host=127.0.0.1 " cmake --build . --config Release --target test + + - name: upload package(Windows) + if: runner.os == 'Windows' + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.os }}-package + path: ./*.zip + + - name: test_install + run: | + cmake --install . --config Release --prefix install_dir + + - name: Archive build + uses: actions/upload-artifact@v3 + with: + name: build-${{ inputs.os }} + retention-days: 1 + path: | + install_dir + + # Tests Ninja generator, non-standard compiler (clang on Linux and gcc on Windows and macOS) + build-ninja: + runs-on: ${{ inputs.os }} + name: Build-ninja + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: install ninja (macOS) + if: runner.os == 'macOS' + run : | + HOMEBREW_NO_AUTO_UPDATE=1 brew install ninja mysql-client libpq + brew link --force mysql-client + + - name: install ninja (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get install ninja-build + + - name: install ninja (Windows) + if: runner.os == 'Windows' + run: | + choco install ninja + + - name: configure and build + run: | + mkdir build + cmake -S . -B build -GNinja -DCMAKE_COMPILE_WARNING_AS_ERROR=1 -DCMAKE_BUILD_TYPE=Debug -DWITH_PGSQL=1 -DCMAKE_C_COMPILER=${{ runner.os == 'Linux' && 'clang' || 'gcc' }} + cmake --build build + + - uses: seanmiddleditch/gha-setup-vsdevenv@master + if: runner.os == 'Windows' + + - name: build with clang-cl + if: runner.os == 'Windows' + run: | + choco install llvm + mkdir clang_build + cmake -S . -B clang_build -GNinja -DCMAKE_C_COMPILER=clang-cl -DCMAKE_COMPILE_WARNING_AS_ERROR=1 -DCMAKE_BUILD_TYPE=Debug -DWITH_PGSQL=1 + cmake --build clang_build + + test-asan: + runs-on: ${{ inputs.os }} + # Linux is excluded, due to buggy ASAN ubuntu + if: ${{ !contains(inputs.os,'ubuntu') }} + name: test-asan + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: install client libs (macOS) + if: runner.os == 'macOS' + run : | + HOMEBREW_NO_AUTO_UPDATE=1 brew install mysql-client + brew link --force mysql-client + - name: compile + run: | + mkdir asan_build + cmake -S . -B asan_build -DCMAKE_BUILD_TYPE=Debug -DWITH_ASAN=1 + cmake --build asan_build --config Debug + cmake --install asan_build --config Debug --prefix install_dir + + - name: install mysql + uses: shogo82148/actions-setup-mysql@v1 + env: + TMPDIR: ${{ runner.temp }} + TMP: ${{ runner.temp }} + TEMP: ${{ runner.temp }} + with: + my-cnf: | + skip-log-bin + + - name: update env (Windows) + if: runner.os == 'Windows' + run: | + echo "$pwd\install_dir\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: update env (non-Windows) + if: runner.os != 'Windows' + run: | + echo "$PWD/install_dir/bin" >> $GITHUB_PATH + echo "LUA_PATH=$PWD/install_dir/share/sysbench/?.lua" >> $GITHUB_ENV + - name: smoke test + run: | + mysql -uroot --host=127.0.0.1 -e "create database sbtest" + sysbench --version + sysbench --help + sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 prepare + sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 run --threads=2 --histogram + sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 cleanup + sysbench cpu run --threads=2 + sysbench fileio --file-test-mode=seqrewr prepare + sysbench fileio --file-test-mode=seqrewr run + sysbench fileio --file-test-mode=seqrewr cleanup + sysbench mutex run --threads=2 + + test-oltp: + strategy: + fail-fast: false + matrix: + db: [mariadb, mysql, postgres] + needs: build + runs-on: ${{ inputs.os }} + #if: contains(inputs.os,'windows') + name: test-${{matrix.db}} + env: + COMMON_SYSBENCH_PARAMS: ${{ matrix.db == 'postgres' && '--db-driver=pgsql --pgsql-password=sbtest' || '--mysql-user=root --mysql-socket=/tmp/mysql.sock'}} --time=30 --table-size=1000000 --report-interval=1 --thread-init-timeout=60 --histogram + + steps: + - name: Download build + uses: actions/download-artifact@v3 + with: + name: build-${{ inputs.os }} + + - name: test fixups + shell: bash + if: runner.os != 'Windows' + run: | + ls -l bin + chmod +x bin/sysbench + + - name: install client drivers (macOS) + if: runner.os == 'macOS' + run : | + HOMEBREW_NO_AUTO_UPDATE=1 brew install mysql-client libpq + brew link --force mysql-client + + - name: update environment (Windows) + if: runner.os == 'Windows' + run: | + echo "$pwd\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + cat $env:GITHUB_PATH + + - name: update environment (non-Windows) + if: runner.os != 'Windows' + run: | + echo "$PWD/bin" >> $GITHUB_PATH + cat $GITHUB_PATH + echo "LUA_PATH=$PWD/share/sysbench/?.lua" >> $GITHUB_ENV + + - name: actions-setup-pgsql + if: matrix.db =='postgres' + uses: ikalnytskyi/action-setup-postgres@v4 + with: + username: sbtest + password: sbtest + database: sbtest + + - name: actions-setup-mysql + if: matrix.db != 'postgres' + uses: shogo82148/actions-setup-mysql@v1 + env: # set temp directory, so that datadir ends up on fast disk + TMPDIR: ${{ runner.temp }} + TMP: ${{ runner.temp }} + TEMP: ${{ runner.temp }} + with: + distribution: ${{ matrix.db }} + my-cnf: | + ${{ matrix.db == 'mysql' && 'innodb_redo_log_capacity' || 'innodb_log_file_size' }}=4G + innodb_buffer_pool_size=512MB + max_allowed_packet=16MB + skip-log-bin + loose-enable-named-pipe + socket=/tmp/mysql.sock + max_connections=1000 + innodb_max_dirty_pages_pct_lwm=10 + + - name: create database sbtest + if: matrix.db != 'postgres' + run: | + mysql -uroot --host=127.0.0.1 --port=3306 -e "create database sbtest" + + - name: oltp_read_write prepare + run: sysbench oltp_read_write ${{ env.COMMON_SYSBENCH_PARAMS }} prepare + - name: oltp_point_select run + run: sysbench oltp_point_select ${{ env.COMMON_SYSBENCH_PARAMS }} --threads=20 run + - name: oltp_read_write + run: sysbench oltp_read_write ${{ env.COMMON_SYSBENCH_PARAMS }} --threads=20 run + - name: oltp_read_only + run: sysbench oltp_read_only ${{ env.COMMON_SYSBENCH_PARAMS }} --threads=20 run + - name: oltp_update_index run (1 thread) + run: sysbench oltp_update_index ${{ env.COMMON_SYSBENCH_PARAMS }} --threads=1 run + - name: oltp_update_index run (20 threads) + run: sysbench oltp_update_index ${{ env.COMMON_SYSBENCH_PARAMS }} --threads=20 run + - name: oltp_read_write cleanup + run: sysbench oltp_update_index ${{ env.COMMON_SYSBENCH_PARAMS }} cleanup + + + # The below fileio test help estimate whether DBMS has "relaxed" durability, in other words cheats on + # + # oltp_update_index test with 1 user can't have qps much higher than sequential rewrite test with fdatasync + # if it does, durability is mostly likely "relaxed" + - name: fileio seqrewr prepare + run: sysbench fileio --file-block-size=4096 --file-test-mode=seqrewr --file-num=1 prepare + - name: fileio seqrewr run fdatasync + if: runner.os != 'macOS' + run: sysbench fileio --file-block-size=4096 --file-test-mode=seqrewr --file-fsync-mode=fdatasync --file-fsync-all=on --file-num=1 --report-interval=1 --time=20 --histogram run + - name: fileio seqrewr run fsync + run: sysbench fileio --file-block-size=4096 --file-test-mode=seqrewr --file-fsync-mode=fsync --file-fsync-all=on --file-num=1 --report-interval=1 --time=20 --histogram run diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml new file mode 100644 index 00000000..6bad80ec --- /dev/null +++ b/.github/workflows/cmake.yml @@ -0,0 +1,14 @@ +name: CMake +on: [push, pull_request] + + +jobs: + CI: + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + uses: ./.github/workflows/cmake-single-os.yml + with: + os: ${{ matrix.os}} + diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index 1942b900..00000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: macOS-CMake -on: [push, pull_request] - -jobs: - build: - strategy: - matrix: - os: [macos-latest] - runs-on: ${{ matrix.os }} - name: Build on ${{ matrix.os }} - steps: - - name: actions-setup-mysql - uses: shogo82148/actions-setup-mysql@v1.17.0 - with: - mysql-version: 'mysql-8.0' - my-cnf: | - innodb_log_file_size=256MB - innodb_buffer_pool_size=512MB - max_allowed_packet=16MB - max_connections=500 - - name: show variables - run: mysql -uroot --host=127.0.0.1 -e "show variables" - - name: install client drivers - run : | - HOMEBREW_NO_AUTO_UPDATE=1 brew install mysql-client libpq - brew link --force mysql-client - - name: checkout - uses: actions/checkout@v3 - - name: configure - run: cmake -B build_dir -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DWITH_PGSQL=1 -DCMAKE_COMPILE_WARNING_AS_ERROR=1 - - name: build - run: cmake --build build_dir --config RelWithDebInfo -j4 - - name: test_install - run: | - cmake --install build_dir --prefix install_dir --config RelWithDebInfo - echo "$GITHUB_WORKSPACE/install_dir/bin" >> $GITHUB_PATH - - name: which sysbench - run: which sysbench - - name: sysbench version - run: sysbench --version - - name: create database sbtest - run: mysql -uroot --host=127.0.0.1 -e "create database sbtest" - - name: sysbench oltp_read_write prepare - run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 --table-size=100000 prepare - - name: sysbench oltp_read_write run - run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 --time=30 --table-size=100000 --threads=4 --report-interval=1 run - - name: sysbench oltp_read_write run ssl - run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 --time=30 --mysql-ssl=preferred --table-size=100000 --threads=4 --report-interval=1 run - - name: sysbench oltp_read_write cleanup - run: sysbench oltp_read_write --mysql-user=root --mysql-host=127.0.0.1 cleanup diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index 5a1f4b2b..00000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: ubuntu-CMake -on: [push, pull_request] - -jobs: - build: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - name: Build on ${{ matrix.os }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: configure - run: cmake . -DWITH_PGSQL=1 -DCMAKE_COMPILE_WARNING_AS_ERROR=1 - - name: Build - run: cmake --build . - - name: MySQL version - run: mysql_config --version - - name: Sysbench version - run: ./src/sysbench --version - - name: test - run: cmake --build . --target test diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index 21cff69f..00000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Windows -on: [push, pull_request] - -jobs: - build: - strategy: - matrix: - os: [windows-latest] - runs-on: ${{ matrix.os }} - name: Build on ${{ matrix.os }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - id: vcpkg_cache - with: - path: ~\AppData\Local\vcpkg - key: ${{ runner.os }}-vcpkg-libpq-zlib - - name: vcpkg install - run: vcpkg install libpq zlib --triplet=x64-windows-release - - name: configure - run: cmake -B build_dir -DCMAKE_COMPILE_WARNING_AS_ERROR=1 -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-release -DWITH_PGSQL=1 - - name: build - run: cmake --build build_dir --config RelWithDebInfo --target package -j 4 - - name: upload package - uses: actions/upload-artifact@v3 - with: - name: package - path: build_dir/*.zip - - name: test_install - run: | - cmake --install build_dir --prefix install_dir --config RelWithDebInfo - echo "$pwd\install_dir\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - cat $env:GITHUB_PATH - - name: sysbench version - run: sysbench --version - - name: sysbench help - run: sysbench --help - - name: actions-setup-mysql - uses: shogo82148/actions-setup-mysql@v1.17.0 - env: - TMPDIR: ${{ runner.temp }} - TMP: ${{ runner.temp }} - TEMP: ${{ runner.temp }} - with: - mysql-version: 'mariadb-10.9' - my-cnf: | - innodb_log_file_size=1G - innodb_buffer_pool_size=512MB - max_allowed_packet=16MB - max_connections=500 - named-pipe=1 - socket=MySQL - - name: actions-setup-pgsql - uses: ikalnytskyi/action-setup-postgres@v4 - with: - username: sbtest - password: sbtest - database: sbtest - - name: create database sbtest - run: mysql -uroot -e "create database sbtest" - - name: sysbench oltp_update_index prepare - run: sysbench oltp_update_index --mysql-user=root --table-size=100000 prepare - - name: sysbench oltp_point_select run - run: sysbench oltp_point_select --mysql-user=root --mysql-host=. --time=30 --table-size=100000 --threads=1 --report-interval=1 --histogram run - - name: sysbench oltp_update_index run ssl - run: sysbench oltp_update_index --mysql-user=root --time=30 --mysql-ssl=on --table-size=100000 --threads=1 --report-interval=1 --histogram run - - name: sysbench oltp_update_index cleanup - run: sysbench oltp_update_index --mysql-user=root cleanup - - name: sysbench pgsql oltp_update_index prepare - run: sysbench oltp_update_index --db-driver=pgsql --pgsql-password=sbtest --table-size=100000 prepare - - name: sysbench pgsql oltp_point_select run - run: sysbench oltp_point_select --db-driver=pgsql --pgsql-password=sbtest --time=30 --table-size=100000 --threads=1 --report-interval=1 --histogram run - - name: sysbench pgsql oltp_update_index run - run: sysbench oltp_update_index --db-driver=pgsql --pgsql-password=sbtest --time=30 --table-size=100000 --threads=1 --report-interval=1 --histogram run - - name: sysbench oltp_update_index cleanup - run: sysbench oltp_update_index --db-driver=pgsql --pgsql-password=sbtest cleanup From 6d18436fc3f0ca6c50b2f61c58a2332760796859 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 9 Sep 2023 21:26:23 +0200 Subject: [PATCH 40/45] Update README.md with instructions on how to build with CMake --- README.md | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 384d928e..07141c05 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Coverage Status][coveralls-badge]][coveralls-url] [![License][license-badge]][license-url] + **Table of Contents** @@ -13,7 +14,6 @@ - [Installing from Binary Packages](#installing-from-binary-packages) - [Linux](#linux) - [macOS](#macos) - - [Windows](#windows) - [Building and Installing From Source](#building-and-installing-from-source) - [Build Requirements](#build-requirements) - [Windows](#windows) @@ -22,6 +22,8 @@ - [Fedora](#fedora) - [macOS](#macos) - [Build and Install](#build-and-install) + - [Using CMake](#using-cmake) + - [Using autotools](#using-autotools) - [Usage](#usage) - [General Syntax](#general-syntax) - [General Command Line Options](#general-command-line-options) @@ -104,18 +106,6 @@ On macOS, up-to-date sysbench packages are available from Homebrew: brew install sysbench ``` -## Windows -As of sysbench 1.0 support for native Windows builds was dropped. It may -be re-introduced in later releases. Currently, the recommended way to -obtain sysbench on Windows is -using -[Windows Subsystem for Linux available in Windows 10](https://msdn.microsoft.com/en-us/commandline/wsl/about). - -After installing WSL and getting into he bash prompt on Windows -following Debian/Ubuntu installation instructions is -sufficient. Alternatively, one can use WSL to build and install sysbench -from source, or use an older sysbench release to build a native binary. - # Building and Installing From Source It is recommended to install sysbench from the official binary @@ -127,15 +117,12 @@ architecture for which no binary packages are available. ## Build Requirements ### Windows -As of sysbench 1.0 support for native Windows builds was -dropped. It may be re-introduced in later versions. Currently, the -recommended way to build sysbench on Windows is using -[Windows Subsystem for Linux available in Windows 10](https://msdn.microsoft.com/en-us/commandline/wsl/about). - -After installing WSL and getting into bash prompt on Windows, following -Debian/Ubuntu build instructions is sufficient. Alternatively, one can -build and use an older 0.5 release on Windows. +If you want to build with postgresql, recommended way to do that +is to install *libpq* via vcpkg dependency manager like this: +``` shell + vcpkg install libpq +``` ### Debian/Ubuntu ``` shell apt -y install make automake libtool pkg-config libaio-dev @@ -177,6 +164,21 @@ Assuming you have Xcode (or Xcode Command Line Tools) and Homebrew installed: ``` ## Build and Install +As of sysbench 1.1.0, support for building with CMake was added on all supported platforms. +### Using CMake +```shell + # add -DWITH_PGSQL=ON to build with PostgreSQL support + cmake . + cmake --build . -j --config Release + cmake --install . +``` +On Windows, cmake will build sysbench on Windows with MySQL support via libmariadb +external project. That means, there is no needs to install the client drivers, but you +will need Git and internet access at the build time. +If this is not desired, pass -DWITH_LIBMARIADB=OFF to cmake command line + +To build without MySQL support, pass -DWITH_MYSQL=OFF +### Using autotools ``` shell ./autogen.sh # Add --with-pgsql to build with PostgreSQL support From 5f337bd6c1f5330af4ff031f8d3859558b5dc020 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 18 Feb 2024 15:46:29 +0100 Subject: [PATCH 41/45] Update libmariadb git tag, remove older workarounds Also link MySQL-specific plugins statically. --- cmake/BuildLibmariadb.cmake | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/cmake/BuildLibmariadb.cmake b/cmake/BuildLibmariadb.cmake index 66f04927..3c19e687 100644 --- a/cmake/BuildLibmariadb.cmake +++ b/cmake/BuildLibmariadb.cmake @@ -18,7 +18,7 @@ endif() set(mariadbclient_IMPORTED_LOCATION ${install_dir}/lib/mariadb/${mariadbclient_LIBRARY_NAME}) -set(_EXTRA_CMAKE_ARGS) +set(_EXTRA_CMAKE_ARGS -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF) if(WIN32) if(MINGW) @@ -33,28 +33,15 @@ if(WIN32) endif() if(MSVC) - list(APPEND _EXTRA_CMAKE_ARGS -DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE=ON - -DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO=ON - -DCMAKE_POLICY_DEFAULT_CMP0069=NEW) -elseif(WIN32) - # gcc/clang is not a native env for the mariadb client library can compile - # only with some extra flags - set(_CFLAGS - "-DWIN32_MEAN_AND_LEAN -DNOGDI -DNOMINMAX -D_CRT_NONSTDC_NO_WARNINGS \ - -D_CRT_SECURE_NO_WARNINGS -D_WINSOCKAPI_ -DNOCRYPT -DNOCOMM\ - ") - if(CMAKE_C_COMPILER_ID STREQUAL "Clang") - string(APPEND _CFLAGS " -Wno-error") - endif() - if(CMAKE_COMPILER_IS_GNUCC) - string(APPEND _CFLAGS - " -Dstrerror_r(errno,buf,len)=strerror_s(buf,len,errno)") - endif() - list(APPEND _EXTRA_CMAKE_ARGS "-DCMAKE_C_FLAGS_INIT=${_CFLAGS}") -endif() +list(APPEND _EXTRA_CMAKE_ARGS + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE=ON + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO=ON + -DCMAKE_POLICY_DEFAULT_CMP0069=NEW +) +ENDIF() set(LIBMARIADB_GIT_TAG - v3.3.4 + v3.3.9 CACHE STRING "Git tag of mariadb client library. Set to empty string to get most recent revision" @@ -100,7 +87,7 @@ ExternalProject_Add( -DCLIENT_PLUGIN_CACHING_SHA2_PASSWORD=STATIC -DCLIENT_PLUGIN_DIALOG=OFF -DCLIENT_PLUGIN_CLIENT_ED25519=OFF - -DCLIENT_PLUGIN_SHA256_PASSWORD=OFF + -DCLIENT_PLUGIN_SHA256_PASSWORD=STATIC -DCLIENT_PLUGIN_MYSQL_CLEAR_PASSWORD=OFF -DCLIENT_PLUGIN_ZSTD=OFF -DSKIP_TESTS=1 From b691a7dcc8cd6a7743ab2d8fb05b261ede270b25 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 18 Feb 2024 15:48:11 +0100 Subject: [PATCH 42/45] Fixes for new mingw and clang. --- cmake/BuildLuaJit.cmake | 2 +- src/sb_options.c | 4 ++-- src/win/pthread/pthread.h | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/BuildLuaJit.cmake b/cmake/BuildLuaJit.cmake index f56eb7f8..2efc08d0 100644 --- a/cmake/BuildLuaJit.cmake +++ b/cmake/BuildLuaJit.cmake @@ -42,7 +42,7 @@ if(NOT WIN32) else() if (MINGW) # GNU compatible make required - find_program(MAKE_EXECUTABLE NAMES gmake REQUIRED) + find_program(MAKE_EXECUTABLE NAMES gmake mingw32-make REQUIRED) set(buildCommand ${MAKE_EXECUTABLE}) set(luajit_implib ${build_dir}/src/libluajit-5.1.dll.a) else() diff --git a/src/sb_options.c b/src/sb_options.c index b7bf496f..51728d51 100644 --- a/src/sb_options.c +++ b/src/sb_options.c @@ -441,7 +441,7 @@ char *sb_print_value_size(char *buf, unsigned int buflen, double value) } -value_t *new_value() +value_t *new_value(void) { value_t *newval; @@ -453,7 +453,7 @@ value_t *new_value() } -option_t *new_option() +option_t *new_option(void) { option_t *newopt; diff --git a/src/win/pthread/pthread.h b/src/win/pthread/pthread.h index 6f2132d0..23aebda2 100644 --- a/src/win/pthread/pthread.h +++ b/src/win/pthread/pthread.h @@ -648,6 +648,7 @@ static inline int pthread_attr_init(pthread_attr_t *attr) static inline int pthread_attr_destroy(pthread_attr_t *attr) { /* No need to do anything */ + (void)attr; return 0; } @@ -775,7 +776,7 @@ static inline int pthread_create(pthread_t *th, const pthread_attr_t *attr, void { struct _pthread_v *tv = malloc(sizeof(struct _pthread_v)); unsigned ssize = 0; - + uintptr_t r; if (!tv) return 1; *th = tv; @@ -798,8 +799,8 @@ static inline int pthread_create(pthread_t *th, const pthread_attr_t *attr, void /* Make sure tv->h has value of -1 */ _ReadWriteBarrier(); - - tv->h = (HANDLE) _beginthreadex(NULL, ssize, pthread_create_wrapper, tv, 0, NULL); + r = _beginthreadex(NULL, ssize, pthread_create_wrapper, tv, 0, NULL); + tv->h = (HANDLE) r; /* Failed */ if (!tv->h) return 1; From ddecca9c473c474a257e70794d38ff366a694390 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 18 Feb 2024 15:49:19 +0100 Subject: [PATCH 43/45] Windows : export one more symbol from sysbench executable, for sysbench-tpcc --- src/lua/internal/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lua/internal/CMakeLists.txt b/src/lua/internal/CMakeLists.txt index 0a009aee..d8bfbe2f 100644 --- a/src/lua/internal/CMakeLists.txt +++ b/src/lua/internal/CMakeLists.txt @@ -38,6 +38,8 @@ if(WIN32) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_FILE ${sysbench_def}.tmp COMMAND_ERROR_IS_FATAL ANY) + # Hack to get percona's tpcc running + file(APPEND ${sysbench_def}.tmp "\nsb_counter_inc") configure_file(${sysbench_def}.tmp ${sysbench_def} COPYONLY) file(REMOVE ${sysbench_def}.tmp) target_sources(sysbench PRIVATE ${sysbench_def}) From 5cfaa10ceae6a7569faa691209c91ad64561fb93 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 18 Feb 2024 17:39:54 +0100 Subject: [PATCH 44/45] Test - Workaround Windows named pipe bugs in MySQL. Also disable Windows ASAN - something bad happened, it can't run on CI Reenable ASAN on Linux though --- .github/workflows/cmake-single-os.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake-single-os.yml b/.github/workflows/cmake-single-os.yml index 575b3255..7c6b2a87 100644 --- a/.github/workflows/cmake-single-os.yml +++ b/.github/workflows/cmake-single-os.yml @@ -128,7 +128,7 @@ jobs: test-asan: runs-on: ${{ inputs.os }} # Linux is excluded, due to buggy ASAN ubuntu - if: ${{ !contains(inputs.os,'ubuntu') }} + if: ${{ !contains(inputs.os,'windows') }} name: test-asan continue-on-error: true steps: @@ -189,7 +189,7 @@ jobs: #if: contains(inputs.os,'windows') name: test-${{matrix.db}} env: - COMMON_SYSBENCH_PARAMS: ${{ matrix.db == 'postgres' && '--db-driver=pgsql --pgsql-password=sbtest' || '--mysql-user=root --mysql-socket=/tmp/mysql.sock'}} --time=30 --table-size=1000000 --report-interval=1 --thread-init-timeout=60 --histogram + COMMON_SYSBENCH_PARAMS: ${{ matrix.db == 'postgres' && '--db-driver=pgsql --pgsql-password=sbtest' || '--mysql-user=root'}} --mysql-socket=${{ contains(inputs.os,'windows') && 'mysql.sock' || '/tmp/mysql.sock' }} --time=30 --table-size=1000000 --report-interval=1 --thread-init-timeout=300 --histogram steps: - name: Download build @@ -246,7 +246,7 @@ jobs: max_allowed_packet=16MB skip-log-bin loose-enable-named-pipe - socket=/tmp/mysql.sock + socket=${{ runner.os == 'Windows' && 'mysql.sock' || '/tmp/mysql.sock' }} max_connections=1000 innodb_max_dirty_pages_pct_lwm=10 From 1edbba5c0be96204edd246b76b3e6753ba6a326e Mon Sep 17 00:00:00 2001 From: Michael Plugin Date: Tue, 30 Jan 2024 15:28:22 +0700 Subject: [PATCH 45/45] mysql_ssl_set replaced with mysql_options --- src/drivers/mysql/drv_mysql.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index 688b243a..e24df2ea 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -403,12 +403,14 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con) if (args.use_ssl) { - DEBUG("mysql_ssl_set(%p, \"%s\", \"%s\", \"%s\", NULL, \"%s\")", con, + DEBUG("mysql_options(%p, \"%s\", \"%s\", \"%s\", \"%s\")", con, SAFESTR(args.ssl_key), SAFESTR(args.ssl_cert), SAFESTR(args.ssl_ca), SAFESTR(args.ssl_cipher)); - mysql_ssl_set(con, args.ssl_key, args.ssl_cert, args.ssl_ca, NULL, - args.ssl_cipher); + mysql_options(con, MYSQL_OPT_SSL_KEY, args.ssl_key); + mysql_options(con, MYSQL_OPT_SSL_CERT, args.ssl_cert); + mysql_options(con, MYSQL_OPT_SSL_CA, args.ssl_ca); + mysql_options(con, MYSQL_OPT_SSL_CIPHER, args.ssl_cipher); } if (args.use_compression)