diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f9774c2327cb0..d918d3aeddd000 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,23 +225,51 @@ endif() # detect OS # -set(LINUX False) -set(FREEBSD False) -set(MACOS False) +set(LINUX False) +set(FREEBSD False) +set(MACOS False) +set(WINDOWS False) +set(FOREIGN_OS False) -if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") +if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") set(MACOS True) set(COMPILED_FOR_MACOS True) - find_library(IOKIT IOKit) find_library(FOUNDATION Foundation) -elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") + message(INFO " Compiling for MacOS... ") +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") set(FREEBSD True) set(COMPILED_FOR_FREEBSD True) -else() + message(INFO " Compiling for FreeBSD... ") +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") set(LINUX True) set(COMPILED_FOR_LINUX True) add_definitions(-D_GNU_SOURCE) + message(INFO " Compiling for Linux... ") +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "CYGWIN" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "MSYS" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows") + set(WINDOWS True) + set(COMPILED_FOR_WINDOWS True) + add_definitions(-D_GNU_SOURCE) + + if($ENV{CLION_IDE}) + # clion needs these to find the includes + if("${CMAKE_SYSTEM_NAME}" STREQUAL "MSYS" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows") + if("$ENV{MSYSTEM}" STREQUAL "MSYS") + include_directories(c:/msys64/usr/include) + include_directories(c:/msys64/usr/include/w32api) + elseif("$ENV{MSYSTEM}" STREQUAL "MINGW64") + include_directories(c:/msys64/mingw64/include) + elseif("$ENV{MSYSTEM}" STREQUAL "UCRT64") + include_directories(c:/msys64/ucrt64/include) + endif() + endif() + endif() + + message(INFO " Compiling for Windows (${CMAKE_SYSTEM_NAME}, MSYSTEM=$ENV{MSYSTEM})... ") +else() + set(FOREIGN_OS True) + set(COMPILED_FOR_FOREIGN_OS True) + message(WARNING " Compiling for Unknown O/S... (${CMAKE_SYSTEM_NAME})") endif() if(ENABLE_PLUGIN_EBPF) @@ -326,6 +354,20 @@ check_include_file("sys/statvfs.h" HAVE_SYS_STATVFS_H) check_include_file("inttypes.h" HAVE_INTTYPES_H) check_include_file("stdint.h" HAVE_STDINT_H) check_include_file("sys/capability.h" HAVE_SYS_CAPABILITY_H) +check_include_file("arpa/inet.h" HAVE_ARPA_INET_H) +check_include_file("netinet/tcp.h" HAVE_NETINET_TCP_H) +check_include_file("sys/ioctl.h" HAVE_SYS_IOCTL_H) +check_include_file("grp.h" HAVE_GRP_H) +check_include_file("pwd.h" HAVE_PWD_H) +check_include_file("net/if.h" HAVE_NET_IF_H) +check_include_file("poll.h" HAVE_POLL_H) +check_include_file("syslog.h" HAVE_SYSLOG_H) +check_include_file("sys/mman.h" HAVE_SYS_MMAN_H) +check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H) +check_include_file("sys/socket.h" HAVE_SYS_SOCKET_H) +check_include_file("sys/wait.h" HAVE_SYS_WAIT_H) +check_include_file("sys/un.h" HAVE_SYS_UN_H) +check_include_file("spawn.h" HAVE_SPAWN_H) # # check symbols @@ -340,9 +382,15 @@ check_symbol_exists(finite "math.h" HAVE_FINITE) check_symbol_exists(isfinite "math.h" HAVE_ISFINITE) check_symbol_exists(dlsym "dlfcn.h" HAVE_DLSYM) +check_function_exists(pthread_getthreadid_np HAVE_PTHREAD_GETTHREADID_NP) +check_function_exists(pthread_threadid_np HAVE_PTHREAD_THREADID_NP) +check_function_exists(gettid HAVE_GETTID) +check_function_exists(waitid HAVE_WAITID) check_function_exists(nice HAVE_NICE) check_function_exists(recvmmsg HAVE_RECVMMSG) check_function_exists(getpriority HAVE_GETPRIORITY) +check_function_exists(setenv HAVE_SETENV) +check_function_exists(strndup HAVE_STRNDUP) check_function_exists(sched_getscheduler HAVE_SCHED_GETSCHEDULER) check_function_exists(sched_setscheduler HAVE_SCHED_SETSCHEDULER) @@ -624,10 +672,10 @@ set(LIBNETDATA_FILES src/libnetdata/log/journal.h src/libnetdata/log/log.c src/libnetdata/log/log.h - src/libnetdata/os.c - src/libnetdata/os.h + src/libnetdata/os/os.c + src/libnetdata/os/os.h src/libnetdata/simple_hashtable.h - src/libnetdata/byteorder.h + src/libnetdata/os/byteorder.h src/libnetdata/onewayalloc/onewayalloc.c src/libnetdata/onewayalloc/onewayalloc.h src/libnetdata/popen/popen.c @@ -682,6 +730,34 @@ set(LIBNETDATA_FILES src/libnetdata/linked-lists.h src/libnetdata/storage-point.h src/libnetdata/bitmap64.h + src/libnetdata/os/waitid.c + src/libnetdata/os/waitid.h + src/libnetdata/os/gettid.c + src/libnetdata/os/gettid.h + src/libnetdata/os/adjtimex.c + src/libnetdata/os/adjtimex.h + src/libnetdata/os/setresuid.c + src/libnetdata/os/setresuid.h + src/libnetdata/os/setresgid.c + src/libnetdata/os/setresgid.h + src/libnetdata/os/getgrouplist.c + src/libnetdata/os/getgrouplist.h + src/libnetdata/os/get_pid_max.c + src/libnetdata/os/get_pid_max.h + src/libnetdata/os/os-freebsd-wrappers.c + src/libnetdata/os/os-freebsd-wrappers.h + src/libnetdata/os/os-macos-wrappers.c + src/libnetdata/os/os-macos-wrappers.h + src/libnetdata/os/get_system_cpus.c + src/libnetdata/os/get_system_cpus.h + src/libnetdata/os/tinysleep.c + src/libnetdata/os/tinysleep.h + src/libnetdata/os/uuid_generate.c + src/libnetdata/os/uuid_generate.h + src/libnetdata/os/setenv.c + src/libnetdata/os/setenv.h + src/libnetdata/os/strndup.c + src/libnetdata/os/strndup.h ) if(ENABLE_PLUGIN_EBPF) @@ -956,6 +1032,16 @@ else() ) endif() +set(INTERNAL_COLLECTORS_FILES + src/collectors/common-contexts/common-contexts.h + src/collectors/common-contexts/disk.io.h + src/collectors/common-contexts/system.io.h + src/collectors/common-contexts/system.ram.h + src/collectors/common-contexts/mem.swap.h + src/collectors/common-contexts/mem.pgfaults.h + src/collectors/common-contexts/mem.available.h +) + set(PLUGINSD_PLUGIN_FILES src/collectors/plugins.d/plugins_d.c src/collectors/plugins.d/plugins_d.h @@ -1197,6 +1283,24 @@ set(FREEBSD_PLUGIN_FILES src/collectors/proc.plugin/zfs_common.h ) +set(WINDOWS_PLUGIN_FILES + src/collectors/windows.plugin/windows_plugin.c + src/collectors/windows.plugin/windows_plugin.h + src/collectors/windows.plugin/GetSystemUptime.c + src/collectors/windows.plugin/GetSystemRAM.c + src/collectors/windows.plugin/GetSystemCPU.c + src/collectors/windows.plugin/perflib.c + src/collectors/windows.plugin/perflib.h + src/collectors/windows.plugin/perflib-rrd.c + src/collectors/windows.plugin/perflib-rrd.h + src/collectors/windows.plugin/perflib-names.c + src/collectors/windows.plugin/perflib-dump.c + src/collectors/windows.plugin/perflib-storage.c + src/collectors/windows.plugin/perflib-processor.c + src/collectors/windows.plugin/perflib-network.c + src/collectors/windows.plugin/perflib-memory.c +) + set(PROC_PLUGIN_FILES src/collectors/proc.plugin/ipc.c src/collectors/proc.plugin/plugin_proc.c @@ -1315,6 +1419,7 @@ if(LINUX) ${PROC_PLUGIN_FILES} ${TC_PLUGIN_FILES} ${TIMEX_PLUGIN_FILES} + ${INTERNAL_COLLECTORS_FILES} ) if(ENABLE_SENTRY) @@ -1327,12 +1432,20 @@ elseif(MACOS) src/daemon/static_threads_macos.c ${MACOS_PLUGIN_FILES} ${TIMEX_PLUGIN_FILES} + ${INTERNAL_COLLECTORS_FILES} ) elseif(FREEBSD) list(APPEND NETDATA_FILES src/daemon/static_threads_freebsd.c ${FREEBSD_PLUGIN_FILES} ${TIMEX_PLUGIN_FILES} + ${INTERNAL_COLLECTORS_FILES} + ) +elseif(WINDOWS) + list(APPEND NETDATA_FILES + src/daemon/static_threads_windows.c + ${WINDOWS_PLUGIN_FILES} + ${INTERNAL_COLLECTORS_FILES} ) endif() @@ -1518,6 +1631,7 @@ target_include_directories(libnetdata BEFORE PUBLIC ${CONFIG_H_DIR} ${CMAKE_SOUR target_link_libraries(libnetdata PUBLIC "$<$>:atomic>" "$<$,$>:pthread;rt>" + "$<$:kernel32;advapi32;winmm;rpcrt4>" "$<$:m>" "${SYSTEMD_LDFLAGS}") @@ -1578,7 +1692,7 @@ if(LIBBROTLI_FOUND) endif() # uuid -if(MACOS) +if(MACOS OR WINDOWS) # UUID functionality is part of the system libraries here, so no extra # stuff needed. else() @@ -1879,9 +1993,9 @@ endif() if(ENABLE_PLUGIN_CUPS) pkg_check_modules(CUPS libcups) - if(NOT CUPS_LIBRARIES) + if(NOT CUPS_FOUND) pkg_check_modules(CUPS cups) - if(NOT CUPS_LIBRARIES) + if(NOT CUPS_FOUND) find_program(CUPS_CONFIG cups-config) if(CUPS_CONFIG) execute_process(COMMAND ${CUPS_CONFIG} --api-version OUTPUT_VARIABLE CUPS_API_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -2914,10 +3028,12 @@ install(FILES COMPONENT netdata DESTINATION ${WEB_DEST}/.well-known/dnt) -# v0 dashboard -install(FILES - src/web/gui/v0/index.html - COMPONENT netdata - DESTINATION ${WEB_DEST}/v0) +if(NOT WINDOWS) + # v0 dashboard + install(FILES + src/web/gui/v0/index.html + COMPONENT netdata + DESTINATION ${WEB_DEST}/v0) +endif() include(Packaging) diff --git a/packaging/cmake/Modules/NetdataProtobuf.cmake b/packaging/cmake/Modules/NetdataProtobuf.cmake index 271252fb5c1663..8135cc315c53d1 100644 --- a/packaging/cmake/Modules/NetdataProtobuf.cmake +++ b/packaging/cmake/Modules/NetdataProtobuf.cmake @@ -56,54 +56,67 @@ endfunction() # Handle detection of Protobuf macro(netdata_detect_protobuf) - if(NOT ENABLE_BUNDLED_PROTOBUF) - if (NOT BUILD_SHARED_LIBS) - set(Protobuf_USE_STATIC_LIBS On) + if(COMPILED_FOR_WINDOWS) + set(PROTOBUF_PROTOC_EXECUTABLE "$ENV{PROTOBUF_PROTOC_EXECUTABLE}") + if(NOT PROTOBUF_PROTOC_EXECUTABLE) + set(PROTOBUF_PROTOC_EXECUTABLE "/bin/protoc") endif() - - # The FindProtobuf CMake module shipped by upstream CMake is - # broken for Protobuf version 22.0 and newer because it does - # not correctly pull in the new Abseil dependencies. Protobuf - # itself sometimes ships a CMake Package Configuration module - # that _does_ work correctly, so use that in preference to the - # Find module shipped with CMake. - # - # The code below works by first attempting to use find_package - # in config mode, and then checking for the existence of the - # target we actually use that gets defined by the protobuf - # CMake Package Configuration Module to determine if that - # worked. A bit of extra logic is required in the case of the - # config mode working, because some systems ship compatibility - # logic for the old FindProtobuf module while others do not. - # - # Upstream bug reference: https://gitlab.kitware.com/cmake/cmake/-/issues/24321 - find_package(Protobuf CONFIG) - - if(NOT TARGET protobuf::libprotobuf) - message(STATUS "Could not find Protobuf using Config mode, falling back to Module mode") - find_package(Protobuf REQUIRED) + set(PROTOBUF_CFLAGS_OTHER "") + set(PROTOBUF_INCLUDE_DIRS "") + set(PROTOBUF_LIBRARIES "-lprotobuf") + + set(ENABLE_PROTOBUF True) + set(HAVE_PROTOBUF True) + else() + if(NOT ENABLE_BUNDLED_PROTOBUF) + if (NOT BUILD_SHARED_LIBS) + set(Protobuf_USE_STATIC_LIBS On) + endif() + + # The FindProtobuf CMake module shipped by upstream CMake is + # broken for Protobuf version 22.0 and newer because it does + # not correctly pull in the new Abseil dependencies. Protobuf + # itself sometimes ships a CMake Package Configuration module + # that _does_ work correctly, so use that in preference to the + # Find module shipped with CMake. + # + # The code below works by first attempting to use find_package + # in config mode, and then checking for the existence of the + # target we actually use that gets defined by the protobuf + # CMake Package Configuration Module to determine if that + # worked. A bit of extra logic is required in the case of the + # config mode working, because some systems ship compatibility + # logic for the old FindProtobuf module while others do not. + # + # Upstream bug reference: https://gitlab.kitware.com/cmake/cmake/-/issues/24321 + find_package(Protobuf CONFIG) + + if(NOT TARGET protobuf::libprotobuf) + message(STATUS "Could not find Protobuf using Config mode, falling back to Module mode") + find_package(Protobuf REQUIRED) + endif() endif() - endif() - if(TARGET protobuf::libprotobuf) - if(NOT Protobuf_PROTOC_EXECUTABLE AND TARGET protobuf::protoc) - set(Protobuf_PROTOC_EXECUTABLE protobuf::protoc) + if(TARGET protobuf::libprotobuf) + if(NOT Protobuf_PROTOC_EXECUTABLE AND TARGET protobuf::protoc) + set(Protobuf_PROTOC_EXECUTABLE protobuf::protoc) + endif() + + # It is technically possible that this may still not + # be set by this point, so we need to check it and + # fail noisily if it isn't because the build won't + # work without it. + if(NOT Protobuf_PROTOC_EXECUTABLE) + message(FATAL_ERROR "Could not determine the location of the protobuf compiler for the detected version of protobuf.") + endif() + + set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) + set(PROTOBUF_LIBRARIES protobuf::libprotobuf) endif() - # It is technically possible that this may still not - # be set by this point, so we need to check it and - # fail noisily if it isn't because the build won't - # work without it. - if(NOT Protobuf_PROTOC_EXECUTABLE) - message(FATAL_ERROR "Could not determine the location of the protobuf compiler for the detected version of protobuf.") - endif() - - set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) - set(PROTOBUF_LIBRARIES protobuf::libprotobuf) + set(ENABLE_PROTOBUF True) + set(HAVE_PROTOBUF True) endif() - - set(ENABLE_PROTOBUF True) - set(HAVE_PROTOBUF True) endmacro() # Helper function to compile protocol definitions into C++ code. diff --git a/packaging/cmake/config.cmake.h.in b/packaging/cmake/config.cmake.h.in index 276b051f9e8be7..c1d93636dabfa4 100644 --- a/packaging/cmake/config.cmake.h.in +++ b/packaging/cmake/config.cmake.h.in @@ -12,6 +12,8 @@ #cmakedefine COMPILED_FOR_FREEBSD #cmakedefine COMPILED_FOR_LINUX #cmakedefine COMPILED_FOR_MACOS +#cmakedefine COMPILED_FOR_WINDOWS +#cmakedefine COMPILED_FOR_FOREIGN_OS // checked headers @@ -28,6 +30,20 @@ #cmakedefine HAVE_INTTYPES_H #cmakedefine HAVE_STDINT_H #cmakedefine HAVE_SYS_CAPABILITY_H +#cmakedefine HAVE_ARPA_INET_H +#cmakedefine HAVE_NETINET_TCP_H +#cmakedefine HAVE_SYS_IOCTL_H +#cmakedefine HAVE_GRP_H +#cmakedefine HAVE_PWD_H +#cmakedefine HAVE_NET_IF_H +#cmakedefine HAVE_POLL_H +#cmakedefine HAVE_SYSLOG_H +#cmakedefine HAVE_SYS_MMAN_H +#cmakedefine HAVE_SYS_RESOURCE_H +#cmakedefine HAVE_SYS_SOCKET_H +#cmakedefine HAVE_SYS_WAIT_H +#cmakedefine HAVE_SYS_UN_H +#cmakedefine HAVE_SPAWN_H #cmakedefine HAVE_CAPABILITY #cmakedefine HAVE_PROTOBUF @@ -44,8 +60,13 @@ #cmakedefine HAVE_FINITE #cmakedefine HAVE_ISFINITE #cmakedefine HAVE_RECVMMSG +#cmakedefine HAVE_PTHREAD_GETTHREADID_NP +#cmakedefine HAVE_PTHREAD_THREADID_NP +#cmakedefine HAVE_GETTID +#cmakedefine HAVE_WAITID #cmakedefine HAVE_NICE #cmakedefine HAVE_GETPRIORITY +#cmakedefine HAVE_SETENV #cmakedefine HAVE_DLSYM #cmakedefine HAVE_BACKTRACE @@ -70,6 +91,8 @@ #cmakedefine HAVE_C__GENERIC #cmakedefine HAVE_C_MALLOPT #cmakedefine HAVE_SETNS +#cmakedefine HAVE_STRNDUP +#cmakedefine SSL_HAS_PENDING #cmakedefine HAVE_FUNC_ATTRIBUTE_FORMAT #cmakedefine HAVE_FUNC_ATTRIBUTE_MALLOC diff --git a/packaging/utils/bash_execute.sh b/packaging/utils/bash_execute.sh new file mode 100644 index 00000000000000..4092db966182ae --- /dev/null +++ b/packaging/utils/bash_execute.sh @@ -0,0 +1,19 @@ +#!/usr/bin/bash + +convert_path() { + local ARG="$1" + ARG="${ARG//C:\\//c/}" + ARG="${ARG//c:\\//c/}" + ARG="${ARG//C:\///c/}" + ARG="${ARG//c:\///c/}" + + echo "$ARG" +} + +declare params=() +for x in "${@}" +do + params+=("$(convert_path "${x}")") +done + +"${params[@]}" diff --git a/packaging/utils/clion-msys-mingw64-environment.bat b/packaging/utils/clion-msys-mingw64-environment.bat new file mode 100644 index 00000000000000..19035d8ebbc60c --- /dev/null +++ b/packaging/utils/clion-msys-mingw64-environment.bat @@ -0,0 +1,17 @@ +@echo off +:: In Clion Toolchains +:: 1. Add a MinGW profile +:: 2. Set Toolset to C:\msys64\mingw64 +:: 3. Add environment and set the full path to this file, like: +:: C:\msys64\home\costa\src\netdata-ktsaou.git\packaging\utils\clion-mingw64-environment.bat +:: 4. Let everything else to Bundled and auto-detected +:: +set "batch_dir=%~dp0" +set "batch_dir=%batch_dir:\=/%" +set MSYSTEM=MINGW64 +set GOROOT=C:\msys64\mingw64 +set PATH="%PATH%;C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\msys64\bin" +::set PKG_CONFIG_EXECUTABLE=C:\msys64\mingw64\bin\pkg-config.exe +::set CMAKE_C_COMPILER=C:\msys64\mingw64\bin\gcc.exe +::set CMAKE_CC_COMPILER=C:\msys64\mingw64\bin\g++.exe +set PROTOBUF_PROTOC_EXECUTABLE=C:/msys64/mingw64/bin/protoc.exe diff --git a/packaging/utils/clion-msys-msys-environment.bat b/packaging/utils/clion-msys-msys-environment.bat new file mode 100644 index 00000000000000..9f0c095d37c354 --- /dev/null +++ b/packaging/utils/clion-msys-msys-environment.bat @@ -0,0 +1,20 @@ +@echo off +:: In Clion Toolchains +:: 1. Add a MinGW profile +:: 2. Set Toolset to C:\msys64\mingw64 +:: 3. Add environment and set the full path to this file, like: +:: C:\msys64\home\costa\src\netdata-ktsaou.git\packaging\utils\clion-mingw64-environment.bat +:: 4. Let everything else to Bundled and auto-detected +:: +set "batch_dir=%~dp0" +set "batch_dir=%batch_dir:\=/%" +set MSYSTEM=MSYS + +:: go exists only mingw64 / ucrt64 / etc, not under msys profile +set GOROOT=C:\msys64\mingw64 + +set PATH="%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin" +::set PKG_CONFIG_EXECUTABLE=C:\msys64\mingw64\bin\pkg-config.exe +::set CMAKE_C_COMPILER=C:\msys64\mingw64\bin\gcc.exe +::set CMAKE_CC_COMPILER=C:\msys64\mingw64\bin\g++.exe +set PROTOBUF_PROTOC_EXECUTABLE=%batch_dir%/protoc.bat diff --git a/packaging/utils/compile-on-windows.sh b/packaging/utils/compile-on-windows.sh new file mode 100644 index 00000000000000..aa44771b8ef4dc --- /dev/null +++ b/packaging/utils/compile-on-windows.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +# On MSYS2, install these dependencies to build netdata: +install_dependencies() { + pacman -S \ + git cmake ninja base-devel msys2-devel \ + libyaml-devel libzstd-devel libutil-linux libutil-linux-devel \ + mingw-w64-x86_64-toolchain mingw-w64-ucrt-x86_64-toolchain \ + mingw64/mingw-w64-x86_64-mold ucrt64/mingw-w64-ucrt-x86_64-mold \ + msys/gdb ucrt64/mingw-w64-ucrt-x86_64-gdb mingw64/mingw-w64-x86_64-gdb \ + msys/zlib-devel mingw64/mingw-w64-x86_64-zlib ucrt64/mingw-w64-ucrt-x86_64-zlib \ + msys/libuv-devel ucrt64/mingw-w64-ucrt-x86_64-libuv mingw64/mingw-w64-x86_64-libuv \ + liblz4-devel mingw64/mingw-w64-x86_64-lz4 ucrt64/mingw-w64-ucrt-x86_64-lz4 \ + openssl-devel mingw64/mingw-w64-x86_64-openssl ucrt64/mingw-w64-ucrt-x86_64-openssl \ + protobuf-devel mingw64/mingw-w64-x86_64-protobuf ucrt64/mingw-w64-ucrt-x86_64-protobuf \ + msys/pcre2-devel mingw64/mingw-w64-x86_64-pcre2 ucrt64/mingw-w64-ucrt-x86_64-pcre2 \ + msys/brotli-devel mingw64/mingw-w64-x86_64-brotli ucrt64/mingw-w64-ucrt-x86_64-brotli \ + msys/ccache ucrt64/mingw-w64-ucrt-x86_64-ccache mingw64/mingw-w64-x86_64-ccache \ + mingw64/mingw-w64-x86_64-go ucrt64/mingw-w64-ucrt-x86_64-go +} + +if [ "${1}" = "install" ] +then + install_dependencies || exit 1 + exit 0 +fi + +export PATH="/usr/local/bin:${PATH}" + +WT_ROOT="$(pwd)" +BUILD_TYPE="Debug" +NULL="" + +if [ -z "${MSYSTEM}" ]; then + build="${WT_ROOT}/build-${OSTYPE}" +else + build="${WT_ROOT}/build-${OSTYPE}-${MSYSTEM}" +fi + +if [ "$USER" = "vk" ]; then + build="${WT_ROOT}/build" +fi + +set -exu -o pipefail + +if [ -d "${build}" ] +then + rm -rf "${build}" +fi + +/usr/bin/cmake -S "${WT_ROOT}" -B "${build}" \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX="/opt/netdata" \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DCMAKE_C_FLAGS="-O0 -ggdb -Wall -Wextra -Wno-char-subscripts -Wa,-mbig-obj -pipe -DNETDATA_INTERNAL_CHECKS=1 -D_FILE_OFFSET_BITS=64 -D__USE_MINGW_ANSI_STDIO=1" \ + -DNETDATA_USER="${USER}" \ + -DDEFAULT_FEATURE_STATE=Off \ + -DENABLE_H2O=Off \ + -DENABLE_LOGS_MANAGEMENT_TESTS=Off \ + -DENABLE_ACLK=On \ + -DENABLE_CLOUD=On \ + -DENABLE_ML=On \ + -DENABLE_BUNDLED_JSONC=On \ + -DENABLE_BUNDLED_PROTOBUF=Off \ + ${NULL} + +ninja -v -C "${build}" || ninja -v -C "${build}" -j 1 + +echo +echo "Compile with:" +echo "ninja -v -C \"${build}\" || ninja -v -C \"${build}\" -j 1" diff --git a/packaging/utils/installer.nsi b/packaging/utils/installer.nsi new file mode 100644 index 00000000000000..b78f52ffe2337e --- /dev/null +++ b/packaging/utils/installer.nsi @@ -0,0 +1,34 @@ +Outfile "netdata-installer.exe" +InstallDir "C:\netdata" + +RequestExecutionLevel admin + +Section + SetOutPath $INSTDIR + WriteUninstaller $INSTDIR\uninstaller.exe +SectionEnd + +Section "Install MSYS2 environment" + SetOutPath $TEMP + + SetCompress off + File "C:\msys64\msys2-installer.exe" + nsExec::ExecToLog 'cmd.exe /C "$TEMP\msys2-installer.exe" in --confirm-command --accept-messages --root $INSTDIR' + + Delete "$TEMP\msys2-installer.exe" +SectionEnd + +Section "Install MSYS2 packages" + ExecWait '"$INSTDIR\usr\bin\bash.exe" -lc "pacman -S --noconfirm msys/libuv msys/protobuf"' +SectionEnd + +Section "Install Netdata" + SetOutPath $INSTDIR\opt\netdata + + SetCompress off + File /r "C:\msys64\opt\netdata\*.*" +SectionEnd + +Section "Uninstall" + nsExec::ExecToLog 'cmd.exe /C "$INSTDIR\uninstall.exe" pr --confirm-command' +SectionEnd diff --git a/packaging/utils/package-windows.sh b/packaging/utils/package-windows.sh new file mode 100644 index 00000000000000..b6996b9ecde1dd --- /dev/null +++ b/packaging/utils/package-windows.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +export PATH="/usr/local/bin:${PATH}" + +WT_ROOT="$(pwd)" +NULL="" + +if [ -z "${MSYSTEM}" ]; then + build="${WT_ROOT}/build-${OSTYPE}" +else + build="${WT_ROOT}/build-${OSTYPE}-${MSYSTEM}" +fi + +if [ "$USER" = "vk" ]; then + build="${WT_ROOT}/build" +fi + +set -exu -o pipefail + +ninja -v -C "${build}" install + +if [ ! -f "/msys2-installer.exe" ]; then + wget -O /msys2-installer.exe \ + "https://github.com/msys2/msys2-installer/releases/download/2024-05-07/msys2-x86_64-20240507.exe" +fi + +makensis "${WT_ROOT}/packaging/utils/installer.nsi" diff --git a/packaging/utils/protoc.bat b/packaging/utils/protoc.bat new file mode 100644 index 00000000000000..fe7a76f27811f5 --- /dev/null +++ b/packaging/utils/protoc.bat @@ -0,0 +1,9 @@ +@echo off +:: +:: The problem with /usr/bin/protoc is that it accepts colon separated (:) paths at its parameters. +:: This makes C:/ being parsed as 2 paths: C and /, which of course both fail. +:: To overcome this problem, we use bash_execute.sh, which replaces all occurences of C: with /c. +:: +set "batch_dir=%~dp0" +set "batch_dir=%batch_dir:\=/%" +C:\msys64\usr\bin\bash.exe %batch_dir%/bash_execute.sh protoc %* diff --git a/packaging/utils/windows-openssh-to-msys.bat b/packaging/utils/windows-openssh-to-msys.bat new file mode 100644 index 00000000000000..829cb4845fa57d --- /dev/null +++ b/packaging/utils/windows-openssh-to-msys.bat @@ -0,0 +1,118 @@ +@echo off +:: +:: This script will: +:: +:: 1. install the windows OpenSSH server (either via dsim or download it) +:: 2. activate the windows OpenSSH service +:: 3. open OpenSSH TCP port at windows firewall +:: 4. create a small batch file to start an MSYS session +:: 5. Set the default OpenSSH startup script to start the MSYS session +:: +:: Problems: +:: On older windows versions, terminal emulation is broken. +:: So, on windows 10 or windows server before 2019, the ssh session +:: will not have proper terminal emulation and will be not be able to +:: be used for editing files. +:: For more info check: +:: https://github.com/PowerShell/Win32-OpenSSH/issues/1260 +:: + +:: Check if OpenSSH Server is already installed +sc query sshd >nul 2>&1 +if %errorlevel% neq 0 ( + echo "OpenSSH Server not found. Attempting to install via dism..." + goto :install_openssh_dism +) else ( + echo "OpenSSH Server is already installed." + goto :configure_openssh +) + +:: Install OpenSSH using dism +:install_openssh_dism +dism /online /Enable-Feature /FeatureName:OpenSSH-Client /All >nul 2>&1 +dism /online /Enable-Feature /FeatureName:OpenSSH-Server /All >nul 2>&1 + +:: Check if dism succeeded in installing OpenSSH +sc query sshd >nul 2>&1 +if %errorlevel% neq 0 ( + echo "OpenSSH installation via dism failed or is unavailable." + goto :install_openssh_manual +) else ( + echo "OpenSSH installed successfully using dism." + goto :configure_openssh +) + +:: Function to Install OpenSSH manually if dism fails +:install_openssh_manual +echo "Installing OpenSSH manually..." + +:: Download the latest OpenSSH release +set DOWNLOAD_URL=https://github.com/PowerShell/Win32-OpenSSH/releases/download/v9.5.0.0p1-Beta/OpenSSH-Win64.zip +set DOWNLOAD_FILE=%temp%\OpenSSH-Win64.zip +set INSTALL_DIR=C:\Program Files\OpenSSH-Win64 + +:: Create the installation directory if it doesn't exist +if not exist "%INSTALL_DIR%" mkdir "%INSTALL_DIR%" + +:: Attempt to download OpenSSH using Invoke-WebRequest and TLS configuration +powershell -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; try { Invoke-WebRequest -Uri '%DOWNLOAD_URL%' -OutFile '%DOWNLOAD_FILE%' -UseBasicParsing; exit 0 } catch { exit 1 }" +if %errorlevel% neq 0 ( + echo "Invoke-WebRequest download failed. Attempting to download using curl..." + curl -L -o "%DOWNLOAD_FILE%" "%DOWNLOAD_URL%" + if %errorlevel% neq 0 ( + echo "Failed to download OpenSSH using curl. Exiting..." + exit /b 1 + ) +) + +:: Unzip directly to INSTALL_DIR (flatten the folder structure) +powershell -Command "Expand-Archive -Path '%DOWNLOAD_FILE%' -DestinationPath '%INSTALL_DIR%' -Force" +if %errorlevel% neq 0 ( + echo "Failed to unzip OpenSSH package." + exit /b 1 +) + +:: Move inner contents to INSTALL_DIR if nested OpenSSH-Win64 folder exists +if exist "%INSTALL_DIR%\OpenSSH-Win64" ( + xcopy "%INSTALL_DIR%\OpenSSH-Win64\*" "%INSTALL_DIR%\" /s /e /y + rmdir "%INSTALL_DIR%\OpenSSH-Win64" /s /q +) + +:: Add the OpenSSH binaries to the system PATH +setx /M PATH "%INSTALL_DIR%;%PATH%" + +:: Register OpenSSH utilities as services using PowerShell +powershell -ExecutionPolicy Bypass -Command "& '%INSTALL_DIR%\install-sshd.ps1'" + +:: Verify if manual installation succeeded +sc query sshd >nul 2>&1 +if %errorlevel% neq 0 ( + echo "Manual OpenSSH installation failed. Exiting..." + exit /b 1 +) else ( + echo "OpenSSH installed successfully manually." + goto :configure_openssh +) + +:configure_openssh +:: Ensure OpenSSH Server service is set to start automatically and start the service +sc config sshd start= auto +net start sshd + +:: Create msys2.bat file with specific content +set MSYS2_PATH=C:\msys64 +if not exist "%MSYS2_PATH%" ( + echo "Error: %MSYS2_PATH% does not exist." + exit /b 1 +) + +echo @%MSYS2_PATH%\msys2_shell.cmd -defterm -here -no-start -msys > %MSYS2_PATH%\msys2.bat + +:: Run PowerShell command to set default shell +powershell -Command "New-ItemProperty -Path 'HKLM:\SOFTWARE\OpenSSH' -Name 'DefaultShell' -Value '%MSYS2_PATH%\msys2.bat' -PropertyType String -Force" + +:: Open the Windows Firewall for sshd (using PowerShell) +powershell -Command "New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd) Incoming' -Description 'Allow incoming SSH traffic via OpenSSH server' -Enabled True -Direction Inbound -Protocol TCP -LocalPort 22 -Action Allow" + +echo "OpenSSH has been successfully configured with MSYS2 as the default shell, and the firewall has been opened for sshd." +pause diff --git a/src/aclk/aclk.c b/src/aclk/aclk.c index 6d583a76a6c8b0..991745491cccf5 100644 --- a/src/aclk/aclk.c +++ b/src/aclk/aclk.c @@ -817,10 +817,6 @@ void *aclk_main(void *ptr) unsigned int proto_hdl_cnt = aclk_init_rx_msg_handlers(); - // This thread is unusual in that it cannot be cancelled by cancel_main_threads() - // as it must notify the far end that it shutdown gracefully and avoid the LWT. - netdata_thread_disable_cancelability(); - #if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK ) nd_log(NDLS_DAEMON, NDLP_INFO, "Killing ACLK thread -> cloud functionality has been disabled"); @@ -861,13 +857,10 @@ void *aclk_main(void *ptr) aclk_stats_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "statistics", global_statistics_enabled); if (aclk_stats_enabled) { stats_thread = callocz(1, sizeof(struct aclk_stats_thread)); - stats_thread->thread = mallocz(sizeof(netdata_thread_t)); stats_thread->query_thread_count = query_threads.count; stats_thread->client = mqttwss_client; aclk_stats_thread_prepare(query_threads.count, proto_hdl_cnt); - netdata_thread_create( - stats_thread->thread, "ACLK_STATS", NETDATA_THREAD_OPTION_JOINABLE, aclk_stats_main_thread, - stats_thread); + stats_thread->thread = nd_thread_create("ACLK_STATS", NETDATA_THREAD_OPTION_JOINABLE, aclk_stats_main_thread, stats_thread); } // Keep reconnecting and talking until our time has come @@ -901,9 +894,8 @@ void *aclk_main(void *ptr) aclk_query_threads_cleanup(&query_threads); if (aclk_stats_enabled) { - netdata_thread_join(*stats_thread->thread, NULL); + nd_thread_join(stats_thread->thread); aclk_stats_thread_cleanup(); - freez(stats_thread->thread); freez(stats_thread); } free_topic_cache(); @@ -919,7 +911,7 @@ void *aclk_main(void *ptr) void aclk_host_state_update(RRDHOST *host, int cmd, int queryable) { - uuid_t node_id; + nd_uuid_t node_id; int ret = 0; if (!aclk_connected) @@ -1165,7 +1157,7 @@ char *aclk_state(void) buffer_strcat(wb, "\n\tAlert Streaming Status:"); fill_alert_status_for_host(wb, host); } - rrd_unlock(); + rrd_rdunlock(); } ret = strdupz(buffer_tostring(wb)); @@ -1315,7 +1307,7 @@ char *aclk_state_json(void) json_object_array_add(grp, nodeinstance); } - rrd_unlock(); + rrd_rdunlock(); json_object_object_add(msg, "node-instances", grp); char *str = strdupz(json_object_to_json_string_ext(msg, JSON_C_TO_STRING_PLAIN)); diff --git a/src/aclk/aclk_query.c b/src/aclk/aclk_query.c index ae8435db7301f4..08bc2acf3713a6 100644 --- a/src/aclk/aclk_query.c +++ b/src/aclk/aclk_query.c @@ -351,8 +351,11 @@ void aclk_query_threads_start(struct aclk_query_threads *query_threads, mqtt_wss if(unlikely(snprintfz(thread_name, TASK_LEN_MAX, "ACLK_QRY[%d]", i) < 0)) netdata_log_error("snprintf encoding error"); - netdata_thread_create( - &query_threads->thread_list[i].thread, thread_name, NETDATA_THREAD_OPTION_JOINABLE, aclk_query_main_thread, + + query_threads->thread_list[i].thread = nd_thread_create( + thread_name, + NETDATA_THREAD_OPTION_JOINABLE, + aclk_query_main_thread, &query_threads->thread_list[i]); } } @@ -361,7 +364,7 @@ void aclk_query_threads_cleanup(struct aclk_query_threads *query_threads) { if (query_threads && query_threads->thread_list) { for (int i = 0; i < query_threads->count; i++) { - netdata_thread_join(query_threads->thread_list[i].thread, NULL); + nd_thread_join(query_threads->thread_list[i].thread); } freez(query_threads->thread_list); } diff --git a/src/aclk/aclk_query.h b/src/aclk/aclk_query.h index 371b8d0a24f7e4..900583237de4e7 100644 --- a/src/aclk/aclk_query.h +++ b/src/aclk/aclk_query.h @@ -18,7 +18,7 @@ extern pthread_mutex_t query_lock_wait; //extern volatile int aclk_connected; struct aclk_query_thread { - netdata_thread_t thread; + ND_THREAD *thread; int idx; mqtt_wss_client client; }; diff --git a/src/aclk/aclk_rx_msgs.c b/src/aclk/aclk_rx_msgs.c index fb6a2b902f4063..60e421928e9b40 100644 --- a/src/aclk/aclk_rx_msgs.c +++ b/src/aclk/aclk_rx_msgs.c @@ -255,7 +255,7 @@ int create_node_instance_result(const char *msg, size_t msg_len) netdata_log_debug(D_ACLK, "CreateNodeInstanceResult: guid:%s nodeid:%s", res.machine_guid, res.node_id); - uuid_t host_id, node_id; + nd_uuid_t host_id, node_id; if (uuid_parse(res.machine_guid, host_id)) { netdata_log_error("Error parsing machine_guid provided by CreateNodeInstanceResult"); freez(res.machine_guid); diff --git a/src/aclk/aclk_stats.c b/src/aclk/aclk_stats.c index 71c56f604a0f95..47a48c366be8ec 100644 --- a/src/aclk/aclk_stats.c +++ b/src/aclk/aclk_stats.c @@ -387,11 +387,12 @@ void *aclk_stats_main_thread(void *ptr) struct aclk_metrics permanent; while (service_running(SERVICE_ACLK | SERVICE_COLLECTORS)) { - netdata_thread_testcancel(); + // ------------------------------------------------------------------------ // Wait for the next iteration point. heartbeat_next(&hb, step_ut); + if (!service_running(SERVICE_ACLK | SERVICE_COLLECTORS)) break; ACLK_STATS_LOCK; diff --git a/src/aclk/aclk_stats.h b/src/aclk/aclk_stats.h index 758347ae26c59c..e132695571d7c5 100644 --- a/src/aclk/aclk_stats.h +++ b/src/aclk/aclk_stats.h @@ -19,7 +19,7 @@ extern netdata_mutex_t aclk_stats_mutex; int aclk_cloud_req_http_type_to_idx(const char *name); struct aclk_stats_thread { - netdata_thread_t *thread; + ND_THREAD *thread; int query_thread_count; mqtt_wss_client client; }; diff --git a/src/aclk/aclk_tx_msgs.c b/src/aclk/aclk_tx_msgs.c index 0e4182a7291011..c1ed68052c6641 100644 --- a/src/aclk/aclk_tx_msgs.c +++ b/src/aclk/aclk_tx_msgs.c @@ -101,7 +101,7 @@ static int aclk_send_message_with_bin_payload(mqtt_wss_client client, json_objec */ static struct json_object *create_hdr(const char *type, const char *msg_id, time_t ts_secs, usec_t ts_us, int version) { - uuid_t uuid; + nd_uuid_t uuid; char uuid_str[36 + 1]; json_object *tmp; json_object *obj = json_object_new_object(); diff --git a/src/claim/claim.c b/src/claim/claim.c index 9b67a6a125dea1..5f4ec9a433febc 100644 --- a/src/claim/claim.c +++ b/src/claim/claim.c @@ -150,7 +150,7 @@ void load_claiming_state(void) #if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK ) netdata_cloud_enabled = false; #else - uuid_t uuid; + nd_uuid_t uuid; // Propagate into aclk and registry. Be kind of atomic... appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL); @@ -240,7 +240,7 @@ void load_cloud_conf(int silent) } static char *netdata_random_session_id_filename = NULL; -static uuid_t netdata_random_session_id = { 0 }; +static nd_uuid_t netdata_random_session_id = { 0 }; bool netdata_random_session_id_generate(void) { static char guid[UUID_STR_LEN] = ""; @@ -291,7 +291,7 @@ bool netdata_random_session_id_matches(const char *guid) { if(uuid_is_null(netdata_random_session_id)) return false; - uuid_t uuid; + nd_uuid_t uuid; if(uuid_parse(guid, uuid)) return false; @@ -306,7 +306,7 @@ static bool check_claim_param(const char *s) { if(!s || !*s) return true; do { - if(isalnum(*s) || *s == '.' || *s == ',' || *s == '-' || *s == ':' || *s == '/' || *s == '_') + if(isalnum((uint8_t)*s) || *s == '.' || *s == ',' || *s == '-' || *s == ':' || *s == '/' || *s == '_') ; else return false; @@ -393,7 +393,7 @@ int api_v2_claim(struct web_client *w, char *url) { appconfig_set_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", CONFIG_BOOLEAN_AUTO); appconfig_set(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", base_url); - uuid_t claimed_id; + nd_uuid_t claimed_id; uuid_generate_random(claimed_id); char claimed_id_str[UUID_STR_LEN]; uuid_unparse_lower(claimed_id, claimed_id_str); diff --git a/src/collectors/all.h b/src/collectors/all.h index ffbd075b97cb12..c1491e8474e1ba 100644 --- a/src/collectors/all.h +++ b/src/collectors/all.h @@ -286,6 +286,7 @@ #define NETDATA_CHART_PRIO_IPV4_BCAST_PACKETS 5105 #define NETDATA_CHART_PRIO_IPV4_MCAST 5150 #define NETDATA_CHART_PRIO_IPV4_MCAST_PACKETS 5155 +#define NETDATA_CHART_PRIO_IPV4_TCP_PACKETS 5170 #define NETDATA_CHART_PRIO_IPV4_TCP_SOCKETS 5180 #define NETDATA_CHART_PRIO_IPV4_TCP_SOCKETS_MEM 5185 #define NETDATA_CHART_PRIO_IPV4_ICMP_PACKETS 5200 @@ -311,7 +312,9 @@ #define NETDATA_CHART_PRIO_IPV6_BCAST 6050 #define NETDATA_CHART_PRIO_IPV6_MCAST 6100 #define NETDATA_CHART_PRIO_IPV6_MCAST_PACKETS 6105 +#define NETDATA_CHART_PRIO_IPV6_TCP_PACKETS 6130 #define NETDATA_CHART_PRIO_IPV6_TCP_SOCKETS 6140 +#define NETDATA_CHART_PRIO_IPV6_ICMP_PACKETS 6145 #define NETDATA_CHART_PRIO_IPV6_ICMP 6150 #define NETDATA_CHART_PRIO_IPV6_ICMP_REDIR 6155 #define NETDATA_CHART_PRIO_IPV6_ICMP_ERRORS 6160 diff --git a/src/collectors/apps.plugin/apps_plugin.c b/src/collectors/apps.plugin/apps_plugin.c index a5914b52b6aac1..b64691f8b76122 100644 --- a/src/collectors/apps.plugin/apps_plugin.c +++ b/src/collectors/apps.plugin/apps_plugin.c @@ -552,7 +552,7 @@ static void normalize_utilization(struct target *root) { // here we try to eliminate them by disabling childs processing either for specific dimensions // or entirely. Of course, either way, we disable it just a single iteration. - kernel_uint_t max_time = get_system_cpus() * time_factor * RATES_DETAIL; + kernel_uint_t max_time = os_get_system_cpus() * time_factor * RATES_DETAIL; kernel_uint_t utime = 0, cutime = 0, stime = 0, cstime = 0, gtime = 0, cgtime = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0; if(global_utime > max_time) global_utime = max_time; @@ -1013,7 +1013,7 @@ int main(int argc, char **argv) { procfile_adaptive_initial_allocation = 1; - get_system_HZ(); + os_get_system_HZ(); #if defined(__FreeBSD__) time_factor = 1000000ULL / RATES_DETAIL; // FreeBSD uses usecs #endif @@ -1025,8 +1025,8 @@ int main(int argc, char **argv) { time_factor = system_hz; // Linux uses clock ticks #endif - get_system_pid_max(); - get_system_cpus_uncached(); + os_get_system_pid_max(); + os_get_system_cpus_uncached(); parse_args(argc, argv); diff --git a/src/collectors/cgroups.plugin/cgroup-discovery.c b/src/collectors/cgroups.plugin/cgroup-discovery.c index e5d029cfb0a6d2..5003e8bbd7d5bf 100644 --- a/src/collectors/cgroups.plugin/cgroup-discovery.c +++ b/src/collectors/cgroups.plugin/cgroup-discovery.c @@ -1261,7 +1261,7 @@ static inline void discovery_find_all_cgroups() { discovery_find_all_cgroups_v2(); } - for (struct cgroup *cg = discovered_cgroup_root; cg; cg = cg->discovered_next) { + for (struct cgroup *cg = discovered_cgroup_root; cg && service_running(SERVICE_COLLECTORS); cg = cg->discovered_next) { worker_is_busy(WORKER_DISCOVERY_PROCESS); discovery_process_cgroup(cg); } @@ -1289,6 +1289,7 @@ static inline void discovery_find_all_cgroups() { void cgroup_discovery_worker(void *ptr) { UNUSED(ptr); + uv_thread_set_name_np("P[cgroupsdisc]"); worker_register("CGROUPSDISC"); worker_register_job_name(WORKER_DISCOVERY_INIT, "init"); diff --git a/src/collectors/cgroups.plugin/cgroup-network.c b/src/collectors/cgroups.plugin/cgroup-network.c index 085a6aa6f19604..6a42ec6dc4f91c 100644 --- a/src/collectors/cgroups.plugin/cgroup-network.c +++ b/src/collectors/cgroups.plugin/cgroup-network.c @@ -3,12 +3,6 @@ #include "libnetdata/libnetdata.h" #include "libnetdata/required_dummies.h" -#ifdef HAVE_SETNS -#ifndef _GNU_SOURCE -#define _GNU_SOURCE /* See feature_test_macros(7) */ -#endif -#endif - char env_netdata_host_prefix[FILENAME_MAX + 50] = ""; char env_netdata_log_method[FILENAME_MAX + 50] = ""; char env_netdata_log_format[FILENAME_MAX + 50] = ""; diff --git a/src/collectors/cgroups.plugin/sys_fs_cgroup.c b/src/collectors/cgroups.plugin/sys_fs_cgroup.c index fd30b4acc40c89..2034eab6d42af2 100644 --- a/src/collectors/cgroups.plugin/sys_fs_cgroup.c +++ b/src/collectors/cgroups.plugin/sys_fs_cgroup.c @@ -1158,7 +1158,7 @@ static inline void update_cpu_limits(char **filename, unsigned long long *value, int ret = -1; if(value == &cg->cpuset_cpus) { - unsigned long ncpus = read_cpuset_cpus(*filename, get_system_cpus()); + unsigned long ncpus = os_read_cpuset_cpus(*filename, os_get_system_cpus()); if(ncpus) { *value = ncpus; ret = 0; @@ -1199,7 +1199,7 @@ static inline void update_cpu_limits2(struct cgroup *cg) { } cg->cpu_cfs_period = str2ull(procfile_lineword(ff, 0, 1), NULL); - cg->cpuset_cpus = get_system_cpus(); + cg->cpuset_cpus = os_get_system_cpus(); char *s = "max\n\0"; if(strcmp(s, procfile_lineword(ff, 0, 0)) == 0){ @@ -1513,13 +1513,14 @@ void update_cgroup_charts() { // ---------------------------------------------------------------------------- // cgroups main -static void cgroup_main_cleanup(void *ptr) { - worker_unregister(); +static void cgroup_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); + worker_unregister(); usec_t max = 2 * USEC_PER_SEC, step = 50000; @@ -1554,13 +1555,13 @@ void cgroup_read_host_total_ram() { } void *cgroups_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(cgroup_main_cleanup) cleanup_ptr = ptr; + worker_register("CGROUPS"); worker_register_job_name(WORKER_CGROUPS_LOCK, "lock"); worker_register_job_name(WORKER_CGROUPS_READ, "read"); worker_register_job_name(WORKER_CGROUPS_CHART, "chart"); - netdata_thread_cleanup_push(cgroup_main_cleanup, ptr); - if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL) { is_inside_k8s = 1; cgroup_enable_cpuacct_cpu_shares = CONFIG_BOOLEAN_YES; @@ -1592,8 +1593,6 @@ void *cgroups_main(void *ptr) { goto exit; } - uv_thread_set_name_np(discovery_thread.thread, "P[cgroups]"); - // we register this only on localhost // for the other nodes, the origin server should register it cgroup_netdev_link_init(); @@ -1613,12 +1612,11 @@ void *cgroups_main(void *ptr) { usec_t step = cgroup_update_every * USEC_PER_SEC; usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0; - netdata_thread_disable_cancelability(); - while(service_running(SERVICE_COLLECTORS)) { worker_is_idle(); usec_t hb_dt = heartbeat_next(&hb, step); + if (unlikely(!service_running(SERVICE_COLLECTORS))) break; @@ -1658,6 +1656,5 @@ void *cgroups_main(void *ptr) { } exit: - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/common-contexts/common-contexts.h b/src/collectors/common-contexts/common-contexts.h new file mode 100644 index 00000000000000..92e714ed61d7ea --- /dev/null +++ b/src/collectors/common-contexts/common-contexts.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_COMMON_CONTEXTS_H +#define NETDATA_COMMON_CONTEXTS_H + +#include "../../libnetdata/libnetdata.h" +#include "../../database/rrd.h" + +#ifndef _COMMON_PLUGIN_NAME +#error You need to set _COMMON_PLUGIN_NAME before including common-contexts.h +#endif + +#ifndef _COMMON_PLUGIN_MODULE_NAME +#error You need to set _COMMON_PLUGIN_MODULE_NAME before including common-contexts.h +#endif + +#define _COMMON_CONFIG_SECTION "plugin:" _COMMON_PLUGIN_NAME ":" _COMMON_PLUGIN_MODULE_NAME + +typedef void (*instance_labels_cb_t)(RRDSET *st, void *data); + +#include "system.io.h" +#include "system.ram.h" +#include "mem.swap.h" +#include "mem.pgfaults.h" +#include "mem.available.h" +#include "disk.io.h" + +#endif //NETDATA_COMMON_CONTEXTS_H diff --git a/src/collectors/common-contexts/disk.io.h b/src/collectors/common-contexts/disk.io.h new file mode 100644 index 00000000000000..26f98b9bea6052 --- /dev/null +++ b/src/collectors/common-contexts/disk.io.h @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_DISK_IO_H +#define NETDATA_DISK_IO_H + +#include "common-contexts.h" + +typedef struct { + RRDSET *st_io; + RRDDIM *rd_io_reads; + RRDDIM *rd_io_writes; +} ND_DISK_IO; + +static inline void common_disk_io(ND_DISK_IO *d, const char *id, const char *name, uint64_t bytes_read, uint64_t bytes_write, int update_every, instance_labels_cb_t cb, void *data) { + if(unlikely(!d->st_io)) { + d->st_io = rrdset_create_localhost( + "disk" + , id + , name + , "io" + , "disk.io" + , "Disk I/O Bandwidth" + , "KiB/s" + , _COMMON_PLUGIN_NAME + , _COMMON_PLUGIN_MODULE_NAME + , NETDATA_CHART_PRIO_DISK_IO + , update_every + , RRDSET_TYPE_AREA + ); + + d->rd_io_reads = rrddim_add(d->st_io, "reads", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); + d->rd_io_writes = rrddim_add(d->st_io, "writes", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); + + if(cb) + cb(d->st_io, data); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(d->st_io, d->rd_io_reads, (collected_number)bytes_read); + rrddim_set_by_pointer(d->st_io, d->rd_io_writes, (collected_number)bytes_write); + rrdset_done(d->st_io); +} + +#endif //NETDATA_DISK_IO_H diff --git a/src/collectors/common-contexts/mem.available.h b/src/collectors/common-contexts/mem.available.h new file mode 100644 index 00000000000000..3f763fe18a2044 --- /dev/null +++ b/src/collectors/common-contexts/mem.available.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_MEM_AVAILABLE_H +#define NETDATA_MEM_AVAILABLE_H +#include "common-contexts.h" + +static inline void common_mem_available(uint64_t available_bytes, int update_every) { + static RRDSET *st_mem_available = NULL; + static RRDDIM *rd_avail = NULL; + + if(unlikely(!st_mem_available)) { + st_mem_available = rrdset_create_localhost( + "mem" + , "available" + , NULL + , "overview" + , NULL + , "Available RAM for applications" + , "MiB" + , _COMMON_PLUGIN_NAME + , _COMMON_PLUGIN_MODULE_NAME + , NETDATA_CHART_PRIO_MEM_SYSTEM_AVAILABLE + , update_every + , RRDSET_TYPE_AREA + ); + + rd_avail = rrddim_add(st_mem_available, "avail", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(st_mem_available, rd_avail, (collected_number)available_bytes); + rrdset_done(st_mem_available); +} + +#endif //NETDATA_MEM_AVAILABLE_H diff --git a/src/collectors/common-contexts/mem.pgfaults.h b/src/collectors/common-contexts/mem.pgfaults.h new file mode 100644 index 00000000000000..503b9f7e81d7a3 --- /dev/null +++ b/src/collectors/common-contexts/mem.pgfaults.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_MEM_PGFAULTS_H +#define NETDATA_MEM_PGFAULTS_H + +#include "common-contexts.h" + +static inline void common_mem_pgfaults(uint64_t minor, uint64_t major, int update_every) { + static RRDSET *st_pgfaults = NULL; + static RRDDIM *rd_minor = NULL, *rd_major = NULL; + + if(unlikely(!st_pgfaults)) { + st_pgfaults = rrdset_create_localhost( + "mem" + , "pgfaults" + , NULL + , "page faults" + , NULL + , "Memory Page Faults" + , "faults/s" + , _COMMON_PLUGIN_NAME + , _COMMON_PLUGIN_MODULE_NAME + , NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS + , update_every + , RRDSET_TYPE_LINE + ); + + rrdset_flag_set(st_pgfaults, RRDSET_FLAG_DETAIL); + + rd_minor = rrddim_add(st_pgfaults, "minor", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_major = rrddim_add(st_pgfaults, "major", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(st_pgfaults, rd_minor, minor); + rrddim_set_by_pointer(st_pgfaults, rd_major, major); + rrdset_done(st_pgfaults); +} + +#endif //NETDATA_MEM_PGFAULTS_H diff --git a/src/collectors/common-contexts/mem.swap.h b/src/collectors/common-contexts/mem.swap.h new file mode 100644 index 00000000000000..6d692ef3bd89bc --- /dev/null +++ b/src/collectors/common-contexts/mem.swap.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "common-contexts.h" + +static inline void common_mem_swap(uint64_t free_bytes, uint64_t used_bytes, int update_every) { + static RRDSET *st_system_swap = NULL; + static RRDDIM *rd_free = NULL, *rd_used = NULL; + + if(unlikely(!st_system_swap)) { + st_system_swap = rrdset_create_localhost( + "mem" + , "swap" + , NULL + , "swap" + , NULL + , "System Swap" + , "MiB" + , _COMMON_PLUGIN_NAME + , _COMMON_PLUGIN_MODULE_NAME + , NETDATA_CHART_PRIO_MEM_SWAP + , update_every + , RRDSET_TYPE_STACKED + ); + + rrdset_flag_set(st_system_swap, RRDSET_FLAG_DETAIL); + + rd_free = rrddim_add(st_system_swap, "free", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_used = rrddim_add(st_system_swap, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(st_system_swap, rd_used, (collected_number)used_bytes); + rrddim_set_by_pointer(st_system_swap, rd_free, (collected_number)free_bytes); + rrdset_done(st_system_swap); +} diff --git a/src/collectors/common-contexts/system.io.h b/src/collectors/common-contexts/system.io.h new file mode 100644 index 00000000000000..84440c9b8479c6 --- /dev/null +++ b/src/collectors/common-contexts/system.io.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SYSTEM_IO_H +#define NETDATA_SYSTEM_IO_H + +#include "common-contexts.h" + +static inline void common_system_io(uint64_t read_bytes, uint64_t write_bytes, int update_every) { + static RRDSET *st_io = NULL; + static RRDDIM *rd_in = NULL, *rd_out = NULL; + + if(unlikely(!st_io)) { + st_io = rrdset_create_localhost( + "system" + , "io" + , NULL + , "disk" + , NULL + , "Disk I/O" + , "KiB/s" + , _COMMON_PLUGIN_NAME + , _COMMON_PLUGIN_MODULE_NAME + , NETDATA_CHART_PRIO_SYSTEM_IO + , update_every + , RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st_io, "in", "reads", 1, 1024, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_io, "out", "writes", -1, 1024, RRD_ALGORITHM_INCREMENTAL); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(st_io, rd_in, (collected_number)read_bytes); + rrddim_set_by_pointer(st_io, rd_out, (collected_number)write_bytes); + rrdset_done(st_io); +} + +#endif //NETDATA_SYSTEM_IO_H diff --git a/src/collectors/common-contexts/system.ram.h b/src/collectors/common-contexts/system.ram.h new file mode 100644 index 00000000000000..f255ce04006eef --- /dev/null +++ b/src/collectors/common-contexts/system.ram.h @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SYSTEM_RAM_H +#define NETDATA_SYSTEM_RAM_H + +#include "common-contexts.h" + +#define _system_ram_chart() \ + rrdset_create_localhost( \ + "system" \ + , "ram" \ + , NULL \ + , "ram" \ + , NULL \ + , "System RAM" \ + , "MiB" \ + , _COMMON_PLUGIN_NAME \ + , _COMMON_PLUGIN_MODULE_NAME \ + , NETDATA_CHART_PRIO_SYSTEM_RAM \ + , update_every \ + , RRDSET_TYPE_STACKED \ + ) + +#if defined(COMPILED_FOR_WINDOWS) +static inline void common_system_ram(uint64_t free_bytes, uint64_t used_bytes, int update_every) { + static RRDSET *st_system_ram = NULL; + static RRDDIM *rd_free = NULL; + static RRDDIM *rd_used = NULL; + + if(unlikely(!st_system_ram)) { + st_system_ram = _system_ram_chart(); + rd_free = rrddim_add(st_system_ram, "free", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_used = rrddim_add(st_system_ram, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(st_system_ram, rd_free, (collected_number)free_bytes); + rrddim_set_by_pointer(st_system_ram, rd_used, (collected_number)used_bytes); + rrdset_done(st_system_ram); +} +#endif + +#if defined(COMPILED_FOR_LINUX) +static inline void common_system_ram(uint64_t free_bytes, uint64_t used_bytes, uint64_t cached_bytes, uint64_t buffers_bytes, int update_every) { + static RRDSET *st_system_ram = NULL; + static RRDDIM *rd_free = NULL; + static RRDDIM *rd_used = NULL; + static RRDDIM *rd_cached = NULL; + static RRDDIM *rd_buffers = NULL; + + if(unlikely(!st_system_ram)) { + st_system_ram = _system_ram_chart(); + rd_free = rrddim_add(st_system_ram, "free", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_used = rrddim_add(st_system_ram, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_cached = rrddim_add(st_system_ram, "cached", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_buffers = rrddim_add(st_system_ram, "buffers", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + } + + // this always have to be in base units, so that exporting sends base units to other time-series db + rrddim_set_by_pointer(st_system_ram, rd_free, (collected_number)free_bytes); + rrddim_set_by_pointer(st_system_ram, rd_used, (collected_number)used_bytes); + rrddim_set_by_pointer(st_system_ram, rd_cached, (collected_number)cached_bytes); + rrddim_set_by_pointer(st_system_ram, rd_buffers, (collected_number)buffers_bytes); + rrdset_done(st_system_ram); +} +#endif + +#endif //NETDATA_SYSTEM_RAM_H diff --git a/src/collectors/diskspace.plugin/plugin_diskspace.c b/src/collectors/diskspace.plugin/plugin_diskspace.c index d93e743ae59208..5180120c4a945d 100644 --- a/src/collectors/diskspace.plugin/plugin_diskspace.c +++ b/src/collectors/diskspace.plugin/plugin_diskspace.c @@ -14,7 +14,7 @@ #define MAX_STAT_USEC 10000LU #define SLOW_UPDATE_EVERY 5 -static netdata_thread_t *diskspace_slow_thread = NULL; +static ND_THREAD *diskspace_slow_thread = NULL; static struct mountinfo *disk_mountinfo_root = NULL; static int check_for_new_mountpoints_every = 15; @@ -513,9 +513,9 @@ static inline void do_slow_disk_space_stats(struct basic_mountinfo *mi, int upda dictionary_acquired_item_release(dict_mountpoints, item); } -static void diskspace_slow_worker_cleanup(void *ptr) -{ - UNUSED(ptr); +static void diskspace_slow_worker_cleanup(void *pptr) { + struct slow_worker_data *data = CLEANUP_FUNCTION_GET_PTR(pptr); + if(data) return; collector_info("cleaning up..."); @@ -526,14 +526,14 @@ static void diskspace_slow_worker_cleanup(void *ptr) #define WORKER_JOB_SLOW_CLEANUP 1 struct slow_worker_data { - netdata_thread_t *slow_thread; int update_every; }; void *diskspace_slow_worker(void *ptr) { struct slow_worker_data *data = (struct slow_worker_data *)ptr; - + CLEANUP_FUNCTION_REGISTER(diskspace_slow_worker_cleanup) cleanup_ptr = data; + worker_register("DISKSPACE_SLOW"); worker_register_job_name(WORKER_JOB_SLOW_MOUNTPOINT, "mountpoint"); worker_register_job_name(WORKER_JOB_SLOW_CLEANUP, "cleanup"); @@ -542,8 +542,6 @@ void *diskspace_slow_worker(void *ptr) int slow_update_every = data->update_every > SLOW_UPDATE_EVERY ? data->update_every : SLOW_UPDATE_EVERY; - netdata_thread_cleanup_push(diskspace_slow_worker_cleanup, data->slow_thread); - usec_t step = slow_update_every * USEC_PER_SEC; usec_t real_step = USEC_PER_SEC; heartbeat_t hb; @@ -600,26 +598,24 @@ void *diskspace_slow_worker(void *ptr) } } - netdata_thread_cleanup_pop(1); - free_basic_mountinfo_list(slow_mountinfo_root); return NULL; } -static void diskspace_main_cleanup(void *ptr) { - rrd_collector_finished(); - worker_unregister(); +static void diskspace_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); - if (diskspace_slow_thread) { - netdata_thread_join(*diskspace_slow_thread, NULL); - freez(diskspace_slow_thread); - } + rrd_collector_finished(); + worker_unregister(); + + if (diskspace_slow_thread) + nd_thread_join(diskspace_slow_thread); free_basic_mountinfo_list(slow_mountinfo_tmp_root); @@ -846,6 +842,8 @@ int diskspace_function_mount_points(BUFFER *wb, const char *function __maybe_unu } void *diskspace_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(diskspace_main_cleanup) cleanup_ptr = ptr; + worker_register("DISKSPACE"); worker_register_job_name(WORKER_JOB_MOUNTINFO, "mountinfo"); worker_register_job_name(WORKER_JOB_MOUNTPOINT, "mountpoint"); @@ -856,8 +854,6 @@ void *diskspace_main(void *ptr) { "top", HTTP_ACCESS_ANONYMOUS_DATA, diskspace_function_mount_points); - netdata_thread_cleanup_push(diskspace_main_cleanup, ptr); - cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points); int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", localhost->rrd_update_every); @@ -870,12 +866,9 @@ void *diskspace_main(void *ptr) { netdata_mutex_init(&slow_mountinfo_mutex); - diskspace_slow_thread = mallocz(sizeof(netdata_thread_t)); - - struct slow_worker_data slow_worker_data = {.slow_thread = diskspace_slow_thread, .update_every = update_every}; + struct slow_worker_data slow_worker_data = { .update_every = update_every }; - netdata_thread_create( - diskspace_slow_thread, + diskspace_slow_thread = nd_thread_create( "P[diskspace slow]", NETDATA_THREAD_OPTION_JOINABLE, diskspace_slow_worker, @@ -926,8 +919,5 @@ void *diskspace_main(void *ptr) { mount_points_cleanup(false); } } - worker_unregister(); - - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf.c b/src/collectors/ebpf.plugin/ebpf.c index 4539c7e629f0a2..92af75f1009a38 100644 --- a/src/collectors/ebpf.plugin/ebpf.c +++ b/src/collectors/ebpf.plugin/ebpf.c @@ -927,7 +927,7 @@ void ebpf_stop_threads(int sig) // Child thread should be closed by itself. pthread_mutex_lock(&ebpf_exit_cleanup); - if (main_thread_id != gettid() || only_one) { + if (main_thread_id != gettid_cached() || only_one) { pthread_mutex_unlock(&ebpf_exit_cleanup); return; } @@ -935,7 +935,7 @@ void ebpf_stop_threads(int sig) int i; for (i = 0; ebpf_modules[i].info.thread_name != NULL; i++) { if (ebpf_modules[i].enabled < NETDATA_THREAD_EBPF_STOPPING) { - netdata_thread_cancel(*ebpf_modules[i].thread->thread); + nd_thread_signal_cancel(ebpf_modules[i].thread->thread); #ifdef NETDATA_DEV_MODE netdata_log_info("Sending cancel for thread %s", ebpf_modules[i].info.thread_name); #endif @@ -945,13 +945,13 @@ void ebpf_stop_threads(int sig) for (i = 0; ebpf_modules[i].info.thread_name != NULL; i++) { if (ebpf_threads[i].thread) - netdata_thread_join(*ebpf_threads[i].thread, NULL); + nd_thread_join(ebpf_threads[i].thread); } ebpf_plugin_exit = true; pthread_mutex_lock(&mutex_cgroup_shm); - netdata_thread_cancel(*cgroup_integration_thread.thread); + nd_thread_signal_cancel(cgroup_integration_thread.thread); #ifdef NETDATA_DEV_MODE netdata_log_info("Sending cancel for thread %s", cgroup_integration_thread.name); #endif @@ -3040,7 +3040,7 @@ void set_global_variables() } isrh = get_redhat_release(); - pid_max = get_system_pid_max(); + pid_max = os_get_system_pid_max(); running_on_kernel = ebpf_get_kernel_version(); } @@ -3974,7 +3974,7 @@ int main(int argc, char **argv) clocks_init(); nd_log_initialize_for_external_plugins(NETDATA_EBPF_PLUGIN_NAME); - main_thread_id = gettid(); + main_thread_id = gettid_cached(); set_global_variables(); ebpf_parse_args(argc, argv); @@ -4010,11 +4010,13 @@ int main(int argc, char **argv) ebpf_set_static_routine(); - cgroup_integration_thread.thread = mallocz(sizeof(netdata_thread_t)); cgroup_integration_thread.start_routine = ebpf_cgroup_integration; - netdata_thread_create(cgroup_integration_thread.thread, cgroup_integration_thread.name, - NETDATA_THREAD_OPTION_DEFAULT, ebpf_cgroup_integration, NULL); + cgroup_integration_thread.thread = nd_thread_create( + cgroup_integration_thread.name, + NETDATA_THREAD_OPTION_DEFAULT, + ebpf_cgroup_integration, + NULL); int i; for (i = 0; ebpf_threads[i].name != NULL; i++) { @@ -4024,10 +4026,9 @@ int main(int argc, char **argv) em->thread = st; em->thread_id = i; if (em->enabled != NETDATA_THREAD_EBPF_NOT_RUNNING) { - st->thread = mallocz(sizeof(netdata_thread_t)); em->enabled = NETDATA_THREAD_EBPF_RUNNING; em->lifetime = EBPF_NON_FUNCTION_LIFE_TIME; - netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_JOINABLE, st->start_routine, em); + st->thread = nd_thread_create(st->name, NETDATA_THREAD_OPTION_JOINABLE, st->start_routine, em); } else { em->lifetime = EBPF_DEFAULT_LIFETIME; } @@ -4041,7 +4042,7 @@ int main(int argc, char **argv) int update_apps_list = update_apps_every - 1; int process_maps_per_core = ebpf_modules[EBPF_MODULE_PROCESS_IDX].maps_per_core; //Plugin will be killed when it receives a signal - for ( ; !ebpf_plugin_exit; global_iterations_counter++) { + for ( ; !ebpf_plugin_stop(); global_iterations_counter++) { (void)heartbeat_next(&hb, step); if (global_iterations_counter % EBPF_DEFAULT_UPDATE_EVERY == 0) { diff --git a/src/collectors/ebpf.plugin/ebpf.h b/src/collectors/ebpf.plugin/ebpf.h index 52e55064256b2d..175c092a1c67a6 100644 --- a/src/collectors/ebpf.plugin/ebpf.h +++ b/src/collectors/ebpf.plugin/ebpf.h @@ -389,6 +389,11 @@ void ebpf_read_local_addresses_unsafe(); extern ebpf_filesystem_partitions_t localfs[]; extern ebpf_sync_syscalls_t local_syscalls[]; extern bool ebpf_plugin_exit; + +static inline bool ebpf_plugin_stop(void) { + return ebpf_plugin_exit || nd_thread_signaled_to_cancel(); +} + void ebpf_stop_threads(int sig); extern netdata_ebpf_judy_pid_t ebpf_judy_pid; diff --git a/src/collectors/ebpf.plugin/ebpf_cachestat.c b/src/collectors/ebpf.plugin/ebpf_cachestat.c index 9bc51e484e4337..7bc0c44a5453ea 100644 --- a/src/collectors/ebpf.plugin/ebpf_cachestat.c +++ b/src/collectors/ebpf.plugin/ebpf_cachestat.c @@ -523,12 +523,13 @@ void ebpf_obsolete_cachestat_apps_charts(struct ebpf_module *em) * * @param ptr thread data. */ -static void ebpf_cachestat_exit(void *ptr) +static void ebpf_cachestat_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (ebpf_read_cachestat.thread) - netdata_thread_cancel(*ebpf_read_cachestat.thread); + nd_thread_signal_cancel(ebpf_read_cachestat.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -840,13 +841,11 @@ void *ebpf_read_cachestat_thread(void *ptr) uint32_t lifetime = em->lifetime; uint32_t running_time = 0; usec_t period = update_every * USEC_PER_SEC; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; - netdata_thread_disable_cancelability(); - pthread_mutex_lock(&collect_data_mutex); ebpf_read_cachestat_apps_table(maps_per_core, max_period); ebpf_resume_apps_data(); @@ -862,7 +861,6 @@ void *ebpf_read_cachestat_thread(void *ptr) em->running_time = running_time; pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_enable_cancelability(); } return NULL; @@ -1402,10 +1400,10 @@ static void cachestat_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -1593,9 +1591,10 @@ static int ebpf_cachestat_load_bpf(ebpf_module_t *em) */ void *ebpf_cachestat_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_cachestat_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_cachestat_exit) cleanup_ptr = em; + em->maps = cachestat_maps; ebpf_update_pid_table(&cachestat_maps[NETDATA_CACHESTAT_PID_STATS], em); @@ -1628,18 +1627,16 @@ void *ebpf_cachestat_thread(void *ptr) pthread_mutex_unlock(&lock); - ebpf_read_cachestat.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_cachestat.thread, - ebpf_read_cachestat.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_cachestat_thread, - em); + ebpf_read_cachestat.thread = nd_thread_create( + ebpf_read_cachestat.name, + NETDATA_THREAD_OPTION_DEFAULT, + ebpf_read_cachestat_thread, + em); cachestat_collector(em); endcachestat: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_cgroup.c b/src/collectors/ebpf.plugin/ebpf_cgroup.c index 7dbb52a2cd2f10..ae3bf3f8af0a49 100644 --- a/src/collectors/ebpf.plugin/ebpf_cgroup.c +++ b/src/collectors/ebpf.plugin/ebpf_cgroup.c @@ -348,18 +348,6 @@ void ebpf_create_charts_on_systemd(ebpf_systemd_args_t *chart) // -------------------------------------------------------------------------------------------------------------------- // Cgroup main thread -/** - * CGROUP exit - * - * Clean up the main thread. - * - * @param ptr thread data. - */ -static void ebpf_cgroup_exit(void *ptr) -{ - UNUSED(ptr); -} - /** * Cgroup integratin * @@ -369,16 +357,14 @@ static void ebpf_cgroup_exit(void *ptr) * * @return It always returns NULL. */ -void *ebpf_cgroup_integration(void *ptr) +void *ebpf_cgroup_integration(void *ptr __maybe_unused) { - netdata_thread_cleanup_push(ebpf_cgroup_exit, ptr); - usec_t step = USEC_PER_SEC; int counter = NETDATA_EBPF_CGROUP_UPDATE - 1; heartbeat_t hb; heartbeat_init(&hb); //Plugin will be killed when it receives a signal - while (!ebpf_plugin_exit) { + while (!ebpf_plugin_stop()) { (void)heartbeat_next(&hb, step); // We are using a small heartbeat time to wake up thread, @@ -392,6 +378,5 @@ void *ebpf_cgroup_integration(void *ptr) } } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_dcstat.c b/src/collectors/ebpf.plugin/ebpf_dcstat.c index dbba1e38c09cf4..be52764004629c 100644 --- a/src/collectors/ebpf.plugin/ebpf_dcstat.c +++ b/src/collectors/ebpf.plugin/ebpf_dcstat.c @@ -451,12 +451,13 @@ static void ebpf_obsolete_dc_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_dcstat_exit(void *ptr) +static void ebpf_dcstat_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (ebpf_read_dcstat.thread) - netdata_thread_cancel(*ebpf_read_dcstat.thread); + nd_thread_signal_cancel(ebpf_read_dcstat.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -641,13 +642,11 @@ void *ebpf_read_dcstat_thread(void *ptr) uint32_t running_time = 0; usec_t period = update_every * USEC_PER_SEC; int max_period = update_every * EBPF_CLEANUP_FACTOR; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; - netdata_thread_disable_cancelability(); - pthread_mutex_lock(&collect_data_mutex); ebpf_read_dc_apps_table(maps_per_core, max_period); ebpf_dc_resume_apps_data(); @@ -663,7 +662,6 @@ void *ebpf_read_dcstat_thread(void *ptr) em->running_time = running_time; pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_enable_cancelability(); } return NULL; @@ -1261,10 +1259,10 @@ static void dcstat_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -1403,9 +1401,9 @@ static int ebpf_dcstat_load_bpf(ebpf_module_t *em) */ void *ebpf_dcstat_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_dcstat_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + CLEANUP_FUNCTION_REGISTER(ebpf_dcstat_exit) cleanup_ptr = em; + em->maps = dcstat_maps; ebpf_update_pid_table(&dcstat_maps[NETDATA_DCSTAT_PID_STATS], em); @@ -1437,18 +1435,13 @@ void *ebpf_dcstat_thread(void *ptr) pthread_mutex_unlock(&lock); - ebpf_read_dcstat.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_dcstat.thread, - ebpf_read_dcstat.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_dcstat_thread, - em); + ebpf_read_dcstat.thread = nd_thread_create(ebpf_read_dcstat.name, NETDATA_THREAD_OPTION_DEFAULT, + ebpf_read_dcstat_thread, em); dcstat_collector(em); enddcstat: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_disk.c b/src/collectors/ebpf.plugin/ebpf_disk.c index 466c2e3bb47b2f..3c30e926b9484b 100644 --- a/src/collectors/ebpf.plugin/ebpf_disk.c +++ b/src/collectors/ebpf.plugin/ebpf_disk.c @@ -506,9 +506,10 @@ static void ebpf_obsolete_disk_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_disk_exit(void *ptr) +static void ebpf_disk_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -779,10 +780,10 @@ static void disk_collector(ebpf_module_t *em) int maps_per_core = em->maps_per_core; uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -890,9 +891,10 @@ static int ebpf_disk_load_bpf(ebpf_module_t *em) */ void *ebpf_disk_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_disk_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_disk_exit) cleanup_ptr = em; + em->maps = disk_maps; if (ebpf_disk_enable_tracepoints()) { @@ -934,7 +936,5 @@ void *ebpf_disk_thread(void *ptr) enddisk: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); - return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_fd.c b/src/collectors/ebpf.plugin/ebpf_fd.c index 0e4165ed1b0f05..bd961396db659a 100644 --- a/src/collectors/ebpf.plugin/ebpf_fd.c +++ b/src/collectors/ebpf.plugin/ebpf_fd.c @@ -546,12 +546,13 @@ static void ebpf_obsolete_fd_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_fd_exit(void *ptr) +static void ebpf_fd_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (ebpf_read_fd.thread) - netdata_thread_cancel(*ebpf_read_fd.thread); + nd_thread_signal_cancel(ebpf_read_fd.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -773,13 +774,11 @@ void *ebpf_read_fd_thread(void *ptr) uint32_t running_time = 0; usec_t period = update_every * USEC_PER_SEC; int max_period = update_every * EBPF_CLEANUP_FACTOR; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; - netdata_thread_disable_cancelability(); - pthread_mutex_lock(&collect_data_mutex); ebpf_read_fd_apps_table(maps_per_core, max_period); ebpf_fd_resume_apps_data(); @@ -795,7 +794,6 @@ void *ebpf_read_fd_thread(void *ptr) em->running_time = running_time; pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_enable_cancelability(); } return NULL; @@ -1205,10 +1203,10 @@ static void fd_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -1439,9 +1437,10 @@ static int ebpf_fd_load_bpf(ebpf_module_t *em) */ void *ebpf_fd_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_fd_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_fd_exit) cleanup_ptr = em; + em->maps = fd_maps; #ifdef LIBBPF_MAJOR_VERSION @@ -1467,18 +1466,12 @@ void *ebpf_fd_thread(void *ptr) pthread_mutex_unlock(&lock); - ebpf_read_fd.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_fd.thread, - ebpf_read_fd.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_fd_thread, - em); + ebpf_read_fd.thread = nd_thread_create(ebpf_read_fd.name, NETDATA_THREAD_OPTION_DEFAULT, ebpf_read_fd_thread, em); fd_collector(em); endfd: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_filesystem.c b/src/collectors/ebpf.plugin/ebpf_filesystem.c index b5c89823219d1f..1861031c440009 100644 --- a/src/collectors/ebpf.plugin/ebpf_filesystem.c +++ b/src/collectors/ebpf.plugin/ebpf_filesystem.c @@ -724,9 +724,10 @@ static void ebpf_obsolete_filesystem_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_filesystem_exit(void *ptr) +static void ebpf_filesystem_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -915,10 +916,10 @@ static void filesystem_collector(ebpf_module_t *em) int counter = update_every - 1; uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -990,9 +991,10 @@ static void ebpf_set_maps() */ void *ebpf_filesystem_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_filesystem_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_filesystem_exit) cleanup_ptr = em; + ebpf_set_maps(); ebpf_update_filesystem(); @@ -1024,6 +1026,5 @@ void *ebpf_filesystem_thread(void *ptr) endfilesystem: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_functions.c b/src/collectors/ebpf.plugin/ebpf_functions.c index 437d1234b26795..4a43bf434c5d17 100644 --- a/src/collectors/ebpf.plugin/ebpf_functions.c +++ b/src/collectors/ebpf.plugin/ebpf_functions.c @@ -20,13 +20,10 @@ static int ebpf_function_start_thread(ebpf_module_t *em, int period) { struct netdata_static_thread *st = em->thread; // another request for thread that already ran, cleanup and restart - if (st->thread) - freez(st->thread); - if (period <= 0) period = EBPF_DEFAULT_LIFETIME; - st->thread = mallocz(sizeof(netdata_thread_t)); + st->thread = NULL; em->enabled = NETDATA_THREAD_EBPF_FUNCTION_RUNNING; em->lifetime = period; @@ -34,7 +31,8 @@ static int ebpf_function_start_thread(ebpf_module_t *em, int period) netdata_log_info("Starting thread %s with lifetime = %d", em->info.thread_name, period); #endif - return netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, em); + st->thread = nd_thread_create(st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, em); + return st->thread ? 0 : 1; } /***************************************************************** @@ -714,10 +712,10 @@ void *ebpf_function_thread(void *ptr) heartbeat_t hb; heartbeat_init(&hb); - while(!ebpf_plugin_exit) { + while(!ebpf_plugin_stop()) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit) { + if (ebpf_plugin_stop()) { break; } } diff --git a/src/collectors/ebpf.plugin/ebpf_hardirq.c b/src/collectors/ebpf.plugin/ebpf_hardirq.c index 465ee6434c59b5..2e20c0a5ab4a52 100644 --- a/src/collectors/ebpf.plugin/ebpf_hardirq.c +++ b/src/collectors/ebpf.plugin/ebpf_hardirq.c @@ -244,9 +244,10 @@ static void ebpf_obsolete_hardirq_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void hardirq_exit(void *ptr) +static void hardirq_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -581,10 +582,10 @@ static void hardirq_collector(ebpf_module_t *em) //This will be cancelled by its parent uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -658,9 +659,10 @@ static int ebpf_hardirq_load_bpf(ebpf_module_t *em) */ void *ebpf_hardirq_thread(void *ptr) { - netdata_thread_cleanup_push(hardirq_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(hardirq_exit) cleanup_ptr = em; + em->maps = hardirq_maps; if (ebpf_enable_tracepoints(hardirq_tracepoints) == 0) { @@ -680,7 +682,5 @@ void *ebpf_hardirq_thread(void *ptr) endhardirq: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); - return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_mdflush.c b/src/collectors/ebpf.plugin/ebpf_mdflush.c index fe33ff6a478f97..9dac4dd50d1e2a 100644 --- a/src/collectors/ebpf.plugin/ebpf_mdflush.c +++ b/src/collectors/ebpf.plugin/ebpf_mdflush.c @@ -157,9 +157,10 @@ static void ebpf_obsolete_mdflush_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void mdflush_exit(void *ptr) +static void mdflush_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -346,10 +347,10 @@ static void mdflush_collector(ebpf_module_t *em) int maps_per_core = em->maps_per_core; uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -424,9 +425,9 @@ static int ebpf_mdflush_load_bpf(ebpf_module_t *em) */ void *ebpf_mdflush_thread(void *ptr) { - netdata_thread_cleanup_push(mdflush_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + CLEANUP_FUNCTION_REGISTER(mdflush_exit) cleanup_ptr = em; + em->maps = mdflush_maps; char *md_flush_request = ebpf_find_symbol("md_flush_request"); @@ -450,7 +451,5 @@ void *ebpf_mdflush_thread(void *ptr) freez(md_flush_request); ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); - return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_mount.c b/src/collectors/ebpf.plugin/ebpf_mount.c index 05c76540a554b2..b2eab079bda1ad 100644 --- a/src/collectors/ebpf.plugin/ebpf_mount.c +++ b/src/collectors/ebpf.plugin/ebpf_mount.c @@ -261,9 +261,10 @@ static void ebpf_obsolete_mount_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_mount_exit(void *ptr) +static void ebpf_mount_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -369,9 +370,9 @@ static void mount_collector(ebpf_module_t *em) int maps_per_core = em->maps_per_core; uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -484,9 +485,9 @@ static int ebpf_mount_load_bpf(ebpf_module_t *em) */ void *ebpf_mount_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_mount_exit, ptr); + ebpf_module_t *em = ptr; + CLEANUP_FUNCTION_REGISTER(ebpf_mount_exit) cleanup_ptr = em; - ebpf_module_t *em = (ebpf_module_t *)ptr; em->maps = mount_maps; #ifdef LIBBPF_MAJOR_VERSION @@ -512,6 +513,5 @@ void *ebpf_mount_thread(void *ptr) endmount: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_oomkill.c b/src/collectors/ebpf.plugin/ebpf_oomkill.c index df0ddd5c537c67..0f79562fe733f1 100644 --- a/src/collectors/ebpf.plugin/ebpf_oomkill.c +++ b/src/collectors/ebpf.plugin/ebpf_oomkill.c @@ -128,9 +128,10 @@ static void ebpf_obsolete_oomkill_apps(ebpf_module_t *em) * * @param ptr thread data. */ -static void oomkill_cleanup(void *ptr) +static void oomkill_cleanup(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -457,9 +458,9 @@ static void oomkill_collector(ebpf_module_t *em) uint32_t running_time = 0; uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -532,9 +533,10 @@ void ebpf_oomkill_create_apps_charts(struct ebpf_module *em, void *ptr) */ void *ebpf_oomkill_thread(void *ptr) { - netdata_thread_cleanup_push(oomkill_cleanup, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(oomkill_cleanup) cleanup_ptr = em; + em->maps = oomkill_maps; #define NETDATA_DEFAULT_OOM_DISABLED_MSG "Disabling OOMKILL thread, because" @@ -578,7 +580,5 @@ void *ebpf_oomkill_thread(void *ptr) endoomkill: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); - return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_process.c b/src/collectors/ebpf.plugin/ebpf_process.c index 695c8c07275c43..3ba6a24f788dd9 100644 --- a/src/collectors/ebpf.plugin/ebpf_process.c +++ b/src/collectors/ebpf.plugin/ebpf_process.c @@ -689,9 +689,10 @@ static void ebpf_process_disable_tracepoints() * * @param ptr thread data. */ -static void ebpf_process_exit(void *ptr) +static void ebpf_process_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -1135,10 +1136,10 @@ static void process_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { usec_t dt = heartbeat_next(&hb, USEC_PER_SEC); (void)dt; - if (ebpf_plugin_exit) + if (ebpf_plugin_stop()) break; if (++counter == update_every) { @@ -1279,9 +1280,10 @@ static int ebpf_process_enable_tracepoints() */ void *ebpf_process_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_process_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_process_exit) cleanup_ptr = em; + em->maps = process_maps; pthread_mutex_lock(&ebpf_exit_cleanup); @@ -1322,6 +1324,5 @@ void *ebpf_process_thread(void *ptr) ebpf_update_disabled_plugin_stats(em); pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_shm.c b/src/collectors/ebpf.plugin/ebpf_shm.c index bffc7b850be0f2..4c5eb6120ae1f6 100644 --- a/src/collectors/ebpf.plugin/ebpf_shm.c +++ b/src/collectors/ebpf.plugin/ebpf_shm.c @@ -448,12 +448,13 @@ static void ebpf_obsolete_shm_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_shm_exit(void *ptr) +static void ebpf_shm_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (ebpf_read_shm.thread) - netdata_thread_cancel(*ebpf_read_shm.thread); + nd_thread_signal_cancel(ebpf_read_shm.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -1066,13 +1067,11 @@ void *ebpf_read_shm_thread(void *ptr) uint32_t running_time = 0; usec_t period = update_every * USEC_PER_SEC; int max_period = update_every * EBPF_CLEANUP_FACTOR; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; - netdata_thread_disable_cancelability(); - pthread_mutex_lock(&collect_data_mutex); ebpf_read_shm_apps_table(maps_per_core, max_period); ebpf_shm_resume_apps_data(); @@ -1088,7 +1087,6 @@ void *ebpf_read_shm_thread(void *ptr) em->running_time = running_time; pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_enable_cancelability(); } return NULL; @@ -1109,9 +1107,9 @@ static void shm_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -1327,9 +1325,10 @@ static int ebpf_shm_load_bpf(ebpf_module_t *em) */ void *ebpf_shm_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_shm_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_shm_exit) cleanup_ptr = em; + em->maps = shm_maps; ebpf_update_pid_table(&shm_maps[NETDATA_PID_SHM_TABLE], em); @@ -1364,18 +1363,12 @@ void *ebpf_shm_thread(void *ptr) ebpf_update_kernel_memory_with_vector(&plugin_statistics, em->maps, EBPF_ACTION_STAT_ADD); pthread_mutex_unlock(&lock); - ebpf_read_shm.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_shm.thread, - ebpf_read_shm.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_shm_thread, - em); + ebpf_read_shm.thread = nd_thread_create(ebpf_read_shm.name, NETDATA_THREAD_OPTION_DEFAULT, ebpf_read_shm_thread, em); shm_collector(em); endshm: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_socket.c b/src/collectors/ebpf.plugin/ebpf_socket.c index 47ec20df22656d..6cd4111f66c83d 100644 --- a/src/collectors/ebpf.plugin/ebpf_socket.c +++ b/src/collectors/ebpf.plugin/ebpf_socket.c @@ -881,12 +881,13 @@ static void ebpf_socket_obsolete_global_charts(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_socket_exit(void *ptr) +static void ebpf_socket_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (ebpf_read_socket.thread) - netdata_thread_cancel(*ebpf_read_socket.thread); + nd_thread_signal_cancel(ebpf_read_socket.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -1678,7 +1679,6 @@ static void ebpf_socket_translate(netdata_socket_plus_t *dst, netdata_socket_idx */ static void ebpf_update_array_vectors(ebpf_module_t *em) { - netdata_thread_disable_cancelability(); netdata_socket_idx_t key = {}; netdata_socket_idx_t next_key = {}; @@ -1776,7 +1776,6 @@ static void ebpf_update_array_vectors(ebpf_module_t *em) memset(values, 0, length); memcpy(&key, &next_key, sizeof(key)); } - netdata_thread_enable_cancelability(); } /** * Resume apps data @@ -1838,9 +1837,9 @@ void *ebpf_read_socket_thread(void *ptr) uint32_t running_time = 0; uint32_t lifetime = em->lifetime; usec_t period = update_every * USEC_PER_SEC; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; pthread_mutex_lock(&collect_data_mutex); @@ -2653,9 +2652,9 @@ static void socket_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -2876,9 +2875,10 @@ static int ebpf_socket_load_bpf(ebpf_module_t *em) */ void *ebpf_socket_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_socket_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_socket_exit) cleanup_ptr = em; + if (em->enabled > NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { collector_error("There is already a thread %s running", em->info.thread_name); return NULL; @@ -2918,12 +2918,8 @@ void *ebpf_socket_thread(void *ptr) socket_aggregated_data, socket_publish_aggregated, socket_dimension_names, socket_id_names, algorithms, NETDATA_MAX_SOCKET_VECTOR); - ebpf_read_socket.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_socket.thread, - ebpf_read_socket.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_socket_thread, - em); + ebpf_read_socket.thread = nd_thread_create(ebpf_read_socket.name, NETDATA_THREAD_OPTION_DEFAULT, + ebpf_read_socket_thread, em); pthread_mutex_lock(&lock); ebpf_socket_create_global_charts(em); @@ -2937,7 +2933,5 @@ void *ebpf_socket_thread(void *ptr) endsocket: ebpf_update_disabled_plugin_stats(em); - - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_softirq.c b/src/collectors/ebpf.plugin/ebpf_softirq.c index 106ff4f291e1f3..6b131ef8adffd7 100644 --- a/src/collectors/ebpf.plugin/ebpf_softirq.c +++ b/src/collectors/ebpf.plugin/ebpf_softirq.c @@ -88,9 +88,10 @@ static void ebpf_obsolete_softirq_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void softirq_cleanup(void *ptr) +static void softirq_cleanup(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -219,9 +220,9 @@ static void softirq_collector(ebpf_module_t *em) //This will be cancelled by its parent uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -258,9 +259,10 @@ static void softirq_collector(ebpf_module_t *em) */ void *ebpf_softirq_thread(void *ptr) { - netdata_thread_cleanup_push(softirq_cleanup, ptr); + ebpf_module_t *em = ptr; + + CLEANUP_FUNCTION_REGISTER(softirq_cleanup) cleanup_ptr = em; - ebpf_module_t *em = (ebpf_module_t *)ptr; em->maps = softirq_maps; if (ebpf_enable_tracepoints(softirq_tracepoints) == 0) { @@ -280,7 +282,5 @@ void *ebpf_softirq_thread(void *ptr) endsoftirq: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); - return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_swap.c b/src/collectors/ebpf.plugin/ebpf_swap.c index b2b5dad347f96b..2b7451b6563663 100644 --- a/src/collectors/ebpf.plugin/ebpf_swap.c +++ b/src/collectors/ebpf.plugin/ebpf_swap.c @@ -394,7 +394,7 @@ static void ebpf_swap_exit(void *ptr) ebpf_module_t *em = (ebpf_module_t *)ptr; if (ebpf_read_swap.thread) - netdata_thread_cancel(*ebpf_read_swap.thread); + nd_thread_signal_cancel(ebpf_read_swap.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -595,13 +595,11 @@ void *ebpf_read_swap_thread(void *ptr) usec_t period = update_every * USEC_PER_SEC; int max_period = update_every * EBPF_CLEANUP_FACTOR; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; - netdata_thread_disable_cancelability(); - pthread_mutex_lock(&collect_data_mutex); ebpf_read_swap_apps_table(maps_per_core, max_period); ebpf_swap_resume_apps_data(); @@ -617,7 +615,6 @@ void *ebpf_read_swap_thread(void *ptr) em->running_time = running_time; pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_enable_cancelability(); } return NULL; @@ -920,9 +917,9 @@ static void swap_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -1131,9 +1128,10 @@ static int ebpf_swap_set_internal_value() */ void *ebpf_swap_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_swap_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_swap_exit) cleanup_ptr = em; + em->maps = swap_maps; ebpf_update_pid_table(&swap_maps[NETDATA_PID_SWAP_TABLE], em); @@ -1161,18 +1159,13 @@ void *ebpf_swap_thread(void *ptr) ebpf_update_kernel_memory_with_vector(&plugin_statistics, em->maps, EBPF_ACTION_STAT_ADD); pthread_mutex_unlock(&lock); - ebpf_read_swap.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_swap.thread, - ebpf_read_swap.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_swap_thread, - em); + ebpf_read_swap.thread = nd_thread_create(ebpf_read_swap.name, NETDATA_THREAD_OPTION_DEFAULT, + ebpf_read_swap_thread, em); swap_collector(em); endswap: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_sync.c b/src/collectors/ebpf.plugin/ebpf_sync.c index a16318107cee52..f7a3b4d2454345 100644 --- a/src/collectors/ebpf.plugin/ebpf_sync.c +++ b/src/collectors/ebpf.plugin/ebpf_sync.c @@ -351,9 +351,10 @@ static void ebpf_obsolete_sync_global(ebpf_module_t *em) * * @param ptr thread data. */ -static void ebpf_sync_exit(void *ptr) +static void ebpf_sync_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -564,9 +565,9 @@ static void sync_collector(ebpf_module_t *em) int maps_per_core = em->maps_per_core; uint32_t running_time = 0; uint32_t lifetime = em->lifetime; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -703,10 +704,10 @@ static void ebpf_set_sync_maps() */ void *ebpf_sync_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_sync_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + CLEANUP_FUNCTION_REGISTER(ebpf_sync_exit) cleanup_ptr = em; + ebpf_set_sync_maps(); ebpf_sync_parse_syscalls(); @@ -734,6 +735,5 @@ void *ebpf_sync_thread(void *ptr) endsync: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/ebpf.plugin/ebpf_vfs.c b/src/collectors/ebpf.plugin/ebpf_vfs.c index c57136f04e7ddf..275cbea5d9d5c9 100644 --- a/src/collectors/ebpf.plugin/ebpf_vfs.c +++ b/src/collectors/ebpf.plugin/ebpf_vfs.c @@ -881,12 +881,13 @@ static void ebpf_obsolete_vfs_global(ebpf_module_t *em) * * @param ptr thread data. **/ -static void ebpf_vfs_exit(void *ptr) +static void ebpf_vfs_exit(void *pptr) { - ebpf_module_t *em = (ebpf_module_t *)ptr; + ebpf_module_t *em = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!em) return; if (ebpf_read_vfs.thread) - netdata_thread_cancel(*ebpf_read_vfs.thread); + nd_thread_signal_cancel(ebpf_read_vfs.thread); if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) { pthread_mutex_lock(&lock); @@ -2049,13 +2050,11 @@ void *ebpf_read_vfs_thread(void *ptr) uint32_t running_time = 0; usec_t period = update_every * USEC_PER_SEC; int max_period = update_every * EBPF_CLEANUP_FACTOR; - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, period); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; - netdata_thread_disable_cancelability(); - pthread_mutex_lock(&collect_data_mutex); ebpf_vfs_read_apps(maps_per_core, max_period); ebpf_vfs_resume_apps_data(); @@ -2071,7 +2070,6 @@ void *ebpf_read_vfs_thread(void *ptr) em->running_time = running_time; pthread_mutex_unlock(&ebpf_exit_cleanup); - netdata_thread_enable_cancelability(); } return NULL; @@ -2095,9 +2093,9 @@ static void vfs_collector(ebpf_module_t *em) uint32_t lifetime = em->lifetime; netdata_idx_t *stats = em->hash_table_stats; memset(stats, 0, sizeof(em->hash_table_stats)); - while (!ebpf_plugin_exit && running_time < lifetime) { + while (!ebpf_plugin_stop() && running_time < lifetime) { (void)heartbeat_next(&hb, USEC_PER_SEC); - if (ebpf_plugin_exit || ++counter != update_every) + if (ebpf_plugin_stop() || ++counter != update_every) continue; counter = 0; @@ -2606,9 +2604,10 @@ static int ebpf_vfs_load_bpf(ebpf_module_t *em) */ void *ebpf_vfs_thread(void *ptr) { - netdata_thread_cleanup_push(ebpf_vfs_exit, ptr); - ebpf_module_t *em = (ebpf_module_t *)ptr; + + CLEANUP_FUNCTION_REGISTER(ebpf_vfs_exit) cleanup_ptr = em; + em->maps = vfs_maps; ebpf_update_pid_table(&vfs_maps[NETDATA_VFS_PID], em); @@ -2637,18 +2636,12 @@ void *ebpf_vfs_thread(void *ptr) pthread_mutex_unlock(&lock); - ebpf_read_vfs.thread = mallocz(sizeof(netdata_thread_t)); - netdata_thread_create(ebpf_read_vfs.thread, - ebpf_read_vfs.name, - NETDATA_THREAD_OPTION_DEFAULT, - ebpf_read_vfs_thread, - em); + ebpf_read_vfs.thread = nd_thread_create(ebpf_read_vfs.name, NETDATA_THREAD_OPTION_DEFAULT, ebpf_read_vfs_thread, em); vfs_collector(em); endvfs: ebpf_update_disabled_plugin_stats(em); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/freebsd.plugin/plugin_freebsd.c b/src/collectors/freebsd.plugin/plugin_freebsd.c index 6b77dcf3a2d15d..feae759c5c402c 100644 --- a/src/collectors/freebsd.plugin/plugin_freebsd.c +++ b/src/collectors/freebsd.plugin/plugin_freebsd.c @@ -71,23 +71,24 @@ static struct freebsd_module { #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 33 #endif -static void freebsd_main_cleanup(void *ptr) +static void freebsd_main_cleanup(void *pptr) { - worker_unregister(); + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } void *freebsd_main(void *ptr) { - worker_register("FREEBSD"); + CLEANUP_FUNCTION_REGISTER(freebsd_main_cleanup) cleanup_ptr = ptr; - netdata_thread_cleanup_push(freebsd_main_cleanup, ptr); + worker_register("FREEBSD"); // initialize FreeBSD plugin if (freebsd_plugin_init()) @@ -131,6 +132,5 @@ void *freebsd_main(void *ptr) } } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/freeipmi.plugin/freeipmi_plugin.c b/src/collectors/freeipmi.plugin/freeipmi_plugin.c index 3b27c5d5a83656..23bdea05796cc0 100644 --- a/src/collectors/freeipmi.plugin/freeipmi_plugin.c +++ b/src/collectors/freeipmi.plugin/freeipmi_plugin.c @@ -1977,12 +1977,9 @@ int main (int argc, char **argv) { }, }; - netdata_thread_t sensors_thread = 0, sel_thread = 0; - - netdata_thread_create(&sensors_thread, "IPMI[sensors]", NETDATA_THREAD_OPTION_DONT_LOG, netdata_ipmi_collection_thread, &sensors_data); - + nd_thread_create("IPMI[sensors]", NETDATA_THREAD_OPTION_DONT_LOG, netdata_ipmi_collection_thread, &sensors_data); if(netdata_do_sel) - netdata_thread_create(&sel_thread, "IPMI[sel]", NETDATA_THREAD_OPTION_DONT_LOG, netdata_ipmi_collection_thread, &sel_data); + nd_thread_create("IPMI[sel]", NETDATA_THREAD_OPTION_DONT_LOG, netdata_ipmi_collection_thread, &sel_data); // ------------------------------------------------------------------------ // the main loop diff --git a/src/collectors/idlejitter.plugin/plugin_idlejitter.c b/src/collectors/idlejitter.plugin/plugin_idlejitter.c index f71c99ad13bed9..bcdc3aff8bb0d8 100644 --- a/src/collectors/idlejitter.plugin/plugin_idlejitter.c +++ b/src/collectors/idlejitter.plugin/plugin_idlejitter.c @@ -4,23 +4,24 @@ #define CPU_IDLEJITTER_SLEEP_TIME_MS 20 -static void cpuidlejitter_main_cleanup(void *ptr) { - worker_unregister(); +static void cpuidlejitter_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!pptr) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } void *cpuidlejitter_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(cpuidlejitter_main_cleanup) cleanup_ptr = ptr; + worker_register("IDLEJITTER"); worker_register_job_name(0, "measurements"); - netdata_thread_cleanup_push(cpuidlejitter_main_cleanup, ptr); - usec_t sleep_ut = config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS) * USEC_PER_MS; if(sleep_ut <= 0) { config_set_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS); @@ -85,7 +86,6 @@ void *cpuidlejitter_main(void *ptr) { } } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/log2journal/log2journal-yaml.c b/src/collectors/log2journal/log2journal-yaml.c index cfdbcf4bdf291f..c4b537f941ecf0 100644 --- a/src/collectors/log2journal/log2journal-yaml.c +++ b/src/collectors/log2journal/log2journal-yaml.c @@ -48,7 +48,7 @@ static const char *yaml_event_name(yaml_event_type_t type) { } #define yaml_error(parser, event, fmt, args...) yaml_error_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__, fmt, ##args) -static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) __attribute__ ((format(__printf__, 6, 7))); +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) PRINTFLIKE(6, 7); static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) { char buf[1024] = ""; // Initialize buf to an empty string const char *type = ""; diff --git a/src/collectors/log2journal/log2journal.c b/src/collectors/log2journal/log2journal.c index e4de6cd2486e72..0fbba0b0c068d9 100644 --- a/src/collectors/log2journal/log2journal.c +++ b/src/collectors/log2journal/log2journal.c @@ -277,7 +277,7 @@ static inline void send_key_value_constant(LOG_JOB *jb __maybe_unused, HASHED_KE // fprintf(stderr, "SET %s=%.*s\n", ht_key->key, (int)ht_key->value.len, ht_key->value.txt); } -static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char *format, ...) __attribute__ ((format(__printf__, 3, 4))); +static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char *format, ...) PRINTFLIKE(3, 4); static inline void send_key_value_error(LOG_JOB *jb, HASHED_KEY *key, const char *format, ...) { HASHED_KEY *ht_key = get_key_from_hashtable(jb, key); diff --git a/src/collectors/log2journal/log2journal.h b/src/collectors/log2journal/log2journal.h index c52e0898eb1178..7d95c0f3a10fc6 100644 --- a/src/collectors/log2journal/log2journal.h +++ b/src/collectors/log2journal/log2journal.h @@ -17,11 +17,34 @@ #include #include +// ---------------------------------------------------------------------------- +// compatibility + +#ifndef HAVE_STRNDUP +// strndup() is not available on Windows +static inline char *os_strndup( const char *s1, size_t n) +{ + char *copy= (char*)malloc( n+1 ); + memcpy( copy, s1, n ); + copy[n] = 0; + return copy; +}; +#define strndup(s, n) os_strndup(s, n) +#endif + +#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT) && !defined(COMPILED_FOR_MACOS) +#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a))) +#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT) +#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a))) +#else +#define PRINTFLIKE(f, a) +#endif + // ---------------------------------------------------------------------------- // logging // enable the compiler to check for printf like errors on our log2stderr() function -static inline void log2stderr(const char *format, ...) __attribute__ ((format(__printf__, 1, 2))); +static inline void log2stderr(const char *format, ...) PRINTFLIKE(1, 2); static inline void log2stderr(const char *format, ...) { va_list args; va_start(args, format); diff --git a/src/collectors/macos.plugin/plugin_macos.c b/src/collectors/macos.plugin/plugin_macos.c index 3aaa46c729b827..94168fe08a413f 100644 --- a/src/collectors/macos.plugin/plugin_macos.c +++ b/src/collectors/macos.plugin/plugin_macos.c @@ -25,23 +25,24 @@ static struct macos_module { #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3 #endif -static void macos_main_cleanup(void *ptr) +static void macos_main_cleanup(void *pptr) { - worker_unregister(); + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } void *macos_main(void *ptr) { - worker_register("MACOS"); + CLEANUP_FUNCTION_REGISTER(macos_main_cleanup) cleanup_ptr = ptr; - netdata_thread_cleanup_push(macos_main_cleanup, ptr); + worker_register("MACOS"); // check the enabled status for each module for (int i = 0; macos_modules[i].name; i++) { @@ -76,6 +77,5 @@ void *macos_main(void *ptr) } } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/network-viewer.plugin/network-viewer.c b/src/collectors/network-viewer.plugin/network-viewer.c index 074535a0b0274d..88c2df169f353b 100644 --- a/src/collectors/network-viewer.plugin/network-viewer.c +++ b/src/collectors/network-viewer.plugin/network-viewer.c @@ -739,7 +739,7 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu int main(int argc __maybe_unused, char **argv __maybe_unused) { clocks_init(); - netdata_thread_set_tag("NETWORK-VIEWER"); + nd_thread_tag_set("NETWORK-VIEWER"); nd_log_initialize_for_external_plugins("network-viewer.plugin"); netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX"); diff --git a/src/collectors/perf.plugin/perf_plugin.c b/src/collectors/perf.plugin/perf_plugin.c index b839263d1972f7..015f73d2cad8da 100644 --- a/src/collectors/perf.plugin/perf_plugin.c +++ b/src/collectors/perf.plugin/perf_plugin.c @@ -246,7 +246,7 @@ static int perf_init() { struct perf_event *current_event = NULL; unsigned long flags = 0; - number_of_cpus = (int)get_system_cpus(); + number_of_cpus = (int)os_get_system_cpus(); // initialize all perf event file descriptors for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) { diff --git a/src/collectors/plugins.d/ndsudo.c b/src/collectors/plugins.d/ndsudo.c index 2795773f0f46fb..30f78c752c90f1 100644 --- a/src/collectors/plugins.d/ndsudo.c +++ b/src/collectors/plugins.d/ndsudo.c @@ -332,8 +332,8 @@ int main(int argc, char *argv[]) { return 3; } - char new_path[] = "/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"; - setenv("PATH", new_path, 1); + char new_path[] = "PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"; + putenv(new_path); bool found = false; char filename[FILENAME_MAX]; diff --git a/src/collectors/plugins.d/plugins_d.c b/src/collectors/plugins.d/plugins_d.c index 0bcb3df632ab37..f5f55b7702b7d5 100644 --- a/src/collectors/plugins.d/plugins_d.c +++ b/src/collectors/plugins.d/plugins_d.c @@ -6,6 +6,16 @@ char *plugin_directories[PLUGINSD_MAX_DIRECTORIES] = { [0] = PLUGINS_DIR, }; struct plugind *pluginsd_root = NULL; +static inline void pluginsd_sleep(const int seconds) { + int timeout_ms = seconds * 1000; + int waited_ms = 0; + while(waited_ms < timeout_ms) { + if(!service_running(SERVICE_COLLECTORS)) break; + sleep_usec(ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS * USEC_PER_MS); + waited_ms += ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS; + } +} + inline size_t pluginsd_initialize_plugin_directories() { char plugins_dirs[(FILENAME_MAX * 2) + 1]; @@ -47,8 +57,9 @@ static inline bool plugin_is_running(struct plugind *cd) { return ret; } -static void pluginsd_worker_thread_cleanup(void *arg) { - struct plugind *cd = (struct plugind *)arg; +static void pluginsd_worker_thread_cleanup(void *pptr) { + struct plugind *cd = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!cd) return; worker_unregister(); @@ -79,7 +90,7 @@ static void pluginsd_worker_thread_cleanup(void *arg) { #define SERIAL_FAILURES_THRESHOLD 10 static void pluginsd_worker_thread_handle_success(struct plugind *cd) { if (likely(cd->successful_collections)) { - sleep((unsigned int)cd->update_every); + pluginsd_sleep(cd->update_every); return; } @@ -88,7 +99,7 @@ static void pluginsd_worker_thread_handle_success(struct plugind *cd) { rrdhost_hostname(cd->host), cd->fullfilename, cd->unsafe.pid, plugin_is_enabled(cd) ? "Waiting a bit before starting it again." : "Will not start it again - it is now disabled."); - sleep((unsigned int)(cd->update_every * 10)); + pluginsd_sleep(cd->update_every * 10); return; } @@ -121,7 +132,8 @@ static void pluginsd_worker_thread_handle_error(struct plugind *cd, int worker_r netdata_log_error("PLUGINSD: 'host:%s', '%s' (pid %d) exited with error code %d, but has given useful output in the past (%zu times). %s", rrdhost_hostname(cd->host), cd->fullfilename, cd->unsafe.pid, worker_ret_code, cd->successful_collections, plugin_is_enabled(cd) ? "Waiting a bit before starting it again." : "Will not start it again - it is disabled."); - sleep((unsigned int)(cd->update_every * 10)); + + pluginsd_sleep(cd->update_every * 10); return; } @@ -138,73 +150,73 @@ static void pluginsd_worker_thread_handle_error(struct plugind *cd, int worker_r #undef SERIAL_FAILURES_THRESHOLD static void *pluginsd_worker_thread(void *arg) { + struct plugind *cd = (struct plugind *) arg; + CLEANUP_FUNCTION_REGISTER(pluginsd_worker_thread_cleanup) cleanup_ptr = cd; + worker_register("PLUGINSD"); - netdata_thread_cleanup_push(pluginsd_worker_thread_cleanup, arg) - { - struct plugind *cd = (struct plugind *) arg; - plugin_set_running(cd); + plugin_set_running(cd); - size_t count = 0; + size_t count = 0; - while(service_running(SERVICE_COLLECTORS)) { - FILE *fp_child_input = NULL; - FILE *fp_child_output = netdata_popen(cd->cmd, &cd->unsafe.pid, &fp_child_input); + while(service_running(SERVICE_COLLECTORS)) { + FILE *fp_child_input = NULL; + FILE *fp_child_output = netdata_popen(cd->cmd, &cd->unsafe.pid, &fp_child_input); - if(unlikely(!fp_child_input || !fp_child_output)) { - netdata_log_error("PLUGINSD: 'host:%s', cannot popen(\"%s\", \"r\").", - rrdhost_hostname(cd->host), cd->cmd); - break; - } + if(unlikely(!fp_child_input || !fp_child_output)) { + netdata_log_error("PLUGINSD: 'host:%s', cannot popen(\"%s\", \"r\").", + rrdhost_hostname(cd->host), cd->cmd); + break; + } - nd_log(NDLS_DAEMON, NDLP_DEBUG, - "PLUGINSD: 'host:%s' connected to '%s' running on pid %d", - rrdhost_hostname(cd->host), - cd->fullfilename, cd->unsafe.pid); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "PLUGINSD: 'host:%s' connected to '%s' running on pid %d", + rrdhost_hostname(cd->host), + cd->fullfilename, cd->unsafe.pid); - const char *plugin = strrchr(cd->fullfilename, '/'); - if(plugin) - plugin++; - else - plugin = cd->fullfilename; + const char *plugin = strrchr(cd->fullfilename, '/'); + if(plugin) + plugin++; + else + plugin = cd->fullfilename; - char module[100]; - snprintfz(module, sizeof(module), "plugins.d[%s]", plugin); - ND_LOG_STACK lgs[] = { - ND_LOG_FIELD_TXT(NDF_MODULE, module), - ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(cd->host)), - ND_LOG_FIELD_TXT(NDF_SRC_TRANSPORT, "pluginsd"), - ND_LOG_FIELD_END(), - }; - ND_LOG_STACK_PUSH(lgs); + char module[100]; + snprintfz(module, sizeof(module), "plugins.d[%s]", plugin); + ND_LOG_STACK lgs[] = { + ND_LOG_FIELD_TXT(NDF_MODULE, module), + ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(cd->host)), + ND_LOG_FIELD_TXT(NDF_SRC_TRANSPORT, "pluginsd"), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); - count = pluginsd_process(cd->host, cd, fp_child_input, fp_child_output, 0); + count = pluginsd_process(cd->host, cd, fp_child_input, fp_child_output, 0); - nd_log(NDLS_DAEMON, NDLP_DEBUG, - "PLUGINSD: 'host:%s', '%s' (pid %d) disconnected after %zu successful data collections (ENDs).", - rrdhost_hostname(cd->host), cd->fullfilename, cd->unsafe.pid, count); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "PLUGINSD: 'host:%s', '%s' (pid %d) disconnected after %zu successful data collections (ENDs).", + rrdhost_hostname(cd->host), cd->fullfilename, cd->unsafe.pid, count); - killpid(cd->unsafe.pid); + killpid(cd->unsafe.pid); - int worker_ret_code = netdata_pclose(fp_child_input, fp_child_output, cd->unsafe.pid); + int worker_ret_code = netdata_pclose(fp_child_input, fp_child_output, cd->unsafe.pid); - if(likely(worker_ret_code == 0)) - pluginsd_worker_thread_handle_success(cd); - else - pluginsd_worker_thread_handle_error(cd, worker_ret_code); + if(likely(worker_ret_code == 0)) + pluginsd_worker_thread_handle_success(cd); + else + pluginsd_worker_thread_handle_error(cd, worker_ret_code); - cd->unsafe.pid = 0; + cd->unsafe.pid = 0; - if(unlikely(!plugin_is_enabled(cd))) - break; - } + if(unlikely(!plugin_is_enabled(cd))) + break; } - netdata_thread_cleanup_pop(1); return NULL; } -static void pluginsd_main_cleanup(void *data) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)data; +static void pluginsd_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_info("PLUGINSD: cleaning up..."); @@ -215,7 +227,7 @@ static void pluginsd_main_cleanup(void *data) { netdata_log_info("PLUGINSD: 'host:%s', stopping plugin thread: %s", rrdhost_hostname(cd->host), cd->id); - netdata_thread_cancel(cd->unsafe.thread); + nd_thread_signal_cancel(cd->unsafe.thread); } spinlock_unlock(&cd->unsafe.spinlock); } @@ -226,9 +238,8 @@ static void pluginsd_main_cleanup(void *data) { worker_unregister(); } -void *pluginsd_main(void *ptr) -{ - netdata_thread_cleanup_push(pluginsd_main_cleanup, ptr); +void *pluginsd_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(pluginsd_main_cleanup) cleanup_ptr = ptr; int automatic_run = config_get_boolean(CONFIG_SECTION_PLUGINS, "enable running new plugins", 1); int scan_frequency = (int)config_get_number(CONFIG_SECTION_PLUGINS, "check for new plugins every", 60); @@ -340,11 +351,8 @@ void *pluginsd_main(void *ptr) snprintfz(tag, NETDATA_THREAD_TAG_MAX, "PD[%s]", pluginname); // spawn a new thread for it - netdata_thread_create(&cd->unsafe.thread, - tag, - NETDATA_THREAD_OPTION_DEFAULT, - pluginsd_worker_thread, - cd); + cd->unsafe.thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, + pluginsd_worker_thread, cd); } } } @@ -352,9 +360,8 @@ void *pluginsd_main(void *ptr) closedir(dir); } - sleep((unsigned int)scan_frequency); + pluginsd_sleep(scan_frequency); } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/plugins.d/plugins_d.h b/src/collectors/plugins.d/plugins_d.h index f831efa3ad81a8..ec17c3145a4b4d 100644 --- a/src/collectors/plugins.d/plugins_d.h +++ b/src/collectors/plugins.d/plugins_d.h @@ -33,7 +33,7 @@ struct plugind { SPINLOCK spinlock; bool running; // do not touch this structure after setting this to 1 bool enabled; // if this is enabled or not - netdata_thread_t thread; + ND_THREAD *thread; pid_t pid; } unsafe; @@ -46,7 +46,7 @@ struct plugind { extern struct plugind *pluginsd_root; size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugin_input, FILE *fp_plugin_output, int trust_durations); -void pluginsd_process_thread_cleanup(void *ptr); +void pluginsd_process_thread_cleanup(void *pptr); size_t pluginsd_initialize_plugin_directories(); diff --git a/src/collectors/plugins.d/pluginsd_functions.h b/src/collectors/plugins.d/pluginsd_functions.h index cd49512e4409f4..ad47dc23aab6e9 100644 --- a/src/collectors/plugins.d/pluginsd_functions.h +++ b/src/collectors/plugins.d/pluginsd_functions.h @@ -6,7 +6,7 @@ #include "pluginsd_internals.h" struct inflight_function { - uuid_t transaction; + nd_uuid_t transaction; int code; int timeout_s; diff --git a/src/collectors/plugins.d/pluginsd_internals.h b/src/collectors/plugins.d/pluginsd_internals.h index 31db02544cd0a4..ae7e994277dea7 100644 --- a/src/collectors/plugins.d/pluginsd_internals.h +++ b/src/collectors/plugins.d/pluginsd_internals.h @@ -88,7 +88,7 @@ static inline void pluginsd_clear_scope_chart(PARSER *parser, const char *keywor static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const char *keyword) { RRDSET *old_st = parser->user.st; pid_t old_collector_tid = (old_st) ? old_st->pluginsd.collector_tid : 0; - pid_t my_collector_tid = gettid(); + pid_t my_collector_tid = gettid_cached(); if(unlikely(old_collector_tid)) { if(old_collector_tid != my_collector_tid) { diff --git a/src/collectors/plugins.d/pluginsd_parser.c b/src/collectors/plugins.d/pluginsd_parser.c index d13fe11d203fc8..3b6f8f8c52af4d 100644 --- a/src/collectors/plugins.d/pluginsd_parser.c +++ b/src/collectors/plugins.d/pluginsd_parser.c @@ -121,7 +121,7 @@ static void pluginsd_host_define_cleanup(PARSER *parser) { parser->user.host_define.parsing_host = false; } -static inline bool pluginsd_validate_machine_guid(const char *guid, uuid_t *uuid, char *output) { +static inline bool pluginsd_validate_machine_guid(const char *guid, nd_uuid_t *uuid, char *output) { if(uuid_parse(guid, *uuid)) return false; @@ -231,7 +231,7 @@ static inline PARSER_RC pluginsd_host(char **words, size_t num_words, PARSER *pa return PARSER_RC_OK; } - uuid_t uuid; + nd_uuid_t uuid; char uuid_str[UUID_STR_LEN]; if(!pluginsd_validate_machine_guid(guid, &uuid, uuid_str)) return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST, "cannot parse MACHINE_GUID - is it a valid UUID?"); @@ -1088,7 +1088,7 @@ static inline PARSER_RC streaming_claimed_id(char **words, size_t num_words, PAR return PARSER_RC_ERROR; } - uuid_t uuid; + nd_uuid_t uuid; RRDHOST *host = parser->user.host; // We don't need the parsed UUID @@ -1130,8 +1130,9 @@ void pluginsd_cleanup_v2(PARSER *parser) { pluginsd_clear_scope_chart(parser, "THREAD CLEANUP"); } -void pluginsd_process_thread_cleanup(void *ptr) { - PARSER *parser = (PARSER *)ptr; +void pluginsd_process_thread_cleanup(void *pptr) { + PARSER *parser = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!parser) return; pluginsd_cleanup_v2(parser); pluginsd_host_define_cleanup(parser); @@ -1218,54 +1219,49 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi size_t count = 0; - // this keeps the parser with its current value - // so, parser needs to be allocated before pushing it - netdata_thread_cleanup_push(pluginsd_process_thread_cleanup, parser) - { - ND_LOG_STACK lgs[] = { - ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &parser->line), - ND_LOG_FIELD_CB(NDF_NIDL_NODE, parser_reconstruct_node, parser), - ND_LOG_FIELD_CB(NDF_NIDL_INSTANCE, parser_reconstruct_instance, parser), - ND_LOG_FIELD_CB(NDF_NIDL_CONTEXT, parser_reconstruct_context, parser), - ND_LOG_FIELD_END(), - }; - ND_LOG_STACK_PUSH(lgs); - - buffered_reader_init(&parser->reader); - CLEAN_BUFFER *buffer = buffer_create(sizeof(parser->reader.read_buffer) + 2, NULL); - while(likely(service_running(SERVICE_COLLECTORS))) { - - if(unlikely(!buffered_reader_next_line(&parser->reader, buffer))) { - buffered_reader_ret_t ret = buffered_reader_read_timeout( - &parser->reader, - fileno((FILE *) parser->fp_input), - 2 * 60 * MSEC_PER_SEC, true - ); + ND_LOG_STACK lgs[] = { + ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &parser->line), + ND_LOG_FIELD_CB(NDF_NIDL_NODE, parser_reconstruct_node, parser), + ND_LOG_FIELD_CB(NDF_NIDL_INSTANCE, parser_reconstruct_instance, parser), + ND_LOG_FIELD_CB(NDF_NIDL_CONTEXT, parser_reconstruct_context, parser), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); - if(unlikely(ret != BUFFERED_READER_READ_OK)) - break; + CLEANUP_FUNCTION_REGISTER(pluginsd_process_thread_cleanup) cleanup_parser = parser; + buffered_reader_init(&parser->reader); + CLEAN_BUFFER *buffer = buffer_create(sizeof(parser->reader.read_buffer) + 2, NULL); + while(likely(service_running(SERVICE_COLLECTORS))) { - continue; - } + if(unlikely(!buffered_reader_next_line(&parser->reader, buffer))) { + buffered_reader_ret_t ret = buffered_reader_read_timeout( + &parser->reader, + fileno((FILE *) parser->fp_input), + 2 * 60 * MSEC_PER_SEC, true + ); - if(unlikely(parser_action(parser, buffer->buffer))) + if(unlikely(ret != BUFFERED_READER_READ_OK)) break; - buffer->len = 0; - buffer->buffer[0] = '\0'; + continue; } - cd->unsafe.enabled = parser->user.enabled; - count = parser->user.data_collections_count; + if(unlikely(parser_action(parser, buffer->buffer))) + break; - if(likely(count)) { - cd->successful_collections += count; - cd->serial_failures = 0; - } - else - cd->serial_failures++; + buffer->len = 0; + buffer->buffer[0] = '\0'; } - netdata_thread_cleanup_pop(1); // free parser with the pop function + + cd->unsafe.enabled = parser->user.enabled; + count = parser->user.data_collections_count; + + if(likely(count)) { + cd->successful_collections += count; + cd->serial_failures = 0; + } + else + cd->serial_failures++; return count; } diff --git a/src/collectors/plugins.d/pluginsd_parser.h b/src/collectors/plugins.d/pluginsd_parser.h index d317a77bee8911..6c126964bb84cf 100644 --- a/src/collectors/plugins.d/pluginsd_parser.h +++ b/src/collectors/plugins.d/pluginsd_parser.h @@ -65,7 +65,7 @@ typedef struct parser_user_object { struct { bool parsing_host; - uuid_t machine_guid; + nd_uuid_t machine_guid; char machine_guid_str[UUID_STR_LEN]; STRING *hostname; RRDLABELS *rrdlabels; diff --git a/src/collectors/proc.plugin/plugin_proc.c b/src/collectors/proc.plugin/plugin_proc.c index cacc70523e17ff..8ae4aace506a87 100644 --- a/src/collectors/proc.plugin/plugin_proc.c +++ b/src/collectors/proc.plugin/plugin_proc.c @@ -84,23 +84,21 @@ static struct proc_module { #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 36 #endif -static netdata_thread_t *netdev_thread = NULL; +static ND_THREAD *netdev_thread = NULL; -static void proc_main_cleanup(void *ptr) +static void proc_main_cleanup(void *pptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); - if (netdev_thread) { - netdata_thread_join(*netdev_thread, NULL); - freez(netdev_thread); - } + nd_thread_join(netdev_thread); + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; - - worker_unregister(); } bool inside_lxc_container = false; @@ -146,70 +144,67 @@ static bool log_proc_module(BUFFER *wb, void *data) { void *proc_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(proc_main_cleanup) cleanup_ptr = ptr; + worker_register("PROC"); rrd_collector_started(); if (config_get_boolean("plugin:proc", "/proc/net/dev", CONFIG_BOOLEAN_YES)) { - netdev_thread = mallocz(sizeof(netdata_thread_t)); netdata_log_debug(D_SYSTEM, "Starting thread %s.", THREAD_NETDEV_NAME); - netdata_thread_create( - netdev_thread, THREAD_NETDEV_NAME, NETDATA_THREAD_OPTION_JOINABLE, netdev_main, netdev_thread); + netdev_thread = nd_thread_create(THREAD_NETDEV_NAME, NETDATA_THREAD_OPTION_JOINABLE, netdev_main, NULL); } - netdata_thread_cleanup_push(proc_main_cleanup, ptr) - { - config_get_boolean("plugin:proc", "/proc/pagetypeinfo", CONFIG_BOOLEAN_NO); - config_get_boolean("plugin:proc", "/proc/spl/kstat/zfs/pool/state", CONFIG_BOOLEAN_NO); + config_get_boolean("plugin:proc", "/proc/pagetypeinfo", CONFIG_BOOLEAN_NO); + config_get_boolean("plugin:proc", "/proc/spl/kstat/zfs/pool/state", CONFIG_BOOLEAN_NO); - // check the enabled status for each module - int i; - for(i = 0; proc_modules[i].name; i++) { - struct proc_module *pm = &proc_modules[i]; + // check the enabled status for each module + int i; + for(i = 0; proc_modules[i].name; i++) { + struct proc_module *pm = &proc_modules[i]; - pm->enabled = config_get_boolean("plugin:proc", pm->name, CONFIG_BOOLEAN_YES); - pm->rd = NULL; + pm->enabled = config_get_boolean("plugin:proc", pm->name, CONFIG_BOOLEAN_YES); + pm->rd = NULL; - worker_register_job_name(i, proc_modules[i].dim); - } + worker_register_job_name(i, proc_modules[i].dim); + } - usec_t step = localhost->rrd_update_every * USEC_PER_SEC; - heartbeat_t hb; - heartbeat_init(&hb); + usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); - inside_lxc_container = is_lxcfs_proc_mounted(); + inside_lxc_container = is_lxcfs_proc_mounted(); #define LGS_MODULE_ID 0 - ND_LOG_STACK lgs[] = { - [LGS_MODULE_ID] = ND_LOG_FIELD_TXT(NDF_MODULE, "proc.plugin"), - ND_LOG_FIELD_END(), - }; - ND_LOG_STACK_PUSH(lgs); + ND_LOG_STACK lgs[] = { + [LGS_MODULE_ID] = ND_LOG_FIELD_TXT(NDF_MODULE, "proc.plugin"), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); - while(service_running(SERVICE_COLLECTORS)) { - worker_is_idle(); - usec_t hb_dt = heartbeat_next(&hb, step); + while(service_running(SERVICE_COLLECTORS)) { + worker_is_idle(); + usec_t hb_dt = heartbeat_next(&hb, step); + if(unlikely(!service_running(SERVICE_COLLECTORS))) + break; + + for(i = 0; proc_modules[i].name; i++) { if(unlikely(!service_running(SERVICE_COLLECTORS))) break; - for(i = 0; proc_modules[i].name; i++) { - if(unlikely(!service_running(SERVICE_COLLECTORS))) - break; - - struct proc_module *pm = &proc_modules[i]; - if(unlikely(!pm->enabled)) - continue; + struct proc_module *pm = &proc_modules[i]; + if(unlikely(!pm->enabled)) + continue; - worker_is_busy(i); - lgs[LGS_MODULE_ID] = ND_LOG_FIELD_CB(NDF_MODULE, log_proc_module, pm); - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); - lgs[LGS_MODULE_ID] = ND_LOG_FIELD_TXT(NDF_MODULE, "proc.plugin"); - } + worker_is_busy(i); + lgs[LGS_MODULE_ID] = ND_LOG_FIELD_CB(NDF_MODULE, log_proc_module, pm); + pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); + lgs[LGS_MODULE_ID] = ND_LOG_FIELD_TXT(NDF_MODULE, "proc.plugin"); } } - netdata_thread_cleanup_pop(1); + return NULL; } diff --git a/src/collectors/proc.plugin/plugin_proc.h b/src/collectors/proc.plugin/plugin_proc.h index 187e76a9713510..25b9e3a941e281 100644 --- a/src/collectors/proc.plugin/plugin_proc.h +++ b/src/collectors/proc.plugin/plugin_proc.h @@ -9,7 +9,7 @@ #define PLUGIN_PROC_NAME PLUGIN_PROC_CONFIG_NAME ".plugin" #define THREAD_NETDEV_NAME "P[proc netdev]" -void *netdev_main(void *ptr); +void *netdev_main(void *ptr_is_null); int do_proc_net_wireless(int update_every, usec_t dt); int do_proc_diskstats(int update_every, usec_t dt); diff --git a/src/collectors/proc.plugin/proc_diskstats.c b/src/collectors/proc.plugin/proc_diskstats.c index 4ff617ff9f4489..3366901081fa21 100644 --- a/src/collectors/proc.plugin/proc_diskstats.c +++ b/src/collectors/proc.plugin/proc_diskstats.c @@ -2,10 +2,13 @@ #include "plugin_proc.h" -#define RRD_TYPE_DISK "disk" #define PLUGIN_PROC_MODULE_DISKSTATS_NAME "/proc/diskstats" #define CONFIG_SECTION_PLUGIN_PROC_DISKSTATS "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_DISKSTATS_NAME +#define _COMMON_PLUGIN_NAME PLUGIN_PROC_CONFIG_NAME +#define _COMMON_PLUGIN_MODULE_NAME PLUGIN_PROC_MODULE_DISKSTATS_NAME +#include "../common-contexts/common-contexts.h" + #define RRDFUNCTIONS_DISKSTATS_HELP "View block device statistics" #define DISK_TYPE_UNKNOWN 0 @@ -75,9 +78,7 @@ static struct disk { usec_t bcache_priority_stats_update_every_usec; usec_t bcache_priority_stats_elapsed_usec; - RRDSET *st_io; - RRDDIM *rd_io_reads; - RRDDIM *rd_io_writes; + ND_DISK_IO disk_io; RRDSET *st_ext_io; RRDDIM *rd_io_discards; @@ -1033,6 +1034,10 @@ static void add_labels_to_disk(struct disk *d, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device_type", get_disk_type_string(d->type), RRDLABEL_SRC_AUTO); } +static void disk_labels_cb(RRDSET *st, void *data) { + add_labels_to_disk(data, st); +} + static int diskstats_function_block_devices(BUFFER *wb, const char *function __maybe_unused) { buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; @@ -1076,8 +1081,8 @@ static int diskstats_function_block_devices(BUFFER *wb, const char *function __m buffer_json_add_array_item_string(wb, d->serial); // IO - double io_reads = rrddim_get_last_stored_value(d->rd_io_reads, &max_io_reads, 1024.0); - double io_writes = rrddim_get_last_stored_value(d->rd_io_writes, &max_io_writes, 1024.0); + double io_reads = rrddim_get_last_stored_value(d->disk_io.rd_io_reads, &max_io_reads, 1024.0); + double io_writes = rrddim_get_last_stored_value(d->disk_io.rd_io_writes, &max_io_writes, 1024.0); double io_total = NAN; if (!isnan(io_reads) && !isnan(io_writes)) { io_total = io_reads + io_writes; @@ -1328,7 +1333,7 @@ static void diskstats_cleanup_disks() { rrdset_obsolete_and_pointer_null(d->st_ext_await); rrdset_obsolete_and_pointer_null(d->st_backlog); rrdset_obsolete_and_pointer_null(d->st_busy); - rrdset_obsolete_and_pointer_null(d->st_io); + rrdset_obsolete_and_pointer_null(d->disk_io.st_io); rrdset_obsolete_and_pointer_null(d->st_ext_io); rrdset_obsolete_and_pointer_null(d->st_iotime); rrdset_obsolete_and_pointer_null(d->st_ext_iotime); @@ -1616,31 +1621,17 @@ int do_proc_diskstats(int update_every, usec_t dt) { netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { d->do_io = CONFIG_BOOLEAN_YES; - if(unlikely(!d->st_io)) { - d->st_io = rrdset_create_localhost( - RRD_TYPE_DISK - , d->chart_id - , d->disk - , family - , "disk.io" - , "Disk I/O Bandwidth" - , "KiB/s" - , PLUGIN_PROC_NAME - , PLUGIN_PROC_MODULE_DISKSTATS_NAME - , NETDATA_CHART_PRIO_DISK_IO - , update_every - , RRDSET_TYPE_AREA - ); - - d->rd_io_reads = rrddim_add(d->st_io, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_INCREMENTAL); - d->rd_io_writes = rrddim_add(d->st_io, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_INCREMENTAL); - - add_labels_to_disk(d, d->st_io); - } - - last_readsectors = rrddim_set_by_pointer(d->st_io, d->rd_io_reads, readsectors); - last_writesectors = rrddim_set_by_pointer(d->st_io, d->rd_io_writes, writesectors); - rrdset_done(d->st_io); + last_readsectors = d->disk_io.rd_io_reads ? d->disk_io.rd_io_reads->collector.last_collected_value : 0; + last_writesectors = d->disk_io.rd_io_writes ? d->disk_io.rd_io_writes->collector.last_collected_value : 0; + + common_disk_io(&d->disk_io, + d->chart_id, + d->disk, + readsectors * d->sector_size, + writesectors * d->sector_size, + update_every, + disk_labels_cb, + d); } if (do_dc_stats && d->do_io == CONFIG_BOOLEAN_YES && d->do_ext != CONFIG_BOOLEAN_NO) { @@ -2468,32 +2459,7 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(global_do_io == CONFIG_BOOLEAN_YES || (global_do_io == CONFIG_BOOLEAN_AUTO && (system_read_kb || system_write_kb || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { - static RRDSET *st_io = NULL; - static RRDDIM *rd_in = NULL, *rd_out = NULL; - - if(unlikely(!st_io)) { - st_io = rrdset_create_localhost( - "system" - , "io" - , NULL - , "disk" - , NULL - , "Disk I/O" - , "KiB/s" - , PLUGIN_PROC_NAME - , PLUGIN_PROC_MODULE_DISKSTATS_NAME - , NETDATA_CHART_PRIO_SYSTEM_IO - , update_every - , RRDSET_TYPE_AREA - ); - - rd_in = rrddim_add(st_io, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st_io, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - } - - rrddim_set_by_pointer(st_io, rd_in, system_read_kb); - rrddim_set_by_pointer(st_io, rd_out, system_write_kb); - rrdset_done(st_io); + common_system_io(system_read_kb * 1024, system_write_kb * 1024, update_every); } return 0; diff --git a/src/collectors/proc.plugin/proc_loadavg.c b/src/collectors/proc.plugin/proc_loadavg.c index c9339525e7abce..154fae3a725f20 100644 --- a/src/collectors/proc.plugin/proc_loadavg.c +++ b/src/collectors/proc.plugin/proc_loadavg.c @@ -48,7 +48,7 @@ int do_proc_loadavg(int update_every, usec_t dt) { unsigned long long active_processes = str2ull(procfile_lineword(ff, 0, 4), NULL); //get system pid_max - unsigned long long max_processes = get_system_pid_max(); + unsigned long long max_processes = os_get_system_pid_max(); // //unsigned long long next_pid = str2ull(procfile_lineword(ff, 0, 5)); diff --git a/src/collectors/proc.plugin/proc_meminfo.c b/src/collectors/proc.plugin/proc_meminfo.c index a357cc7824f17a..aa3336761b8053 100644 --- a/src/collectors/proc.plugin/proc_meminfo.c +++ b/src/collectors/proc.plugin/proc_meminfo.c @@ -5,6 +5,10 @@ #define PLUGIN_PROC_MODULE_MEMINFO_NAME "/proc/meminfo" #define CONFIG_SECTION_PLUGIN_PROC_MEMINFO "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_MEMINFO_NAME +#define _COMMON_PLUGIN_NAME PLUGIN_PROC_NAME +#define _COMMON_PLUGIN_MODULE_NAME PLUGIN_PROC_MODULE_MEMINFO_NAME +#include "../common-contexts/common-contexts.h" + int do_proc_meminfo(int update_every, usec_t dt) { (void)dt; @@ -242,65 +246,10 @@ int do_proc_meminfo(int update_every, usec_t dt) { } if(do_ram) { - { - static RRDSET *st_system_ram = NULL; - static RRDDIM *rd_free = NULL, *rd_used = NULL, *rd_cached = NULL, *rd_buffers = NULL; - - if(unlikely(!st_system_ram)) { - st_system_ram = rrdset_create_localhost( - "system" - , "ram" - , NULL - , "ram" - , NULL - , "System RAM" - , "MiB" - , PLUGIN_PROC_NAME - , PLUGIN_PROC_MODULE_MEMINFO_NAME - , NETDATA_CHART_PRIO_SYSTEM_RAM - , update_every - , RRDSET_TYPE_STACKED - ); - - rd_free = rrddim_add(st_system_ram, "free", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - rd_used = rrddim_add(st_system_ram, "used", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - rd_cached = rrddim_add(st_system_ram, "cached", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - rd_buffers = rrddim_add(st_system_ram, "buffers", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - } - - rrddim_set_by_pointer(st_system_ram, rd_free, MemFree); - rrddim_set_by_pointer(st_system_ram, rd_used, MemUsed); - rrddim_set_by_pointer(st_system_ram, rd_cached, MemCached); - rrddim_set_by_pointer(st_system_ram, rd_buffers, Buffers); - rrdset_done(st_system_ram); - } - - if(arl_memavailable->flags & ARL_ENTRY_FLAG_FOUND) { - static RRDSET *st_mem_available = NULL; - static RRDDIM *rd_avail = NULL; - - if(unlikely(!st_mem_available)) { - st_mem_available = rrdset_create_localhost( - "mem" - , "available" - , NULL - , "overview" - , NULL - , "Available RAM for applications" - , "MiB" - , PLUGIN_PROC_NAME - , PLUGIN_PROC_MODULE_MEMINFO_NAME - , NETDATA_CHART_PRIO_MEM_SYSTEM_AVAILABLE - , update_every - , RRDSET_TYPE_AREA - ); - - rd_avail = rrddim_add(st_mem_available, "MemAvailable", "avail", 1, 1024, RRD_ALGORITHM_ABSOLUTE); - } + common_system_ram(MemFree * 1024, MemUsed * 1024, MemCached * 1024, Buffers * 1024, update_every); - rrddim_set_by_pointer(st_mem_available, rd_avail, MemAvailable); - rrdset_done(st_mem_available); - } + if(arl_memavailable->flags & ARL_ENTRY_FLAG_FOUND) + common_mem_available(MemAvailable * 1024, update_every); } unsigned long long SwapUsed = SwapTotal - SwapFree; @@ -309,35 +258,7 @@ int do_proc_meminfo(int update_every, usec_t dt) { (SwapTotal || SwapUsed || SwapFree || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { do_swap = CONFIG_BOOLEAN_YES; - - static RRDSET *st_system_swap = NULL; - static RRDDIM *rd_free = NULL, *rd_used = NULL; - - if(unlikely(!st_system_swap)) { - st_system_swap = rrdset_create_localhost( - "mem" - , "swap" - , NULL - , "swap" - , NULL - , "System Swap" - , "MiB" - , PLUGIN_PROC_NAME - , PLUGIN_PROC_MODULE_MEMINFO_NAME - , NETDATA_CHART_PRIO_MEM_SWAP - , update_every - , RRDSET_TYPE_STACKED - ); - - rrdset_flag_set(st_system_swap, RRDSET_FLAG_DETAIL); - - rd_free = rrddim_add(st_system_swap, "free", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - rd_used = rrddim_add(st_system_swap, "used", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - } - - rrddim_set_by_pointer(st_system_swap, rd_used, SwapUsed); - rrddim_set_by_pointer(st_system_swap, rd_free, SwapFree); - rrdset_done(st_system_swap); + common_mem_swap(SwapFree * 1024, SwapUsed * 1024, update_every); { static RRDSET *st_mem_swap_cached = NULL; diff --git a/src/collectors/proc.plugin/proc_net_dev.c b/src/collectors/proc.plugin/proc_net_dev.c index d29bb7a72f56a5..6f90f7fffd958e 100644 --- a/src/collectors/proc.plugin/proc_net_dev.c +++ b/src/collectors/proc.plugin/proc_net_dev.c @@ -1738,17 +1738,19 @@ int do_proc_net_dev(int update_every, usec_t dt) { return 0; } -static void netdev_main_cleanup(void *ptr) -{ - UNUSED(ptr); +static void netdev_main_cleanup(void *pptr) { + if(CLEANUP_FUNCTION_GET_PTR(pptr) != (void *)0x01) + return; collector_info("cleaning up..."); worker_unregister(); } -void *netdev_main(void *ptr) +void *netdev_main(void *ptr_is_null __maybe_unused) { + CLEANUP_FUNCTION_REGISTER(netdev_main_cleanup) cleanup_ptr = (void *)0x01; + worker_register("NETDEV"); worker_register_job_name(0, "netdev"); @@ -1760,29 +1762,26 @@ void *netdev_main(void *ptr) "top", HTTP_ACCESS_ANONYMOUS_DATA, netdev_function_net_interfaces); - netdata_thread_cleanup_push(netdev_main_cleanup, ptr) { - usec_t step = localhost->rrd_update_every * USEC_PER_SEC; - heartbeat_t hb; - heartbeat_init(&hb); + usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); - while (service_running(SERVICE_COLLECTORS)) { - worker_is_idle(); - usec_t hb_dt = heartbeat_next(&hb, step); + while (service_running(SERVICE_COLLECTORS)) { + worker_is_idle(); + usec_t hb_dt = heartbeat_next(&hb, step); - if (unlikely(!service_running(SERVICE_COLLECTORS))) - break; + if (unlikely(!service_running(SERVICE_COLLECTORS))) + break; - cgroup_netdev_reset_all(); + cgroup_netdev_reset_all(); - worker_is_busy(0); + worker_is_busy(0); - netdata_mutex_lock(&netdev_mutex); - if (do_proc_net_dev(localhost->rrd_update_every, hb_dt)) - break; - netdata_mutex_unlock(&netdev_mutex); - } + netdata_mutex_lock(&netdev_mutex); + if (do_proc_net_dev(localhost->rrd_update_every, hb_dt)) + break; + netdata_mutex_unlock(&netdev_mutex); } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/proc.plugin/proc_net_netstat.c b/src/collectors/proc.plugin/proc_net_netstat.c index 4a999803fb8b03..1d82f2b68a575f 100644 --- a/src/collectors/proc.plugin/proc_net_netstat.c +++ b/src/collectors/proc.plugin/proc_net_netstat.c @@ -400,8 +400,8 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_AREA ); - rd_received = rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rd_sent = rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_received = rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_received, Ip6InOctets); @@ -438,10 +438,10 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_LINE ); - rd_received = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_sent = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_forwarded = rrddim_add(st, "OutForwDatagrams", "forwarded", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_delivers = rrddim_add(st, "InDelivers", "delivers", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_forwarded = rrddim_add(st, "forwarded", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_delivers = rrddim_add(st, "delivered", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_received, Ip6InReceives); @@ -619,8 +619,8 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_LINE ); - rd_received = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_sent = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_received, Udp6InDatagrams); @@ -703,8 +703,8 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_LINE ); - rd_received = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_sent = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_received, UdpLite6InDatagrams); @@ -786,8 +786,8 @@ static void do_proc_net_snmp6(int update_every) { ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rd_Ip6InMcastOctets = rrddim_add(st, "InMcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rd_Ip6OutMcastOctets = rrddim_add(st, "OutMcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6InMcastOctets = rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6OutMcastOctets = rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_Ip6InMcastOctets, Ip6InMcastOctets); @@ -821,8 +821,8 @@ static void do_proc_net_snmp6(int update_every) { ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rd_Ip6InBcastOctets = rrddim_add(st, "InBcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rd_Ip6OutBcastOctets = rrddim_add(st, "OutBcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6InBcastOctets = rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6OutBcastOctets = rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_Ip6InBcastOctets, Ip6InBcastOctets); @@ -856,8 +856,8 @@ static void do_proc_net_snmp6(int update_every) { ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rd_Ip6InMcastPkts = rrddim_add(st, "InMcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_Ip6OutMcastPkts = rrddim_add(st, "OutMcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6InMcastPkts = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6OutMcastPkts = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_Ip6InMcastPkts, Ip6InMcastPkts); @@ -890,8 +890,8 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_LINE ); - rd_Icmp6InMsgs = rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_Icmp6OutMsgs = rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Icmp6InMsgs = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Icmp6OutMsgs = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_Icmp6InMsgs, Icmp6InMsgs); @@ -924,8 +924,8 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_LINE ); - rd_Icmp6InRedirects = rrddim_add(st, "InRedirects", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_Icmp6OutRedirects = rrddim_add(st, "OutRedirects", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Icmp6InRedirects = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Icmp6OutRedirects = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_Icmp6InRedirects, Icmp6InRedirects); @@ -1203,8 +1203,8 @@ static void do_proc_net_snmp6(int update_every) { , RRDSET_TYPE_LINE ); - rd_InMLDv2Reports = rrddim_add(st, "InMLDv2Reports", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_OutMLDv2Reports = rrddim_add(st, "OutMLDv2Reports", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InMLDv2Reports = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutMLDv2Reports = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_InMLDv2Reports, Icmp6InMLDv2Reports); @@ -1865,8 +1865,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_AREA ); - rd_in = rrddim_add(st_system_ip, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st_system_ip, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_in = rrddim_add(st_system_ip, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_system_ip, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_system_ip, rd_in, ipext_InOctets); @@ -1900,8 +1900,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { rrdset_flag_set(st_ip_mcast, RRDSET_FLAG_DETAIL); - rd_in = rrddim_add(st_ip_mcast, "InMcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st_ip_mcast, "OutMcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_in = rrddim_add(st_ip_mcast, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_ip_mcast, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_ip_mcast, rd_in, ipext_InMcastOctets); @@ -1939,8 +1939,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { rrdset_flag_set(st_ip_bcast, RRDSET_FLAG_DETAIL); - rd_in = rrddim_add(st_ip_bcast, "InBcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st_ip_bcast, "OutBcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_in = rrddim_add(st_ip_bcast, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_ip_bcast, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_ip_bcast, rd_in, ipext_InBcastOctets); @@ -1978,8 +1978,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { rrdset_flag_set(st_ip_mcastpkts, RRDSET_FLAG_DETAIL); - rd_in = rrddim_add(st_ip_mcastpkts, "InMcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st_ip_mcastpkts, "OutMcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_in = rrddim_add(st_ip_mcastpkts, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_ip_mcastpkts, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_ip_mcastpkts, rd_in, ipext_InMcastPkts); @@ -2014,8 +2014,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { rrdset_flag_set(st_ip_bcastpkts, RRDSET_FLAG_DETAIL); - rd_in = rrddim_add(st_ip_bcastpkts, "InBcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st_ip_bcastpkts, "OutBcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_in = rrddim_add(st_ip_bcastpkts, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_ip_bcastpkts, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_ip_bcastpkts, rd_in, ipext_InBcastPkts); @@ -2253,9 +2253,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_LINE ); - rd_received = rrddim_add(st_syncookies, "SyncookiesRecv", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_sent = rrddim_add(st_syncookies, "SyncookiesSent", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_failed = rrddim_add(st_syncookies, "SyncookiesFailed", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_received = rrddim_add(st_syncookies, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st_syncookies, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_failed = rrddim_add(st_syncookies, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_syncookies, rd_received, tcpext_SyncookiesRecv); @@ -2369,10 +2369,10 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_LINE ); - rd_InReceives = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_OutRequests = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_ForwDatagrams = rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_InDelivers = rrddim_add(st, "InDelivers", "delivered", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InReceives = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutRequests = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ForwDatagrams = rrddim_add(st, "forwarded", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDelivers = rrddim_add(st, "delivered", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_OutRequests, (collected_number)snmp_root.ip_OutRequests); @@ -2557,8 +2557,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_LINE ); - rd_InMsgs = rrddim_add(st_packets, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_OutMsgs = rrddim_add(st_packets, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InMsgs = rrddim_add(st_packets, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutMsgs = rrddim_add(st_packets, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st_packets, rd_InMsgs, (collected_number)snmp_root.icmp_InMsgs); @@ -2773,8 +2773,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_LINE ); - rd_InSegs = rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_OutSegs = rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InSegs = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutSegs = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_InSegs, (collected_number)snmp_root.tcp_InSegs); @@ -2932,8 +2932,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_LINE ); - rd_InDatagrams = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_OutDatagrams = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDatagrams = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDatagrams = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_InDatagrams, (collected_number)snmp_root.udp_InDatagrams); @@ -3030,8 +3030,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , RRDSET_TYPE_LINE ); - rd_InDatagrams = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_OutDatagrams = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDatagrams = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDatagrams = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } rrddim_set_by_pointer(st, rd_InDatagrams, (collected_number)snmp_root.udplite_InDatagrams); diff --git a/src/collectors/proc.plugin/proc_stat.c b/src/collectors/proc.plugin/proc_stat.c index 481cb906a06dfd..bbd6723dc61317 100644 --- a/src/collectors/proc.plugin/proc_stat.c +++ b/src/collectors/proc.plugin/proc_stat.c @@ -484,7 +484,7 @@ int do_proc_stat(int update_every, usec_t dt) { *time_in_state_filename = NULL, *schedstat_filename = NULL, *cpuidle_name_filename = NULL, *cpuidle_time_filename = NULL; static const RRDVAR_ACQUIRED *cpus_var = NULL; static int accurate_freq_avail = 0, accurate_freq_is_used = 0; - size_t cores_found = (size_t)get_system_cpus(); + size_t cores_found = (size_t)os_get_system_cpus(); if(unlikely(do_cpu == -1)) { do_cpu = config_get_boolean("plugin:proc:/proc/stat", "cpu utilization", CONFIG_BOOLEAN_YES); @@ -495,7 +495,7 @@ int do_proc_stat(int update_every, usec_t dt) { do_processes = config_get_boolean("plugin:proc:/proc/stat", "processes running", CONFIG_BOOLEAN_YES); // give sane defaults based on the number of processors - if(unlikely(get_system_cpus() > 128)) { + if(unlikely(os_get_system_cpus() > 128)) { // the system has too many processors keep_per_core_fds_open = CONFIG_BOOLEAN_NO; do_core_throttle_count = CONFIG_BOOLEAN_NO; @@ -511,7 +511,7 @@ int do_proc_stat(int update_every, usec_t dt) { do_cpu_freq = CONFIG_BOOLEAN_YES; do_cpuidle = CONFIG_BOOLEAN_NO; } - if(unlikely(get_system_cpus() > 24)) { + if(unlikely(os_get_system_cpus() > 24)) { // the system has too many processors keep_cpuidle_fds_open = CONFIG_BOOLEAN_NO; } diff --git a/src/collectors/proc.plugin/proc_vmstat.c b/src/collectors/proc.plugin/proc_vmstat.c index b44733b6abb80c..a70ed48c1ac8ac 100644 --- a/src/collectors/proc.plugin/proc_vmstat.c +++ b/src/collectors/proc.plugin/proc_vmstat.c @@ -6,6 +6,10 @@ #define OOM_KILL_STRING "oom_kill" +#define _COMMON_PLUGIN_NAME PLUGIN_PROC_NAME +#define _COMMON_PLUGIN_MODULE_NAME PLUGIN_PROC_MODULE_VMSTAT_NAME +#include "../common-contexts/common-contexts.h" + int do_proc_vmstat(int update_every, usec_t dt) { (void)dt; @@ -328,34 +332,7 @@ int do_proc_vmstat(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_pgfaults) { - static RRDSET *st_pgfaults = NULL; - static RRDDIM *rd_minor = NULL, *rd_major = NULL; - - if(unlikely(!st_pgfaults)) { - st_pgfaults = rrdset_create_localhost( - "mem" - , "pgfaults" - , NULL - , "page faults" - , NULL - , "Memory Page Faults" - , "faults/s" - , PLUGIN_PROC_NAME - , PLUGIN_PROC_MODULE_VMSTAT_NAME - , NETDATA_CHART_PRIO_MEM_SYSTEM_PGFAULTS - , update_every - , RRDSET_TYPE_LINE - ); - - rrdset_flag_set(st_pgfaults, RRDSET_FLAG_DETAIL); - - rd_minor = rrddim_add(st_pgfaults, "minor", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_major = rrddim_add(st_pgfaults, "major", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - } - - rrddim_set_by_pointer(st_pgfaults, rd_minor, pgfault); - rrddim_set_by_pointer(st_pgfaults, rd_major, pgmajfault); - rrdset_done(st_pgfaults); + common_mem_pgfaults(pgfault, pgmajfault, update_every); } // -------------------------------------------------------------------- diff --git a/src/collectors/profile.plugin/plugin_profile.cc b/src/collectors/profile.plugin/plugin_profile.cc index f97c192a05d1bc..390bca29ead2ae 100644 --- a/src/collectors/profile.plugin/plugin_profile.cc +++ b/src/collectors/profile.plugin/plugin_profile.cc @@ -180,8 +180,10 @@ static void *subprofile_main(void* Arg) { return nullptr; } -static void profile_main_cleanup(void *ptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *) ptr; +static void profile_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_info("cleaning up..."); @@ -190,7 +192,7 @@ static void profile_main_cleanup(void *ptr) { } extern "C" void *profile_main(void *ptr) { - netdata_thread_cleanup_push(profile_main_cleanup, ptr); + CLEANUP_FUNCTION_REGISTER(profile_main_cleanup) cleanup_ptr = ptr; int UpdateEvery = (int) config_get_number(CONFIG_SECTION_PROFILE, "update every", 1); if (UpdateEvery < localhost->rrd_update_every) @@ -209,18 +211,18 @@ extern "C" void *profile_main(void *ptr) { Profilers.push_back(P); } - std::vector Threads(NumThreads); + std::vector Threads(NumThreads); for (size_t Idx = 0; Idx != NumThreads; Idx++) { char Tag[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "PROFILER[%zu]", Idx); - netdata_thread_create(&Threads[Idx], Tag, NETDATA_THREAD_OPTION_JOINABLE, subprofile_main, static_cast(&Profilers[Idx])); + Threads[Idx] = nd_thread_create(Tag, NETDATA_THREAD_OPTION_JOINABLE, + subprofile_main, static_cast(&Profilers[Idx])); } for (size_t Idx = 0; Idx != NumThreads; Idx++) - netdata_thread_join(Threads[Idx], nullptr); + nd_thread_join(Threads[Idx]); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/statsd.plugin/statsd.c b/src/collectors/statsd.plugin/statsd.c index b892133d26356e..f83818059fe278 100644 --- a/src/collectors/statsd.plugin/statsd.c +++ b/src/collectors/statsd.plugin/statsd.c @@ -237,7 +237,7 @@ struct collection_thread_status { bool running; uint32_t max_sockets; - netdata_thread_t thread; + ND_THREAD *thread; }; static struct statsd { @@ -1078,8 +1078,10 @@ static int statsd_snd_callback(POLLINFO *pi, short int *events) { // -------------------------------------------------------------------------------------------------------------------- // statsd child thread to collect metrics from network -void statsd_collector_thread_cleanup(void *data) { - struct statsd_udp *d = data; +void statsd_collector_thread_cleanup(void *pptr) { + struct statsd_udp *d = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!d) return; + spinlock_lock(&d->status->spinlock); d->status->running = false; spinlock_unlock(&d->status->spinlock); @@ -1115,12 +1117,12 @@ void *statsd_collector_thread(void *ptr) { worker_register_job_name(WORKER_JOB_TYPE_RCV_DATA, "receive"); worker_register_job_name(WORKER_JOB_TYPE_SND_DATA, "send"); - collector_info("STATSD collector thread started with taskid %d", gettid()); + collector_info("STATSD collector thread started with taskid %d", gettid_cached()); struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1); d->status = status; - netdata_thread_cleanup_push(statsd_collector_thread_cleanup, d); + CLEANUP_FUNCTION_REGISTER(statsd_collector_thread_cleanup) cleanup_ptr = d; #ifdef HAVE_RECVMMSG d->type = STATSD_SOCKET_DATA_TYPE_UDP; @@ -1154,7 +1156,6 @@ void *statsd_collector_thread(void *ptr) { , status->max_sockets ); - netdata_thread_cleanup_pop(1); return NULL; } @@ -2393,8 +2394,10 @@ static int statsd_listen_sockets_setup(void) { return listen_sockets_setup(&statsd.sockets); } -static void statsd_main_cleanup(void *data) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)data; +static void statsd_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); @@ -2402,13 +2405,14 @@ static void statsd_main_cleanup(void *data) { int i; for (i = 0; i < statsd.threads; i++) { spinlock_lock(&statsd.collection_threads_status[i].spinlock); + if(statsd.collection_threads_status[i].running) { - collector_info("STATSD: stopping data collection thread %d...", i + 1); - netdata_thread_cancel(statsd.collection_threads_status[i].thread); + collector_info("STATSD: signalling data collection thread %d to stop...", i + 1); + nd_thread_signal_cancel(statsd.collection_threads_status[i].thread); } - else { + else collector_info("STATSD: data collection thread %d found stopped.", i + 1); - } + spinlock_unlock(&statsd.collection_threads_status[i].spinlock); } } @@ -2445,6 +2449,8 @@ static void statsd_main_cleanup(void *data) { #endif void *statsd_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(statsd_main_cleanup) cleanup_ptr = ptr; + worker_register("STATSDFLUSH"); worker_register_job_name(WORKER_STATSD_FLUSH_GAUGES, "gauges"); worker_register_job_name(WORKER_STATSD_FLUSH_COUNTERS, "counters"); @@ -2455,8 +2461,6 @@ void *statsd_main(void *ptr) { worker_register_job_name(WORKER_STATSD_FLUSH_DICTIONARIES, "dictionaries"); worker_register_job_name(WORKER_STATSD_FLUSH_STATS, "statistics"); - netdata_thread_cleanup_push(statsd_main_cleanup, ptr); - statsd.gauges.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0); statsd.meters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0); statsd.counters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0); @@ -2585,7 +2589,8 @@ void *statsd_main(void *ptr) { char tag[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_IN[%d]", i + 1); spinlock_init(&statsd.collection_threads_status[i].spinlock); - netdata_thread_create(&statsd.collection_threads_status[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, statsd_collector_thread, &statsd.collection_threads_status[i]); + statsd.collection_threads_status[i].thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, + statsd_collector_thread, &statsd.collection_threads_status[i]); } // ---------------------------------------------------------------------------------------------------------------- @@ -2887,6 +2892,5 @@ void *statsd_main(void *ptr) { } cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/systemd-journal.plugin/systemd-main.c b/src/collectors/systemd-journal.plugin/systemd-main.c index 89052b9f9c15dc..e3afe4e86f8b75 100644 --- a/src/collectors/systemd-journal.plugin/systemd-main.c +++ b/src/collectors/systemd-journal.plugin/systemd-main.c @@ -19,7 +19,7 @@ static bool journal_data_directories_exist() { int main(int argc __maybe_unused, char **argv __maybe_unused) { clocks_init(); - netdata_thread_set_tag("sd-jrnl.plugin"); + nd_thread_tag_set("sd-jrnl.plugin"); nd_log_initialize_for_external_plugins("systemd-journal.plugin"); netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX"); @@ -67,9 +67,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { // ------------------------------------------------------------------------ // watcher thread - netdata_thread_t watcher_thread; - netdata_thread_create(&watcher_thread, "SDWATCH", - NETDATA_THREAD_OPTION_DONT_LOG, journal_watcher_main, NULL); + nd_thread_create("SDWATCH", NETDATA_THREAD_OPTION_DONT_LOG, journal_watcher_main, NULL); // ------------------------------------------------------------------------ // the event loop for functions diff --git a/src/collectors/tc.plugin/plugin_tc.c b/src/collectors/tc.plugin/plugin_tc.c index eae70453fb5646..ed8f1a8a4ee179 100644 --- a/src/collectors/tc.plugin/plugin_tc.c +++ b/src/collectors/tc.plugin/plugin_tc.c @@ -848,12 +848,13 @@ static inline void tc_split_words(char *str, char **words, int max_words) { static pid_t tc_child_pid = 0; -static void tc_main_cleanup(void *ptr) { - worker_unregister(); +static void tc_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + worker_unregister(); tc_device_index_destroy(); - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; collector_info("cleaning up..."); @@ -892,6 +893,8 @@ static void tc_main_cleanup(void *ptr) { #endif void *tc_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(tc_main_cleanup) cleanup_ptr = ptr; + worker_register("TC"); worker_register_job_name(WORKER_TC_CLASS, "class"); worker_register_job_name(WORKER_TC_BEGIN, "begin"); @@ -909,7 +912,6 @@ void *tc_main(void *ptr) { worker_register_job_custom_metric(WORKER_TC_CLASSES, "number of classes", "classes", WORKER_METRIC_ABSOLUTE); tc_device_index_init(); - netdata_thread_cleanup_push(tc_main_cleanup, ptr); char command[FILENAME_MAX + 1]; char *words[PLUGINSD_MAX_WORDS] = { NULL }; @@ -1036,10 +1038,8 @@ void *tc_main(void *ptr) { // netdata_log_debug(D_TC_LOOP, "END line"); if(likely(device)) { - netdata_thread_disable_cancelability(); tc_device_commit(device); // tc_device_free(device); - netdata_thread_enable_cancelability(); } device = NULL; @@ -1177,7 +1177,5 @@ void *tc_main(void *ptr) { } cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement - worker_unregister(); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/timex.plugin/plugin_timex.c b/src/collectors/timex.plugin/plugin_timex.c index 025b699a17eddc..ef4ffea5ca1037 100644 --- a/src/collectors/timex.plugin/plugin_timex.c +++ b/src/collectors/timex.plugin/plugin_timex.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "daemon/common.h" -#include "libnetdata/os.h" +#include "libnetdata/os/os.h" #define PLUGIN_TIMEX_NAME "timex.plugin" @@ -30,25 +30,25 @@ struct status_codes { {NULL, 0, NULL}, }; -static void timex_main_cleanup(void *ptr) +static void timex_main_cleanup(void *pptr) { - worker_unregister(); + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_info("cleaning up..."); + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } void *timex_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(timex_main_cleanup) cleanup_ptr = ptr; + worker_register("TIMEX"); worker_register_job_name(0, "clock check"); - netdata_thread_cleanup_push(timex_main_cleanup, ptr); - int update_every = (int)config_get_number(CONFIG_SECTION_TIMEX, "update every", 10); if (update_every < localhost->rrd_update_every) update_every = localhost->rrd_update_every; @@ -73,7 +73,7 @@ void *timex_main(void *ptr) int sync_state = 0; static int prev_sync_state = 0; - sync_state = ADJUST_TIMEX(&timex_buf); + sync_state = os_adjtimex(&timex_buf); int non_seq_failure = (sync_state == -1 && prev_sync_state != -1); prev_sync_state = sync_state; @@ -171,6 +171,5 @@ void *timex_main(void *ptr) } exit: - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/collectors/windows.plugin/GetSystemCPU.c b/src/collectors/windows.plugin/GetSystemCPU.c new file mode 100644 index 00000000000000..a7a0aca83a5cff --- /dev/null +++ b/src/collectors/windows.plugin/GetSystemCPU.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +int do_GetSystemCPU(int update_every, usec_t dt __maybe_unused) { + FILETIME idleTime, kernelTime, userTime; + + if(GetSystemTimes(&idleTime, &kernelTime, &userTime) == 0) { + netdata_log_error("GetSystemTimes() failed."); + return 1; + } + + ULONGLONG idle = FileTimeToULL(idleTime); + ULONGLONG kernel = FileTimeToULL(kernelTime); + ULONGLONG user = FileTimeToULL(userTime); + + // kernel includes idle + kernel -= idle; + + static RRDSET *st = NULL; + static RRDDIM *rd_user = NULL, *rd_kernel = NULL, *rd_idle = NULL; + if(!st) { + st = rrdset_create_localhost( + "system" + , "cpu" + , NULL + , "cpu" + , "system.cpu" + , "Total CPU utilization" + , "percentage" + , PLUGIN_WINDOWS_NAME + , "GetSystemTimes" + , NETDATA_CHART_PRIO_SYSTEM_CPU + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_user = rrddim_add(st, "user", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_kernel = rrddim_add(st, "system", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_idle = rrddim_add(st, "idle", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rrddim_hide(st, "idle"); + } + + rrddim_set_by_pointer(st, rd_user, (collected_number )user); + rrddim_set_by_pointer(st, rd_kernel, (collected_number )kernel); + rrddim_set_by_pointer(st, rd_idle, (collected_number )idle); + rrdset_done(st); + + return 0; +} diff --git a/src/collectors/windows.plugin/GetSystemRAM.c b/src/collectors/windows.plugin/GetSystemRAM.c new file mode 100644 index 00000000000000..9dae9a2529e3c5 --- /dev/null +++ b/src/collectors/windows.plugin/GetSystemRAM.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +#define _COMMON_PLUGIN_NAME "windows.plugin" +#define _COMMON_PLUGIN_MODULE_NAME "GetSystemRam" +#include "../common-contexts/common-contexts.h" + +int do_GetSystemRAM(int update_every, usec_t dt __maybe_unused) { + MEMORYSTATUSEX memStat = { 0 }; + memStat.dwLength = sizeof(memStat); + + if (!GlobalMemoryStatusEx(&memStat)) { + netdata_log_error("GlobalMemoryStatusEx() failed."); + return 1; + } + + { + ULONGLONG total_bytes = memStat.ullTotalPhys; + ULONGLONG free_bytes = memStat.ullAvailPhys; + ULONGLONG used_bytes = total_bytes - free_bytes; + common_system_ram(free_bytes, used_bytes, update_every); + } + + { + DWORDLONG total_bytes = memStat.ullTotalPageFile; + DWORDLONG free_bytes = memStat.ullAvailPageFile; + DWORDLONG used_bytes = total_bytes - free_bytes; + common_mem_swap(free_bytes, used_bytes, update_every); + } + + return 0; +} diff --git a/src/collectors/windows.plugin/GetSystemUptime.c b/src/collectors/windows.plugin/GetSystemUptime.c new file mode 100644 index 00000000000000..9ed939ca081254 --- /dev/null +++ b/src/collectors/windows.plugin/GetSystemUptime.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +int do_GetSystemUptime(int update_every, usec_t dt __maybe_unused) { + ULONGLONG uptime = GetTickCount64(); // in milliseconds + + static RRDSET *st = NULL; + static RRDDIM *rd_uptime = NULL; + if (!st) { + st = rrdset_create_localhost( + "system" + , "uptime" + , NULL + , "uptime" + , "system.uptime" + , "System Uptime" + , "seconds" + , PLUGIN_WINDOWS_NAME + , "GetSystemUptime" + , NETDATA_CHART_PRIO_SYSTEM_UPTIME + , update_every + , RRDSET_TYPE_LINE + ); + + rd_uptime = rrddim_add(st, "uptime", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + + rrddim_set_by_pointer(st, rd_uptime, (collected_number)uptime); + rrdset_done(st); + + return 0; +} diff --git a/src/collectors/windows.plugin/perflib-dump.c b/src/collectors/windows.plugin/perflib-dump.c new file mode 100644 index 00000000000000..e01813a49ffcdf --- /dev/null +++ b/src/collectors/windows.plugin/perflib-dump.c @@ -0,0 +1,529 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib.h" +#include "windows-internals.h" + +static const char *getCounterType(DWORD CounterType) { + switch (CounterType) { + case PERF_COUNTER_COUNTER: + return "PERF_COUNTER_COUNTER"; + + case PERF_COUNTER_TIMER: + return "PERF_COUNTER_TIMER"; + + case PERF_COUNTER_QUEUELEN_TYPE: + return "PERF_COUNTER_QUEUELEN_TYPE"; + + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + return "PERF_COUNTER_LARGE_QUEUELEN_TYPE"; + + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + return "PERF_COUNTER_100NS_QUEUELEN_TYPE"; + + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + return "PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE"; + + case PERF_COUNTER_BULK_COUNT: + return "PERF_COUNTER_BULK_COUNT"; + + case PERF_COUNTER_TEXT: + return "PERF_COUNTER_TEXT"; + + case PERF_COUNTER_RAWCOUNT: + return "PERF_COUNTER_RAWCOUNT"; + + case PERF_COUNTER_LARGE_RAWCOUNT: + return "PERF_COUNTER_LARGE_RAWCOUNT"; + + case PERF_COUNTER_RAWCOUNT_HEX: + return "PERF_COUNTER_RAWCOUNT_HEX"; + + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + return "PERF_COUNTER_LARGE_RAWCOUNT_HEX"; + + case PERF_SAMPLE_FRACTION: + return "PERF_SAMPLE_FRACTION"; + + case PERF_SAMPLE_COUNTER: + return "PERF_SAMPLE_COUNTER"; + + case PERF_COUNTER_NODATA: + return "PERF_COUNTER_NODATA"; + + case PERF_COUNTER_TIMER_INV: + return "PERF_COUNTER_TIMER_INV"; + + case PERF_SAMPLE_BASE: + return "PERF_SAMPLE_BASE"; + + case PERF_AVERAGE_TIMER: + return "PERF_AVERAGE_TIMER"; + + case PERF_AVERAGE_BASE: + return "PERF_AVERAGE_BASE"; + + case PERF_AVERAGE_BULK: + return "PERF_AVERAGE_BULK"; + + case PERF_OBJ_TIME_TIMER: + return "PERF_OBJ_TIME_TIMER"; + + case PERF_100NSEC_TIMER: + return "PERF_100NSEC_TIMER"; + + case PERF_100NSEC_TIMER_INV: + return "PERF_100NSEC_TIMER_INV"; + + case PERF_COUNTER_MULTI_TIMER: + return "PERF_COUNTER_MULTI_TIMER"; + + case PERF_COUNTER_MULTI_TIMER_INV: + return "PERF_COUNTER_MULTI_TIMER_INV"; + + case PERF_COUNTER_MULTI_BASE: + return "PERF_COUNTER_MULTI_BASE"; + + case PERF_100NSEC_MULTI_TIMER: + return "PERF_100NSEC_MULTI_TIMER"; + + case PERF_100NSEC_MULTI_TIMER_INV: + return "PERF_100NSEC_MULTI_TIMER_INV"; + + case PERF_RAW_FRACTION: + return "PERF_RAW_FRACTION"; + + case PERF_LARGE_RAW_FRACTION: + return "PERF_LARGE_RAW_FRACTION"; + + case PERF_RAW_BASE: + return "PERF_RAW_BASE"; + + case PERF_LARGE_RAW_BASE: + return "PERF_LARGE_RAW_BASE"; + + case PERF_ELAPSED_TIME: + return "PERF_ELAPSED_TIME"; + + case PERF_COUNTER_HISTOGRAM_TYPE: + return "PERF_COUNTER_HISTOGRAM_TYPE"; + + case PERF_COUNTER_DELTA: + return "PERF_COUNTER_DELTA"; + + case PERF_COUNTER_LARGE_DELTA: + return "PERF_COUNTER_LARGE_DELTA"; + + case PERF_PRECISION_SYSTEM_TIMER: + return "PERF_PRECISION_SYSTEM_TIMER"; + + case PERF_PRECISION_100NS_TIMER: + return "PERF_PRECISION_100NS_TIMER"; + + case PERF_PRECISION_OBJECT_TIMER: + return "PERF_PRECISION_OBJECT_TIMER"; + + default: + return "UNKNOWN_COUNTER_TYPE"; + } +} + +static const char *getCounterDescription(DWORD CounterType) { + switch (CounterType) { + case PERF_COUNTER_COUNTER: + return "32-bit Counter. Divide delta by delta time. Display suffix: \"/sec\""; + + case PERF_COUNTER_TIMER: + return "64-bit Timer. Divide delta by delta time. Display suffix: \"%\""; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + return "Queue Length Space-Time Product. Divide delta by delta time. No Display Suffix"; + + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + return "Queue Length Space-Time Product using 100 Ns timebase. Divide delta by delta time. No Display Suffix"; + + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + return "Queue Length Space-Time Product using Object specific timebase. Divide delta by delta time. No Display Suffix."; + + case PERF_COUNTER_BULK_COUNT: + return "64-bit Counter. Divide delta by delta time. Display Suffix: \"/sec\""; + + case PERF_COUNTER_TEXT: + return "Unicode text Display as text."; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + return "A counter which should not be time averaged on display (such as an error counter on a serial line). Display as is. No Display Suffix."; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + return "Special case for RAWCOUNT which should be displayed in hex. A counter which should not be time averaged on display (such as an error counter on a serial line). Display as is. No Display Suffix."; + + case PERF_SAMPLE_FRACTION: + return "A count which is either 1 or 0 on each sampling interrupt (% busy). Divide delta by delta base. Display Suffix: \"%\""; + + case PERF_SAMPLE_COUNTER: + return "A count which is sampled on each sampling interrupt (queue length). Divide delta by delta time. No Display Suffix."; + + case PERF_COUNTER_NODATA: + return "A label: no data is associated with this counter (it has 0 length). Do not display."; + + case PERF_COUNTER_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 - delta divided by delta time. Display suffix: \"%\""; + + case PERF_SAMPLE_BASE: + return "The divisor for a sample, used with the previous counter to form a sampled %. You must check for >0 before dividing by this! This counter will directly follow the numerator counter. It should not be displayed to the user."; + + case PERF_AVERAGE_TIMER: + return "A timer which, when divided by an average base, produces a time in seconds which is the average time of some operation. This timer times total operations, and the base is the number of operations. Display Suffix: \"sec\""; + + case PERF_AVERAGE_BASE: + return "Used as the denominator in the computation of time or count averages. Must directly follow the numerator counter. Not displayed to the user."; + + case PERF_AVERAGE_BULK: + return "A bulk count which, when divided (typically) by the number of operations, gives (typically) the number of bytes per operation. No Display Suffix."; + + case PERF_OBJ_TIME_TIMER: + return "64-bit Timer in object specific units. Display delta divided by delta time as returned in the object type header structure. Display suffix: \"%\""; + + case PERF_100NSEC_TIMER: + return "64-bit Timer in 100 nsec units. Display delta divided by delta time. Display suffix: \"%\""; + + case PERF_100NSEC_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 - delta divided by delta time. Display suffix: \"%\""; + + case PERF_COUNTER_MULTI_TIMER: + return "64-bit Timer. Divide delta by delta time. Display suffix: \"%\". Timer for multiple instances, so result can exceed 100%."; + + case PERF_COUNTER_MULTI_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 * _MULTI_BASE - delta divided by delta time. Display suffix: \"%\" Timer for multiple instances, so result can exceed 100%. Followed by a counter of type _MULTI_BASE."; + + case PERF_COUNTER_MULTI_BASE: + return "Number of instances to which the preceding _MULTI_..._INV counter applies. Used as a factor to get the percentage."; + + case PERF_100NSEC_MULTI_TIMER: + return "64-bit Timer in 100 nsec units. Display delta divided by delta time. Display suffix: \"%\" Timer for multiple instances, so result can exceed 100%."; + + case PERF_100NSEC_MULTI_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 * _MULTI_BASE - delta divided by delta time. Display suffix: \"%\" Timer for multiple instances, so result can exceed 100%. Followed by a counter of type _MULTI_BASE."; + + case PERF_LARGE_RAW_FRACTION: + case PERF_RAW_FRACTION: + return "Indicates the data is a fraction of the following counter which should not be time averaged on display (such as free space over total space.) Display as is. Display the quotient as \"%\""; + + case PERF_RAW_BASE: + case PERF_LARGE_RAW_BASE: + return "Indicates the data is a base for the preceding counter which should not be time averaged on display (such as free space over total space.)"; + + case PERF_ELAPSED_TIME: + return "The data collected in this counter is actually the start time of the item being measured. For display, this data is subtracted from the sample time to yield the elapsed time as the difference between the two. In the definition below, the PerfTime field of the Object contains the sample time as indicated by the PERF_OBJECT_TIMER bit and the difference is scaled by the PerfFreq of the Object to convert the time units into seconds."; + + case PERF_COUNTER_HISTOGRAM_TYPE: + return "Counter type can be used with the preceding types to define a range of values to be displayed in a histogram."; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + return "This counter is used to display the difference from one sample to the next. The counter value is a constantly increasing number and the value displayed is the difference between the current value and the previous value. Negative numbers are not allowed which shouldn't be a problem as long as the counter value is increasing or unchanged."; + + case PERF_PRECISION_SYSTEM_TIMER: + return "The precision counters are timers that consist of two counter values:\r\n\t1) the count of elapsed time of the event being monitored\r\n\t2) the \"clock\" time in the same units\r\nthe precision timers are used where the standard system timers are not precise enough for accurate readings. It's assumed that the service providing the data is also providing a timestamp at the same time which will eliminate any error that may occur since some small and variable time elapses between the time the system timestamp is captured and when the data is collected from the performance DLL. Only in extreme cases has this been observed to be problematic.\r\nwhen using this type of timer, the definition of the PERF_PRECISION_TIMESTAMP counter must immediately follow the definition of the PERF_PRECISION_*_TIMER in the Object header\r\nThe timer used has the same frequency as the System Performance Timer"; + + case PERF_PRECISION_100NS_TIMER: + return "The precision counters are timers that consist of two counter values:\r\n\t1) the count of elapsed time of the event being monitored\r\n\t2) the \"clock\" time in the same units\r\nthe precision timers are used where the standard system timers are not precise enough for accurate readings. It's assumed that the service providing the data is also providing a timestamp at the same time which will eliminate any error that may occur since some small and variable time elapses between the time the system timestamp is captured and when the data is collected from the performance DLL. Only in extreme cases has this been observed to be problematic.\r\nwhen using this type of timer, the definition of the PERF_PRECISION_TIMESTAMP counter must immediately follow the definition of the PERF_PRECISION_*_TIMER in the Object header\r\nThe timer used has the same frequency as the 100 NanoSecond Timer"; + + case PERF_PRECISION_OBJECT_TIMER: + return "The precision counters are timers that consist of two counter values:\r\n\t1) the count of elapsed time of the event being monitored\r\n\t2) the \"clock\" time in the same units\r\nthe precision timers are used where the standard system timers are not precise enough for accurate readings. It's assumed that the service providing the data is also providing a timestamp at the same time which will eliminate any error that may occur since some small and variable time elapses between the time the system timestamp is captured and when the data is collected from the performance DLL. Only in extreme cases has this been observed to be problematic.\r\nwhen using this type of timer, the definition of the PERF_PRECISION_TIMESTAMP counter must immediately follow the definition of the PERF_PRECISION_*_TIMER in the Object header\r\nThe timer used is of the frequency specified in the Object header's. PerfFreq field (PerfTime is ignored)"; + + default: + return ""; + } +} + +static const char *getCounterAlgorithm(DWORD CounterType) { + switch (CounterType) + { + case PERF_COUNTER_COUNTER: + case PERF_SAMPLE_COUNTER: + case PERF_COUNTER_BULK_COUNT: + return "(data1 - data0) / ((time1 - time0) / frequency)"; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + case PERF_AVERAGE_BULK: // normally not displayed + return "(data1 - data0) / (time1 - time0)"; + + case PERF_OBJ_TIME_TIMER: + case PERF_COUNTER_TIMER: + case PERF_100NSEC_TIMER: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + case PERF_SAMPLE_FRACTION: + return "100 * (data1 - data0) / (time1 - time0)"; + + case PERF_COUNTER_TIMER_INV: + return "100 * (1 - ((data1 - data0) / (time1 - time0)))"; + + case PERF_100NSEC_TIMER_INV: + return "100 * (1- (data1 - data0) / (time1 - time0))"; + + case PERF_COUNTER_MULTI_TIMER: + return "100 * ((data1 - data0) / ((time1 - time0) / frequency1)) / multi1"; + + case PERF_100NSEC_MULTI_TIMER: + return "100 * ((data1 - data0) / (time1 - time0)) / multi1"; + + case PERF_COUNTER_MULTI_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER_INV: + return "100 * (multi1 - ((data1 - data0) / (time1 - time0)))"; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + return "data0"; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + return "hex(data0)"; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + return "data1 - data0"; + + case PERF_RAW_FRACTION: + case PERF_LARGE_RAW_FRACTION: + return "100 * data0 / time0"; + + case PERF_AVERAGE_TIMER: + return "((data1 - data0) / frequency1) / (time1 - time0)"; + + case PERF_ELAPSED_TIME: + return "(time0 - data0) / frequency0"; + + case PERF_COUNTER_TEXT: + case PERF_SAMPLE_BASE: + case PERF_AVERAGE_BASE: + case PERF_COUNTER_MULTI_BASE: + case PERF_RAW_BASE: + case PERF_COUNTER_NODATA: + case PERF_PRECISION_TIMESTAMP: + default: + return ""; + } +} + +void dumpSystemTime(BUFFER *wb, SYSTEMTIME *st) { + buffer_json_member_add_uint64(wb, "Year", st->wYear); + buffer_json_member_add_uint64(wb, "Month", st->wMonth); + buffer_json_member_add_uint64(wb, "DayOfWeek", st->wDayOfWeek); + buffer_json_member_add_uint64(wb, "Day", st->wDay); + buffer_json_member_add_uint64(wb, "Hour", st->wHour); + buffer_json_member_add_uint64(wb, "Minute", st->wMinute); + buffer_json_member_add_uint64(wb, "Second", st->wSecond); + buffer_json_member_add_uint64(wb, "Milliseconds", st->wMilliseconds); +} + +bool dumpDataCb(PERF_DATA_BLOCK *pDataBlock, void *data) { + char name[4096]; + if(!getSystemName(pDataBlock, name, sizeof(name))) + strncpyz(name, "[failed]", sizeof(name) - 1); + + BUFFER *wb = data; + buffer_json_member_add_string(wb, "SystemName", name); + + // Number of types of objects being reported + // Type: DWORD + buffer_json_member_add_int64(wb, "NumObjectTypes", pDataBlock->NumObjectTypes); + + buffer_json_member_add_int64(wb, "LittleEndian", pDataBlock->LittleEndian); + + // Version and Revision of these data structures. + // Version starts at 1. + // Revision starts at 0 for each Version. + // Type: DWORD + buffer_json_member_add_int64(wb, "Version", pDataBlock->Version); + buffer_json_member_add_int64(wb, "Revision", pDataBlock->Revision); + + // Object Title Index of default object to display when data from this system is retrieved + // (-1 = none, but this is not expected to be used) + // Type: LONG + buffer_json_member_add_int64(wb, "DefaultObject", pDataBlock->DefaultObject); + + // Performance counter frequency at the system under measurement + // Type: LARGE_INTEGER + buffer_json_member_add_int64(wb, "PerfFreq", pDataBlock->PerfFreq.QuadPart); + + // Performance counter value at the system under measurement + // Type: LARGE_INTEGER + buffer_json_member_add_int64(wb, "PerfTime", pDataBlock->PerfTime.QuadPart); + + // Performance counter time in 100 nsec units at the system under measurement + // Type: LARGE_INTEGER + buffer_json_member_add_int64(wb, "PerfTime100nSec", pDataBlock->PerfTime100nSec.QuadPart); + + // Time at the system under measurement in UTC + // Type: SYSTEMTIME + buffer_json_member_add_object(wb, "SystemTime"); + dumpSystemTime(wb, &pDataBlock->SystemTime); + buffer_json_object_close(wb); + + if(pDataBlock->NumObjectTypes) + buffer_json_member_add_array(wb, "Objects"); + + return true; +} + +static const char *GetDetailLevel(DWORD num) { + switch (num) { + case 100: + return "Novice (100)"; + case 200: + return "Advanced (200)"; + case 300: + return "Expert (300)"; + case 400: + return "Wizard (400)"; + + default: + return "Unknown"; + } +} + +bool dumpObjectCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, void *data) { + (void)pDataBlock; + BUFFER *wb = data; + if(!pObjectType) { + buffer_json_array_close(wb); // instances or counters + buffer_json_object_close(wb); // objectType + return true; + } + + buffer_json_add_array_item_object(wb); // objectType + buffer_json_member_add_int64(wb, "NameId", pObjectType->ObjectNameTitleIndex); + buffer_json_member_add_string(wb, "Name", RegistryFindNameByID(pObjectType->ObjectNameTitleIndex)); + buffer_json_member_add_int64(wb, "HelpId", pObjectType->ObjectHelpTitleIndex); + buffer_json_member_add_string(wb, "Help", RegistryFindHelpByID(pObjectType->ObjectHelpTitleIndex)); + buffer_json_member_add_int64(wb, "NumInstances", pObjectType->NumInstances); + buffer_json_member_add_int64(wb, "NumCounters", pObjectType->NumCounters); + buffer_json_member_add_int64(wb, "PerfTime", pObjectType->PerfTime.QuadPart); + buffer_json_member_add_int64(wb, "PerfFreq", pObjectType->PerfFreq.QuadPart); + buffer_json_member_add_int64(wb, "CodePage", pObjectType->CodePage); + buffer_json_member_add_int64(wb, "DefaultCounter", pObjectType->DefaultCounter); + buffer_json_member_add_string(wb, "DetailLevel", GetDetailLevel(pObjectType->DetailLevel)); + + if(ObjectTypeHasInstances(pDataBlock, pObjectType)) + buffer_json_member_add_array(wb, "Instances"); + else + buffer_json_member_add_array(wb, "Counters"); + + return true; +} + +bool dumpInstanceCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, void *data) { + (void)pDataBlock; + BUFFER *wb = data; + if(!pInstance) { + buffer_json_array_close(wb); // counters + buffer_json_object_close(wb); // instance + return true; + } + + char name[4096]; + if(!getInstanceName(pDataBlock, pObjectType, pInstance, name, sizeof(name))) + strncpyz(name, "[failed]", sizeof(name) - 1); + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "Instance", name); + buffer_json_member_add_int64(wb, "UniqueID", pInstance->UniqueID); + buffer_json_member_add_array(wb, "Labels"); + { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "key", RegistryFindNameByID(pObjectType->ObjectNameTitleIndex)); + buffer_json_member_add_string(wb, "value", name); + } + buffer_json_object_close(wb); + + if(pInstance->ParentObjectTitleIndex) { + PERF_INSTANCE_DEFINITION *pi = pInstance; + while(pi->ParentObjectTitleIndex) { + PERF_OBJECT_TYPE *po = getObjectTypeByIndex(pDataBlock, pInstance->ParentObjectTitleIndex); + pi = getInstanceByPosition(pDataBlock, po, pi->ParentObjectInstance); + + if(!getInstanceName(pDataBlock, po, pi, name, sizeof(name))) + strncpyz(name, "[failed]", sizeof(name) - 1); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "key", RegistryFindNameByID(po->ObjectNameTitleIndex)); + buffer_json_member_add_string(wb, "value", name); + } + buffer_json_object_close(wb); + } + } + } + buffer_json_array_close(wb); // rrdlabels + + buffer_json_member_add_array(wb, "Counters"); + return true; +} + +void dumpSample(BUFFER *wb, RAW_DATA *d) { + buffer_json_member_add_object(wb, "Value"); + buffer_json_member_add_uint64(wb, "data", d->Data); + buffer_json_member_add_int64(wb, "time", d->Time); + buffer_json_member_add_uint64(wb, "type", d->CounterType); + buffer_json_member_add_int64(wb, "multi", d->MultiCounterData); + buffer_json_member_add_int64(wb, "frequency", d->Frequency); + buffer_json_object_close(wb); +} + +bool dumpCounterCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data) { + (void)pDataBlock; + (void)pObjectType; + BUFFER *wb = data; + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "Counter", RegistryFindNameByID(pCounter->CounterNameTitleIndex)); + dumpSample(wb, sample); + buffer_json_member_add_string(wb, "Help", RegistryFindHelpByID(pCounter->CounterHelpTitleIndex)); + buffer_json_member_add_string(wb, "Type", getCounterType(pCounter->CounterType)); + buffer_json_member_add_string(wb, "Algorithm", getCounterAlgorithm(pCounter->CounterType)); + buffer_json_member_add_string(wb, "Description", getCounterDescription(pCounter->CounterType)); + buffer_json_object_close(wb); + return true; +} + +bool dumpInstanceCounterCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data) { + (void)pInstance; + return dumpCounterCb(pDataBlock, pObjectType, pCounter, sample, data); +} + + +int windows_perflib_dump(const char *key) { + if(key && !*key) + key = NULL; + + PerflibNamesRegistryInitialize(); + + DWORD id = 0; + if(key) { + id = RegistryFindIDByName(key); + if(id == PERFLIB_REGISTRY_NAME_NOT_FOUND) { + fprintf(stderr, "Cannot find key '%s' in Windows Performance Counters Registry.\n", key); + exit(1); + } + } + + CLEAN_BUFFER *wb = buffer_create(0, NULL); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + + perflibQueryAndTraverse(id, dumpDataCb, dumpObjectCb, dumpInstanceCb, dumpInstanceCounterCb, dumpCounterCb, wb); + + buffer_json_finalize(wb); + printf("\n%s\n", buffer_tostring(wb)); + + perflibFreePerformanceData(); + + return 0; +} diff --git a/src/collectors/windows.plugin/perflib-memory.c b/src/collectors/windows.plugin/perflib-memory.c new file mode 100644 index 00000000000000..91d27357f4f49f --- /dev/null +++ b/src/collectors/windows.plugin/perflib-memory.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +#define _COMMON_PLUGIN_NAME "windows.plugin" +#define _COMMON_PLUGIN_MODULE_NAME "PerflibMemory" +#include "../common-contexts/common-contexts.h" + +static void initialize(void) { + ; +} + +static bool do_memory(PERF_DATA_BLOCK *pDataBlock, int update_every) { + PERF_OBJECT_TYPE *pObjectType = perflibFindObjectTypeByName(pDataBlock, "Memory"); + if (!pObjectType) + return false; + + static COUNTER_DATA pagesPerSec = { .key = "Pages/sec" }; + static COUNTER_DATA pageFaultsPerSec = { .key = "Page Faults/sec" }; + + if(perflibGetObjectCounter(pDataBlock, pObjectType, &pageFaultsPerSec) && + perflibGetObjectCounter(pDataBlock, pObjectType, &pagesPerSec)) { + ULONGLONG total = pageFaultsPerSec.current.Data; + ULONGLONG major = pagesPerSec.current.Data; + ULONGLONG minor = (total > major) ? total - major : 0; + common_mem_pgfaults(minor, major, update_every); + } + + static COUNTER_DATA availableBytes = { .key = "Available Bytes" }; + static COUNTER_DATA availableKBytes = { .key = "Available KBytes" }; + static COUNTER_DATA availableMBytes = { .key = "Available MBytes" }; + ULONGLONG available_bytes = 0; + + if(perflibGetObjectCounter(pDataBlock, pObjectType, &availableBytes)) + available_bytes = availableBytes.current.Data; + else if(perflibGetObjectCounter(pDataBlock, pObjectType, &availableKBytes)) + available_bytes = availableKBytes.current.Data * 1024; + else if(perflibGetObjectCounter(pDataBlock, pObjectType, &availableMBytes)) + available_bytes = availableMBytes.current.Data * 1024; + + common_mem_available(available_bytes, update_every); + + return true; +} + +int do_PerflibMemory(int update_every, usec_t dt __maybe_unused) { + static bool initialized = false; + + if(unlikely(!initialized)) { + initialize(); + initialized = true; + } + + DWORD id = RegistryFindIDByName("Memory"); + if(id == PERFLIB_REGISTRY_NAME_NOT_FOUND) + return -1; + + PERF_DATA_BLOCK *pDataBlock = perflibGetPerformanceData(id); + if(!pDataBlock) return -1; + + do_memory(pDataBlock, update_every); + + return 0; +} diff --git a/src/collectors/windows.plugin/perflib-names.c b/src/collectors/windows.plugin/perflib-names.c new file mode 100644 index 00000000000000..5b47cbce79fe36 --- /dev/null +++ b/src/collectors/windows.plugin/perflib-names.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib.h" + +#define REGISTRY_KEY "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Perflib\\009" + +typedef struct perflib_registry { + DWORD id; + char *key; + char *help; +} perfLibRegistryEntry; + +static inline bool compare_perfLibRegistryEntry(const char *k1, const char *k2) { + return strcmp(k1, k2) == 0; +} + +static inline const char *value2key_perfLibRegistryEntry(perfLibRegistryEntry *entry) { + return entry->key; +} + +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compare_perfLibRegistryEntry +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION value2key_perfLibRegistryEntry +#define SIMPLE_HASHTABLE_KEY_TYPE const char +#define SIMPLE_HASHTABLE_VALUE_TYPE perfLibRegistryEntry +#define SIMPLE_HASHTABLE_NAME _PERFLIB +#include "libnetdata/simple_hashtable.h" + +static struct { + SPINLOCK spinlock; + size_t size; + perfLibRegistryEntry **array; + struct simple_hashtable_PERFLIB hashtable; + FILETIME lastWriteTime; +} names_globals = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .size = 0, + .array = NULL, +}; + +DWORD RegistryFindIDByName(const char *name) { + DWORD rc = PERFLIB_REGISTRY_NAME_NOT_FOUND; + + spinlock_lock(&names_globals.spinlock); + XXH64_hash_t hash = XXH3_64bits((void *)name, strlen(name)); + SIMPLE_HASHTABLE_SLOT_PERFLIB *sl = simple_hashtable_get_slot_PERFLIB(&names_globals.hashtable, hash, name, false); + perfLibRegistryEntry *e = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(e) rc = e->id; + spinlock_unlock(&names_globals.spinlock); + + return rc; +} + +static inline void RegistryAddToHashTable_unsafe(perfLibRegistryEntry *entry) { + XXH64_hash_t hash = XXH3_64bits((void *)entry->key, strlen(entry->key)); + SIMPLE_HASHTABLE_SLOT_PERFLIB *sl = simple_hashtable_get_slot_PERFLIB(&names_globals.hashtable, hash, entry->key, true); + perfLibRegistryEntry *e = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!e || e->id > entry->id) + simple_hashtable_set_slot_PERFLIB(&names_globals.hashtable, sl, hash, entry); +} + +static void RegistrySetData_unsafe(DWORD id, const char *key, const char *help) { + if(id >= names_globals.size) { + // increase the size of the array + + size_t old_size = names_globals.size; + + if(!names_globals.size) + names_globals.size = 20000; + else + names_globals.size *= 2; + + names_globals.array = reallocz(names_globals.array, names_globals.size * sizeof(perfLibRegistryEntry *)); + + memset(names_globals.array + old_size, 0, (names_globals.size - old_size) * sizeof(perfLibRegistryEntry *)); + } + + perfLibRegistryEntry *entry = names_globals.array[id]; + if(!entry) + entry = names_globals.array[id] = (perfLibRegistryEntry *)calloc(1, sizeof(perfLibRegistryEntry)); + + bool add_to_hash = false; + if(key && !entry->key) { + entry->key = strdup(key); + add_to_hash = true; + } + + if(help && !entry->help) + entry->help = strdup(help); + + entry->id = id; + + if(add_to_hash) + RegistryAddToHashTable_unsafe(entry); +} + +const char *RegistryFindNameByID(DWORD id) { + const char *s = ""; + spinlock_lock(&names_globals.spinlock); + + if(id < names_globals.size) { + perfLibRegistryEntry *titleEntry = names_globals.array[id]; + if(titleEntry && titleEntry->key) + s = titleEntry->key; + } + + spinlock_unlock(&names_globals.spinlock); + return s; +} + +const char *RegistryFindHelpByID(DWORD id) { + const char *s = ""; + spinlock_lock(&names_globals.spinlock); + + if(id < names_globals.size) { + perfLibRegistryEntry *titleEntry = names_globals.array[id]; + if(titleEntry && titleEntry->help) + s = titleEntry->help; + } + + spinlock_unlock(&names_globals.spinlock); + return s; +} + +// ---------------------------------------------------------- + +static inline void readRegistryKeys_unsafe(BOOL helps) { + TCHAR *pData = NULL; + + HKEY hKey; + DWORD dwType; + DWORD dwSize = 0; + LONG lStatus; + + LPCSTR valueName; + if(helps) + valueName = TEXT("help"); + else + valueName = TEXT("CounterDefinition"); + + // Open the key for the English counters + lStatus = RegOpenKeyEx(HKEY_LOCAL_MACHINE, TEXT(REGISTRY_KEY), 0, KEY_READ, &hKey); + if (lStatus != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to open registry key HKEY_LOCAL_MACHINE, subkey '%s', error %ld\n", REGISTRY_KEY, (long)lStatus); + return; + } + + // Get the size of the 'Counters' data + lStatus = RegQueryValueEx(hKey, valueName, NULL, &dwType, NULL, &dwSize); + if (lStatus != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to get registry key HKEY_LOCAL_MACHINE, subkey '%s', value '%s', size of data, error %ld\n", + REGISTRY_KEY, (const char *)valueName, (long)lStatus); + goto cleanup; + } + + // Allocate memory for the data + pData = mallocz(dwSize); + + // Read the 'Counters' data + lStatus = RegQueryValueEx(hKey, valueName, NULL, &dwType, (LPBYTE)pData, &dwSize); + if (lStatus != ERROR_SUCCESS || dwType != REG_MULTI_SZ) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to get registry key HKEY_LOCAL_MACHINE, subkey '%s', value '%s', data, error %ld\n", + REGISTRY_KEY, (const char *)valueName, (long)lStatus); + goto cleanup; + } + + // Process the counter data + TCHAR *ptr = pData; + while (*ptr) { + TCHAR *sid = ptr; // First string is the ID + ptr += lstrlen(ptr) + 1; // Move to the next string + TCHAR *name = ptr; // Second string is the name + ptr += lstrlen(ptr) + 1; // Move to the next pair + + DWORD id = strtoul(sid, NULL, 10); + + if(helps) + RegistrySetData_unsafe(id, NULL, name); + else + RegistrySetData_unsafe(id, name, NULL); + } + +cleanup: + if(pData) freez(pData); + RegCloseKey(hKey); +} + +static BOOL RegistryKeyModification(FILETIME *lastWriteTime) { + HKEY hKey; + LONG lResult; + BOOL ret = FALSE; + + // Open the registry key + lResult = RegOpenKeyEx(HKEY_LOCAL_MACHINE, TEXT(REGISTRY_KEY), 0, KEY_READ, &hKey); + if (lResult != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to open registry key HKEY_LOCAL_MACHINE, subkey '%s', error %ld\n", REGISTRY_KEY, (long)lResult); + return FALSE; + } + + // Get the last write time + lResult = RegQueryInfoKey(hKey, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, lastWriteTime); + if (lResult != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to query registry key HKEY_LOCAL_MACHINE, subkey '%s', last write time, error %ld\n", REGISTRY_KEY, (long)lResult); + ret = FALSE; + } + else + ret = TRUE; + + RegCloseKey(hKey); + return ret; +} + +static inline void RegistryFetchAll_unsafe(void) { + readRegistryKeys_unsafe(FALSE); + readRegistryKeys_unsafe(TRUE); +} + +void PerflibNamesRegistryInitialize(void) { + spinlock_lock(&names_globals.spinlock); + simple_hashtable_init_PERFLIB(&names_globals.hashtable, 20000); + RegistryKeyModification(&names_globals.lastWriteTime); + RegistryFetchAll_unsafe(); + spinlock_unlock(&names_globals.spinlock); +} + +void PerflibNamesRegistryUpdate(void) { + FILETIME lastWriteTime = { 0 }; + RegistryKeyModification(&lastWriteTime); + + if(CompareFileTime(&lastWriteTime, &names_globals.lastWriteTime) > 0) { + spinlock_lock(&names_globals.spinlock); + if(CompareFileTime(&lastWriteTime, &names_globals.lastWriteTime) > 0) { + names_globals.lastWriteTime = lastWriteTime; + RegistryFetchAll_unsafe(); + } + spinlock_unlock(&names_globals.spinlock); + } +} diff --git a/src/collectors/windows.plugin/perflib-network.c b/src/collectors/windows.plugin/perflib-network.c new file mode 100644 index 00000000000000..2f1bc3c531fc3c --- /dev/null +++ b/src/collectors/windows.plugin/perflib-network.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +// -------------------------------------------------------------------------------------------------------------------- +// network protocols + +struct network_protocol { + const char *protocol; + + struct { + COUNTER_DATA received; + COUNTER_DATA sent; + COUNTER_DATA delivered; + COUNTER_DATA forwarded; + RRDSET *st; + RRDDIM *rd_received; + RRDDIM *rd_sent; + RRDDIM *rd_forwarded; + RRDDIM *rd_delivered; + const char *type; + const char *id; + const char *family; + const char *context; + const char *title; + long priority; + } packets; + +} networks[] = { + { + .protocol = "IPv4", + .packets = { + .received = { .key = "Datagrams Received/sec" }, + .sent = { .key = "Datagrams Sent/sec" }, + .delivered = { .key = "Datagrams Received Delivered/sec" }, + .forwarded = { .key = "Datagrams Forwarded/sec" }, + .type = "ipv4", + .id = "packets", + .family = "packets", + .context = "ipv4.packets", + .title = "IPv4 Packets", + .priority = NETDATA_CHART_PRIO_IPV4_PACKETS, + }, + }, + { + .protocol = "IPv6", + .packets = { + .received = { .key = "Datagrams Received/sec" }, + .sent = { .key = "Datagrams Sent/sec" }, + .delivered = { .key = "Datagrams Received Delivered/sec" }, + .forwarded = { .key = "Datagrams Forwarded/sec" }, + .type = "ipv6", + .id = "packets", + .family = "packets", + .context = "ip6.packets", + .title = "IPv6 Packets", + .priority = NETDATA_CHART_PRIO_IPV6_PACKETS, + }, + }, + { + .protocol = "TCPv4", + .packets = { + .received = { .key = "Segments Received/sec" }, + .sent = { .key = "Segments Sent/sec" }, + .type = "ipv4", + .id = "tcppackets", + .family = "tcp", + .context = "ipv4.tcppackets", + .title = "IPv4 TCP Packets", + .priority = NETDATA_CHART_PRIO_IPV4_TCP_PACKETS, + }, + }, + { + .protocol = "TCPv6", + .packets = { + .received = { .key = "Segments Received/sec" }, + .sent = { .key = "Segments Sent/sec" }, + .type = "ipv6", + .id = "tcppackets", + .family = "tcp6", + .context = "ipv6.tcppackets", + .title = "IPv6 TCP Packets", + .priority = NETDATA_CHART_PRIO_IPV6_TCP_PACKETS, + }, + }, + { + .protocol = "UDPv4", + .packets = { + .received = { .key = "Datagrams Received/sec" }, + .sent = { .key = "Datagrams Sent/sec" }, + .type = "ipv4", + .id = "udppackets", + .family = "udp", + .context = "ipv4.udppackets", + .title = "IPv4 UDP Packets", + .priority = NETDATA_CHART_PRIO_IPV4_UDP_PACKETS, + }, + }, + { + .protocol = "UDPv6", + .packets = { + .received = { .key = "Datagrams Received/sec" }, + .sent = { .key = "Datagrams Sent/sec" }, + .type = "ipv6", + .id = "udppackets", + .family = "udp6", + .context = "ipv6.udppackets", + .title = "IPv6 UDP Packets", + .priority = NETDATA_CHART_PRIO_IPV6_UDP_PACKETS, + }, + }, + { + .protocol = "ICMP", + .packets = { + .received = { .key = "Messages Received/sec" }, + .sent = { .key = "Messages Sent/sec" }, + .type = "ipv4", + .id = "icmp", + .family = "icmp", + .context = "ipv4.icmp", + .title = "IPv4 ICMP Packets", + .priority = NETDATA_CHART_PRIO_IPV4_ICMP_PACKETS, + }, + }, + { + .protocol = "ICMPv6", + .packets = { + .received = { .key = "Messages Received/sec" }, + .sent = { .key = "Messages Sent/sec" }, + .type = "ipv6", + .id = "icmp", + .family = "icmp6", + .context = "ipv6.icmp", + .title = "IPv6 ICMP Packets", + .priority = NETDATA_CHART_PRIO_IPV6_ICMP_PACKETS, + }, + }, + + // terminator + { + .protocol = NULL, + } +}; + +struct network_protocol tcp46 = { + .packets = { + .type = "ip", + .id = "tcppackets", + .family = "tcp", + .context = "ip.tcppackets", + .title = "TCP Packets", + .priority = NETDATA_CHART_PRIO_IP_TCP_PACKETS, + } +}; + +static void protocol_packets_chart_update(struct network_protocol *p, int update_every) { + if(!p->packets.st) { + p->packets.st = rrdset_create_localhost( + p->packets.type + , p->packets.id + , NULL + , p->packets.family + , NULL + , p->packets.title + , "packets/s" + , PLUGIN_WINDOWS_NAME + , "PerflibNetwork" + , p->packets.priority + , update_every + , RRDSET_TYPE_AREA + ); + + p->packets.rd_received = rrddim_add(p->packets.st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + p->packets.rd_sent = rrddim_add(p->packets.st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + + if(p->packets.forwarded.key) + p->packets.rd_forwarded = rrddim_add(p->packets.st, "forwarded", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + + if(p->packets.delivered.key) + p->packets.rd_delivered = rrddim_add(p->packets.st, "delivered", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + + if(p->packets.received.updated) + rrddim_set_by_pointer(p->packets.st, p->packets.rd_received, (collected_number)p->packets.received.current.Data); + + if(p->packets.sent.updated) + rrddim_set_by_pointer(p->packets.st, p->packets.rd_sent, (collected_number)p->packets.sent.current.Data); + + if(p->packets.forwarded.key && p->packets.forwarded.updated) + rrddim_set_by_pointer(p->packets.st, p->packets.rd_forwarded, (collected_number)p->packets.forwarded.current.Data); + + if(p->packets.delivered.key && p->packets.delivered.updated) + rrddim_set_by_pointer(p->packets.st, p->packets.rd_delivered, (collected_number)p->packets.delivered.current.Data); + + rrdset_done(p->packets.st); +} + +static bool do_network_protocol(PERF_DATA_BLOCK *pDataBlock, int update_every, struct network_protocol *p) { + if(!p || !p->protocol) return false; + + PERF_OBJECT_TYPE *pObjectType = perflibFindObjectTypeByName(pDataBlock, p->protocol); + if(!pObjectType) return false; + + size_t packets = 0; + if(p->packets.received.key) + packets += perflibGetObjectCounter(pDataBlock, pObjectType, &p->packets.received) ? 1 : 0; + + if(p->packets.sent.key) + packets += perflibGetObjectCounter(pDataBlock, pObjectType, &p->packets.sent) ? 1 : 0; + + if(p->packets.delivered.key) + packets += perflibGetObjectCounter(pDataBlock, pObjectType, &p->packets.delivered) ? 1 :0; + + if(p->packets.forwarded.key) + packets += perflibGetObjectCounter(pDataBlock, pObjectType, &p->packets.forwarded) ? 1 : 0; + + if(packets) + protocol_packets_chart_update(p, update_every); + + return true; +} + +// -------------------------------------------------------------------------------------------------------------------- +// network interfaces + +struct network_interface { + bool collected_metadata; + + struct { + COUNTER_DATA received; + COUNTER_DATA sent; + + RRDSET *st; + RRDDIM *rd_received; + RRDDIM *rd_sent; + } packets; + + struct { + COUNTER_DATA received; + COUNTER_DATA sent; + + RRDSET *st; + RRDDIM *rd_received; + RRDDIM *rd_sent; + } traffic; +}; + +static DICTIONARY *physical_interfaces = NULL, *virtual_interfaces = NULL; + +static void network_interface_init(struct network_interface *ni) { + ni->packets.received.key = "Packets Received/sec"; + ni->packets.sent.key = "Packets Sent/sec"; + + ni->traffic.received.key = "Bytes Received/sec"; + ni->traffic.sent.key = "Bytes Sent/sec"; +} + +void dict_interface_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct network_interface *ni = value; + network_interface_init(ni); +} + +static void initialize(void) { + physical_interfaces = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | + DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct network_interface)); + + virtual_interfaces = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | + DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct network_interface)); + + dictionary_register_insert_callback(physical_interfaces, dict_interface_insert_cb, NULL); + dictionary_register_insert_callback(virtual_interfaces, dict_interface_insert_cb, NULL); +} + +static void add_interface_labels(RRDSET *st, const char *name, bool physical) { + rrdlabels_add(st->rrdlabels, "device", name, RRDLABEL_SRC_AUTO); + rrdlabels_add(st->rrdlabels, "interface_type", physical ? "real" : "virtual", RRDLABEL_SRC_AUTO); +} + +static bool is_physical_interface(const char *name) { + void *d = dictionary_get(physical_interfaces, name); + return d ? true : false; +} + +static bool do_network_interface(PERF_DATA_BLOCK *pDataBlock, int update_every, bool physical) { + DICTIONARY *dict = physical_interfaces; + + PERF_OBJECT_TYPE *pObjectType = perflibFindObjectTypeByName(pDataBlock, physical ? "Network Interface" : "Network Adapter"); + if(!pObjectType) return false; + + uint64_t total_received = 0, total_sent = 0; + + PERF_INSTANCE_DEFINITION *pi = NULL; + for(LONG i = 0; i < pObjectType->NumInstances ; i++) { + pi = perflibForEachInstance(pDataBlock, pObjectType, pi); + if(!pi) break; + + if(!getInstanceName(pDataBlock, pObjectType, pi, windows_shared_buffer, sizeof(windows_shared_buffer))) + strncpyz(windows_shared_buffer, "[unknown]", sizeof(windows_shared_buffer) - 1); + + if(strcasecmp(windows_shared_buffer, "_Total") == 0) + continue; + + if(!physical && is_physical_interface(windows_shared_buffer)) + // this virtual interface is already reported as physical interface + continue; + + struct network_interface *d = dictionary_set(dict, windows_shared_buffer, NULL, sizeof(*d)); + + if(!d->collected_metadata) { + // TODO - get metadata about the network interface + d->collected_metadata = true; + } + + if(perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->traffic.received) || + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->traffic.sent)) { + + if(d->traffic.received.current.Data == 0 && d->traffic.sent.current.Data == 0) + // this interface has not received or sent any traffic + continue; + + if (unlikely(!d->traffic.st)) { + d->traffic.st = rrdset_create_localhost( + "net", + windows_shared_buffer, + NULL, + windows_shared_buffer, + "net.net", + "Bandwidth", + "kilobits/s", + PLUGIN_WINDOWS_NAME, + "PerflibNetwork", + NETDATA_CHART_PRIO_FIRST_NET_IFACE, + update_every, + RRDSET_TYPE_AREA); + + rrdset_flag_set(d->traffic.st, RRDSET_FLAG_DETAIL); + + add_interface_labels(d->traffic.st, windows_shared_buffer, physical); + + d->traffic.rd_received = rrddim_add(d->traffic.st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + d->traffic.rd_sent = rrddim_add(d->traffic.st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + } + + total_received += d->traffic.received.current.Data; + total_sent += d->traffic.sent.current.Data; + + rrddim_set_by_pointer(d->traffic.st, d->traffic.rd_received, (collected_number)d->traffic.received.current.Data); + rrddim_set_by_pointer(d->traffic.st, d->traffic.rd_sent, (collected_number)d->traffic.sent.current.Data); + rrdset_done(d->traffic.st); + } + + if(perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->packets.received) || + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->packets.sent)) { + + if (unlikely(!d->packets.st)) { + d->packets.st = rrdset_create_localhost( + "net_packets", + windows_shared_buffer, + NULL, + windows_shared_buffer, + "net.packets", + "Packets", + "packets/s", + PLUGIN_WINDOWS_NAME, + "PerflibNetwork", + NETDATA_CHART_PRIO_FIRST_NET_IFACE + 1, + update_every, + RRDSET_TYPE_LINE); + + rrdset_flag_set(d->packets.st, RRDSET_FLAG_DETAIL); + + add_interface_labels(d->traffic.st, windows_shared_buffer, physical); + + d->packets.rd_received = rrddim_add(d->packets.st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + d->packets.rd_sent = rrddim_add(d->packets.st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + + rrddim_set_by_pointer(d->packets.st, d->packets.rd_received, (collected_number)d->packets.received.current.Data); + rrddim_set_by_pointer(d->packets.st, d->packets.rd_sent, (collected_number)d->packets.sent.current.Data); + rrdset_done(d->packets.st); + } + } + + if(physical) { + static RRDSET *st = NULL; + static RRDDIM *rd_received = NULL, *rd_sent = NULL; + + if (unlikely(!st)) { + st = rrdset_create_localhost( + "system", + "net", + NULL, + "network", + "system.net", + "Physical Network Interfaces Aggregated Bandwidth", + "kilobits/s", + PLUGIN_WINDOWS_NAME, + "PerflibNetwork", + NETDATA_CHART_PRIO_SYSTEM_NET, + update_every, + RRDSET_TYPE_AREA); + + rd_received = rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + } + + rrddim_set_by_pointer(st, rd_received, (collected_number)total_received); + rrddim_set_by_pointer(st, rd_sent, (collected_number)total_sent); + rrdset_done(st); + } + + return true; +} + +int do_PerflibNetwork(int update_every, usec_t dt __maybe_unused) { + static bool initialized = false; + + if(unlikely(!initialized)) { + initialize(); + initialized = true; + } + + DWORD id = RegistryFindIDByName("Network Interface"); + if(id == PERFLIB_REGISTRY_NAME_NOT_FOUND) + return -1; + + PERF_DATA_BLOCK *pDataBlock = perflibGetPerformanceData(id); + if(!pDataBlock) return -1; + + do_network_interface(pDataBlock, update_every, true); + do_network_interface(pDataBlock, update_every, false); + + struct network_protocol *tcp4 = NULL, *tcp6 = NULL; + for(size_t i = 0; networks[i].protocol ;i++) { + do_network_protocol(pDataBlock, update_every, &networks[i]); + + if(!tcp4 && strcmp(networks[i].protocol, "TCPv4") == 0) + tcp4 = &networks[i]; + if(!tcp6 && strcmp(networks[i].protocol, "TCPv6") == 0) + tcp6 = &networks[i]; + } + + if(tcp4 && tcp6) { + tcp46.packets.received = tcp4->packets.received; + tcp46.packets.sent = tcp4->packets.sent; + tcp46.packets.received.current.Data += tcp6->packets.received.current.Data; + tcp46.packets.sent.current.Data += tcp6->packets.sent.current.Data; + protocol_packets_chart_update(&tcp46, update_every); + } + + return 0; +} diff --git a/src/collectors/windows.plugin/perflib-processor.c b/src/collectors/windows.plugin/perflib-processor.c new file mode 100644 index 00000000000000..d149c6aad46efd --- /dev/null +++ b/src/collectors/windows.plugin/perflib-processor.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +struct processor { + bool collected_metadata; + + RRDSET *st; + RRDDIM *rd_user; + RRDDIM *rd_system; + RRDDIM *rd_irq; + RRDDIM *rd_dpc; + RRDDIM *rd_idle; + +// RRDSET *st2; +// RRDDIM *rd2_busy; + + COUNTER_DATA percentProcessorTime; + COUNTER_DATA percentUserTime; + COUNTER_DATA percentPrivilegedTime; + COUNTER_DATA percentDPCTime; + COUNTER_DATA percentInterruptTime; + COUNTER_DATA percentIdleTime; +}; + +struct processor total = { 0 }; + +void initialize_processor_keys(struct processor *p) { + p->percentProcessorTime.key = "% Processor Time"; + p->percentUserTime.key = "% User Time"; + p->percentPrivilegedTime.key = "% Privileged Time"; + p->percentDPCTime.key = "% DPC Time"; + p->percentInterruptTime.key = "% Interrupt Time"; + p->percentIdleTime.key = "% Idle Time"; +} + +void dict_processor_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct processor *p = value; + initialize_processor_keys(p); +} + +static DICTIONARY *processors = NULL; + +static void initialize(void) { + initialize_processor_keys(&total); + + processors = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | + DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct processor)); + + dictionary_register_insert_callback(processors, dict_processor_insert_cb, NULL); +} + +static bool do_processors(PERF_DATA_BLOCK *pDataBlock, int update_every) { + PERF_OBJECT_TYPE *pObjectType = perflibFindObjectTypeByName(pDataBlock, "Processor"); + if(!pObjectType) return false; + + static const RRDVAR_ACQUIRED *cpus_var = NULL; + int cores_found = 0; + + PERF_INSTANCE_DEFINITION *pi = NULL; + for(LONG i = 0; i < pObjectType->NumInstances ; i++) { + pi = perflibForEachInstance(pDataBlock, pObjectType, pi); + if(!pi) break; + + if(!getInstanceName(pDataBlock, pObjectType, pi, windows_shared_buffer, sizeof(windows_shared_buffer))) + strncpyz(windows_shared_buffer, "[unknown]", sizeof(windows_shared_buffer) - 1); + + bool is_total = false; + struct processor *p; + int cpu = -1; + if(strcasecmp(windows_shared_buffer, "_Total") == 0) { + p = &total; + is_total = true; + cpu = -1; + } + else { + p = dictionary_set(processors, windows_shared_buffer, NULL, sizeof(*p)); + is_total = false; + cpu = str2i(windows_shared_buffer); + snprintfz(windows_shared_buffer, sizeof(windows_shared_buffer), "cpu%d", cpu); + + if(cpu + 1 > cores_found) + cores_found = cpu + 1; + } + + if(!is_total && !p->collected_metadata) { + // TODO collect processor metadata + p->collected_metadata = true; + } + + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &p->percentProcessorTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &p->percentUserTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &p->percentPrivilegedTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &p->percentDPCTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &p->percentInterruptTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &p->percentIdleTime); + + if(!p->st) { + p->st = rrdset_create_localhost( + is_total ? "system" : "cpu" + , is_total ? "cpu" : windows_shared_buffer, NULL + , is_total ? "cpu" : "utilization" + , is_total ? "system.cpu" : "cpu.cpu" + , is_total ? "Total CPU Utilization" : "Core Utilization" + , "percentage" + , PLUGIN_WINDOWS_NAME + , "PerflibProcessor" + , is_total ? NETDATA_CHART_PRIO_SYSTEM_CPU : NETDATA_CHART_PRIO_CPU_PER_CORE + , update_every + , RRDSET_TYPE_STACKED + ); + + p->rd_irq = rrddim_add(p->st, "interrupts", "irq", 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + p->rd_user = rrddim_add(p->st, "user", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + p->rd_system = rrddim_add(p->st, "privileged", "system", 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + p->rd_dpc = rrddim_add(p->st, "dpc", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + p->rd_idle = rrddim_add(p->st, "idle", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rrddim_hide(p->st, "idle"); + + if(!is_total) + rrdlabels_add(p->st->rrdlabels, "cpu", windows_shared_buffer, RRDLABEL_SRC_AUTO); + else + cpus_var = rrdvar_host_variable_add_and_acquire(localhost, "active_processors"); + } + + uint64_t user = p->percentUserTime.current.Data; + uint64_t system = p->percentPrivilegedTime.current.Data; + uint64_t dpc = p->percentDPCTime.current.Data; + uint64_t irq = p->percentInterruptTime.current.Data; + uint64_t idle = p->percentIdleTime.current.Data; + + rrddim_set_by_pointer(p->st, p->rd_user, (collected_number)user); + rrddim_set_by_pointer(p->st, p->rd_system, (collected_number)system); + rrddim_set_by_pointer(p->st, p->rd_irq, (collected_number)irq); + rrddim_set_by_pointer(p->st, p->rd_dpc, (collected_number)dpc); + rrddim_set_by_pointer(p->st, p->rd_idle, (collected_number)idle); + rrdset_done(p->st); + +// if(!p->st2) { +// p->st2 = rrdset_create_localhost( +// is_total ? "system" : "cpu2" +// , is_total ? "cpu3" : buffer +// , NULL +// , is_total ? "utilization" : buffer +// , is_total ? "system.cpu3" : "cpu2.cpu" +// , is_total ? "Total CPU Utilization" : "Core Utilization" +// , "percentage" +// , PLUGIN_WINDOWS_NAME +// , "PerflibProcessor" +// , is_total ? NETDATA_CHART_PRIO_SYSTEM_CPU : NETDATA_CHART_PRIO_CPU_PER_CORE +// , update_every +// , RRDSET_TYPE_STACKED +// ); +// +// p->rd2_busy = perflib_rrddim_add(p->st2, "busy", NULL, 1, 1, &p->percentProcessorTime); +// rrddim_hide(p->st2, "idle"); +// +// if(!is_total) +// rrdlabels_add(p->st->rrdlabels, "cpu", buffer, RRDLABEL_SRC_AUTO); +// } +// +// perflib_rrddim_set_by_pointer(p->st2, p->rd2_busy, &p->percentProcessorTime); +// rrdset_done(p->st2); + } + + if(cpus_var) + rrdvar_host_variable_set(localhost, cpus_var, cores_found); + + return true; +} + +int do_PerflibProcessor(int update_every, usec_t dt __maybe_unused) { + static bool initialized = false; + + if(unlikely(!initialized)) { + initialize(); + initialized = true; + } + + DWORD id = RegistryFindIDByName("Processor"); + if(id == PERFLIB_REGISTRY_NAME_NOT_FOUND) + return -1; + + PERF_DATA_BLOCK *pDataBlock = perflibGetPerformanceData(id); + if(!pDataBlock) return -1; + + do_processors(pDataBlock, update_every); + + return 0; +} diff --git a/src/collectors/windows.plugin/perflib-rrd.c b/src/collectors/windows.plugin/perflib-rrd.c new file mode 100644 index 00000000000000..d425307eecaf44 --- /dev/null +++ b/src/collectors/windows.plugin/perflib-rrd.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib-rrd.h" + +#define COLLECTED_NUMBER_PRECISION 10000 + +RRDDIM *perflib_rrddim_add(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divider, COUNTER_DATA *cd) { + RRD_ALGORITHM algorithm = RRD_ALGORITHM_ABSOLUTE; + + switch (cd->current.CounterType) { + case PERF_COUNTER_COUNTER: + case PERF_SAMPLE_COUNTER: + case PERF_COUNTER_BULK_COUNT: + // (N1 - N0) / ((D1 - D0) / F) + // multiplier *= cd->current.Frequency / 10000000; + // tested, the frequency is not that useful for netdata + // we get right results without it. + algorithm = RRD_ALGORITHM_INCREMENTAL; + break; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + case PERF_AVERAGE_BULK: // normally not displayed + // (N1 - N0) / (D1 - D0) + algorithm = RRD_ALGORITHM_INCREMENTAL; + break; + + case PERF_OBJ_TIME_TIMER: + case PERF_COUNTER_TIMER: + case PERF_100NSEC_TIMER: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + case PERF_SAMPLE_FRACTION: + // 100 * (N1 - N0) / (D1 - D0) + multiplier *= 100; + algorithm = RRD_ALGORITHM_INCREMENTAL; + break; + + case PERF_COUNTER_TIMER_INV: + case PERF_100NSEC_TIMER_INV: + // 100 * (1 - ((N1 - N0) / (D1 - D0))) + divider *= COLLECTED_NUMBER_PRECISION; + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_COUNTER_MULTI_TIMER: + // 100 * ((N1 - N0) / ((D1 - D0) / TB)) / B1 + divider *= COLLECTED_NUMBER_PRECISION; + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_100NSEC_MULTI_TIMER: + // 100 * ((N1 - N0) / (D1 - D0)) / B1 + divider *= COLLECTED_NUMBER_PRECISION; + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_COUNTER_MULTI_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER_INV: + // 100 * (B1 - ((N1 - N0) / (D1 - D0))) + divider *= COLLECTED_NUMBER_PRECISION; + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + // N as decimal + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + // N as hexadecimal + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + // N1 - N0 + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_RAW_FRACTION: + case PERF_LARGE_RAW_FRACTION: + // 100 * N / B + algorithm = RRD_ALGORITHM_ABSOLUTE; + divider *= COLLECTED_NUMBER_PRECISION; + break; + + case PERF_AVERAGE_TIMER: + // ((N1 - N0) / TB) / (B1 - B0) + // divider *= cd->current.Frequency / 10000000; + algorithm = RRD_ALGORITHM_INCREMENTAL; + break; + + case PERF_ELAPSED_TIME: + // (D0 - N0) / F + algorithm = RRD_ALGORITHM_ABSOLUTE; + break; + + case PERF_COUNTER_TEXT: + case PERF_SAMPLE_BASE: + case PERF_AVERAGE_BASE: + case PERF_COUNTER_MULTI_BASE: + case PERF_RAW_BASE: + case PERF_COUNTER_NODATA: + case PERF_PRECISION_TIMESTAMP: + default: + break; + } + + return rrddim_add(st, id, name, multiplier, divider, algorithm); +} + +#define VALID_DELTA(cd) \ + ((cd)->previous.Time > 0 && (cd)->current.Data >= (cd)->previous.Data && (cd)->current.Time > (cd)->previous.Time) + +collected_number perflib_rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, COUNTER_DATA *cd) { + ULONGLONG numerator = 0; + LONGLONG denominator = 0; + double doubleValue = 0.0; + collected_number value; + + switch(cd->current.CounterType) { + case PERF_COUNTER_COUNTER: + case PERF_SAMPLE_COUNTER: + case PERF_COUNTER_BULK_COUNT: + // (N1 - N0) / ((D1 - D0) / F) + value = (collected_number)cd->current.Data; + break; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + case PERF_AVERAGE_BULK: // normally not displayed + // (N1 - N0) / (D1 - D0) + value = (collected_number)cd->current.Data; + break; + + case PERF_OBJ_TIME_TIMER: + case PERF_COUNTER_TIMER: + case PERF_100NSEC_TIMER: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + case PERF_SAMPLE_FRACTION: + // 100 * (N1 - N0) / (D1 - D0) + value = (collected_number)cd->current.Data; + break; + + case PERF_COUNTER_TIMER_INV: + case PERF_100NSEC_TIMER_INV: + // 100 * (1 - ((N1 - N0) / (D1 - D0))) + if(!VALID_DELTA(cd)) return 0; + numerator = cd->current.Data - cd->previous.Data; + denominator = cd->current.Time - cd->previous.Time; + doubleValue = 100.0 * (1.0 - ((double)numerator / (double)denominator)); + // printf("Display value is (timer-inv): %f%%\n", doubleValue); + value = (collected_number)(doubleValue * COLLECTED_NUMBER_PRECISION); + break; + + case PERF_COUNTER_MULTI_TIMER: + // 100 * ((N1 - N0) / ((D1 - D0) / TB)) / B1 + if(!VALID_DELTA(cd)) return 0; + numerator = cd->current.Data - cd->previous.Data; + denominator = cd->current.Time - cd->previous.Time; + denominator /= cd->current.Frequency; + doubleValue = 100.0 * ((double)numerator / (double)denominator) / cd->current.MultiCounterData; + // printf("Display value is (multi-timer): %f%%\n", doubleValue); + value = (collected_number)(doubleValue * COLLECTED_NUMBER_PRECISION); + break; + + case PERF_100NSEC_MULTI_TIMER: + // 100 * ((N1 - N0) / (D1 - D0)) / B1 + if(!VALID_DELTA(cd)) return 0; + numerator = cd->current.Data - cd->previous.Data; + denominator = cd->current.Time - cd->previous.Time; + doubleValue = 100.0 * ((double)numerator / (double)denominator) / (double)cd->current.MultiCounterData; + // printf("Display value is (100ns multi-timer): %f%%\n", doubleValue); + value = (collected_number)(doubleValue * COLLECTED_NUMBER_PRECISION); + break; + + case PERF_COUNTER_MULTI_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER_INV: + // 100 * (B1 - ((N1 - N0) / (D1 - D0))) + if(!VALID_DELTA(cd)) return 0; + numerator = cd->current.Data - cd->previous.Data; + denominator = cd->current.Time - cd->previous.Time; + doubleValue = 100.0 * ((double)cd->current.MultiCounterData - ((double)numerator / (double)denominator)); + // printf("Display value is (multi-timer-inv): %f%%\n", doubleValue); + value = (collected_number)(doubleValue * COLLECTED_NUMBER_PRECISION); + break; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + // N as decimal + value = (collected_number)cd->current.Data; + break; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + // N as hexadecimal + value = (collected_number)cd->current.Data; + break; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + if(!VALID_DELTA(cd)) return 0; + value = (collected_number)(cd->current.Data - cd->previous.Data); + break; + + case PERF_RAW_FRACTION: + case PERF_LARGE_RAW_FRACTION: + // 100 * N / B + if(!cd->current.Time) return 0; + doubleValue = 100.0 * (double)cd->current.Data / (double)cd->current.Time; + // printf("Display value is (fraction): %f%%\n", doubleValue); + value = (collected_number)(doubleValue * COLLECTED_NUMBER_PRECISION); + break; + + default: + return 0; + } + + return rrddim_set_by_pointer(st, rd, value); +} + +/* +double perflibCalculateValue(RAW_DATA *current, RAW_DATA *previous) { + ULONGLONG numerator = 0; + LONGLONG denominator = 0; + double doubleValue = 0.0; + DWORD dwordValue = 0; + + if (NULL == previous) { + // Return error if the counter type requires two samples to calculate the value. + switch (current->CounterType) { + default: + if (PERF_DELTA_COUNTER != (current->CounterType & PERF_DELTA_COUNTER)) + break; + __fallthrough; + // fallthrough + + case PERF_AVERAGE_TIMER: // Special case. + case PERF_AVERAGE_BULK: // Special case. + // printf(" > The counter type requires two samples but only one sample was provided.\n"); + return NAN; + } + } + else { + if (current->CounterType != previous->CounterType) { + // printf(" > The samples have inconsistent counter types.\n"); + return NAN; + } + + // Check for integer overflow or bad data from provider (the data from + // sample 2 must be greater than the data from sample 1). + if (current->Data < previous->Data) + { + // Can happen for various reasons. Commonly occurs with the Process counterset when + // multiple processes have the same name and one of them starts or stops. + // Normally you'll just drop the older sample and continue. + // printf("> current (%llu) is smaller than previous (%llu).\n", current->Data, previous->Data); + return NAN; + } + } + + switch (current->CounterType) { + case PERF_COUNTER_COUNTER: + case PERF_SAMPLE_COUNTER: + case PERF_COUNTER_BULK_COUNT: + // (N1 - N0) / ((D1 - D0) / F) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + dwordValue = (DWORD)(numerator / ((double)denominator / current->Frequency)); + //printf("Display value is (counter): %lu%s\n", (unsigned long)dwordValue, + // (previous->CounterType == PERF_SAMPLE_COUNTER) ? "" : "/sec"); + return (double)dwordValue; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + case PERF_AVERAGE_BULK: // normally not displayed + // (N1 - N0) / (D1 - D0) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = (double)numerator / denominator; + if (previous->CounterType != PERF_AVERAGE_BULK) { + // printf("Display value is (queuelen): %f\n", doubleValue); + return doubleValue; + } + return NAN; + + case PERF_OBJ_TIME_TIMER: + case PERF_COUNTER_TIMER: + case PERF_100NSEC_TIMER: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + case PERF_SAMPLE_FRACTION: + // 100 * (N1 - N0) / (D1 - D0) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = (double)(100 * numerator) / denominator; + // printf("Display value is (timer): %f%%\n", doubleValue); + return doubleValue; + + case PERF_COUNTER_TIMER_INV: + // 100 * (1 - ((N1 - N0) / (D1 - D0))) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = 100 * (1 - ((double)numerator / denominator)); + // printf("Display value is (timer-inv): %f%%\n", doubleValue); + return doubleValue; + + case PERF_100NSEC_TIMER_INV: + // 100 * (1- (N1 - N0) / (D1 - D0)) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = 100 * (1 - (double)numerator / denominator); + // printf("Display value is (100ns-timer-inv): %f%%\n", doubleValue); + return doubleValue; + + case PERF_COUNTER_MULTI_TIMER: + // 100 * ((N1 - N0) / ((D1 - D0) / TB)) / B1 + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + denominator /= current->Frequency; + doubleValue = 100 * ((double)numerator / denominator) / current->MultiCounterData; + // printf("Display value is (multi-timer): %f%%\n", doubleValue); + return doubleValue; + + case PERF_100NSEC_MULTI_TIMER: + // 100 * ((N1 - N0) / (D1 - D0)) / B1 + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = 100 * ((double)numerator / (double)denominator) / (double)current->MultiCounterData; + // printf("Display value is (100ns multi-timer): %f%%\n", doubleValue); + return doubleValue; + + case PERF_COUNTER_MULTI_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER_INV: + // 100 * (B1 - ((N1 - N0) / (D1 - D0))) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = 100.0 * ((double)current->MultiCounterData - ((double)numerator / (double)denominator)); + // printf("Display value is (multi-timer-inv): %f%%\n", doubleValue); + return doubleValue; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + // N as decimal + // printf("Display value is (rawcount): %llu\n", current->Data); + return (double)current->Data; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + // N as hexadecimal + // printf("Display value is (hex): 0x%llx\n", current->Data); + return (double)current->Data; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + // N1 - N0 + // printf("Display value is (delta): %llu\n", current->Data - previous->Data); + return (double)(current->Data - previous->Data); + + case PERF_RAW_FRACTION: + case PERF_LARGE_RAW_FRACTION: + // 100 * N / B + doubleValue = 100.0 * (double)current->Data / (double)current->Time; + // printf("Display value is (fraction): %f%%\n", doubleValue); + return doubleValue; + + case PERF_AVERAGE_TIMER: + // ((N1 - N0) / TB) / (B1 - B0) + numerator = current->Data - previous->Data; + denominator = current->Time - previous->Time; + doubleValue = (double)numerator / (double)current->Frequency / (double)denominator; + // printf("Display value is (average timer): %f seconds\n", doubleValue); + return doubleValue; + + case PERF_ELAPSED_TIME: + // (D0 - N0) / F + doubleValue = (double)(current->Time - current->Data) / (double)current->Frequency; + // printf("Display value is (elapsed time): %f seconds\n", doubleValue); + return doubleValue; + + case PERF_COUNTER_TEXT: + case PERF_SAMPLE_BASE: + case PERF_AVERAGE_BASE: + case PERF_COUNTER_MULTI_BASE: + case PERF_RAW_BASE: + case PERF_COUNTER_NODATA: + case PERF_PRECISION_TIMESTAMP: + // printf(" > Non-printing counter type: 0x%08x\n", current->CounterType); + return NAN; + break; + + default: + // printf(" > Unrecognized counter type: 0x%08x\n", current->CounterType); + return NAN; + break; + } +} +*/ diff --git a/src/collectors/windows.plugin/perflib-rrd.h b/src/collectors/windows.plugin/perflib-rrd.h new file mode 100644 index 00000000000000..0b91de2ec384ad --- /dev/null +++ b/src/collectors/windows.plugin/perflib-rrd.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PERFLIB_RRD_H +#define NETDATA_PERFLIB_RRD_H + +#include "perflib.h" +#include "database/rrd.h" + +RRDDIM *perflib_rrddim_add(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divider, COUNTER_DATA *cd); +collected_number perflib_rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, COUNTER_DATA *cd); + +#endif //NETDATA_PERFLIB_RRD_H diff --git a/src/collectors/windows.plugin/perflib-storage.c b/src/collectors/windows.plugin/perflib-storage.c new file mode 100644 index 00000000000000..d3b80052f97bb4 --- /dev/null +++ b/src/collectors/windows.plugin/perflib-storage.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" +#include "windows-internals.h" + +#define _COMMON_PLUGIN_NAME PLUGIN_WINDOWS_NAME +#define _COMMON_PLUGIN_MODULE_NAME "PerflibStorage" +#include "../common-contexts/common-contexts.h" + +struct logical_disk { + bool collected_metadata; + + STRING *filesystem; + + RRDSET *st_disk_space; + RRDDIM *rd_disk_space_used; + RRDDIM *rd_disk_space_free; + + COUNTER_DATA percentDiskFree; + // COUNTER_DATA freeMegabytes; +}; + +struct physical_disk { + bool collected_metadata; + + STRING *device; + STRING *mount_point; + + ND_DISK_IO disk_io; + COUNTER_DATA diskReadBytesPerSec; + COUNTER_DATA diskWriteBytesPerSec; + + COUNTER_DATA percentIdleTime; + COUNTER_DATA percentDiskTime; + COUNTER_DATA percentDiskReadTime; + COUNTER_DATA percentDiskWriteTime; + COUNTER_DATA currentDiskQueueLength; + COUNTER_DATA averageDiskQueueLength; + COUNTER_DATA averageDiskReadQueueLength; + COUNTER_DATA averageDiskWriteQueueLength; + COUNTER_DATA averageDiskSecondsPerTransfer; + COUNTER_DATA averageDiskSecondsPerRead; + COUNTER_DATA averageDiskSecondsPerWrite; + COUNTER_DATA diskTransfersPerSec; + COUNTER_DATA diskReadsPerSec; + COUNTER_DATA diskWritesPerSec; + COUNTER_DATA diskBytesPerSec; + COUNTER_DATA averageDiskBytesPerTransfer; + COUNTER_DATA averageDiskBytesPerRead; + COUNTER_DATA averageDiskBytesPerWrite; + COUNTER_DATA splitIoPerSec; +}; + +struct physical_disk system_physical_total = { + .collected_metadata = true, +}; + +void dict_logical_disk_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct logical_disk *ld = value; + + ld->percentDiskFree.key = "% Free Space"; + // ld->freeMegabytes.key = "Free Megabytes"; +} + +void initialize_physical_disk(struct physical_disk *pd) { + pd->percentIdleTime.key = "% Idle Time"; + pd->percentDiskTime.key = "% Disk Time"; + pd->percentDiskReadTime.key = "% Disk Read Time"; + pd->percentDiskWriteTime.key = "% Disk Write Time"; + pd->currentDiskQueueLength.key = "Current Disk Queue Length"; + pd->averageDiskQueueLength.key = "Avg. Disk Queue Length"; + pd->averageDiskReadQueueLength.key = "Avg. Disk Read Queue Length"; + pd->averageDiskWriteQueueLength.key = "Avg. Disk Write Queue Length"; + pd->averageDiskSecondsPerTransfer.key = "Avg. Disk sec/Transfer"; + pd->averageDiskSecondsPerRead.key = "Avg. Disk sec/Read"; + pd->averageDiskSecondsPerWrite.key = "Avg. Disk sec/Write"; + pd->diskTransfersPerSec.key = "Disk Transfers/sec"; + pd->diskReadsPerSec.key = "Disk Reads/sec"; + pd->diskWritesPerSec.key = "Disk Writes/sec"; + pd->diskBytesPerSec.key = "Disk Bytes/sec"; + pd->diskReadBytesPerSec.key = "Disk Read Bytes/sec"; + pd->diskWriteBytesPerSec.key = "Disk Write Bytes/sec"; + pd->averageDiskBytesPerTransfer.key = "Avg. Disk Bytes/Transfer"; + pd->averageDiskBytesPerRead.key = "Avg. Disk Bytes/Read"; + pd->averageDiskBytesPerWrite.key = "Avg. Disk Bytes/Write"; + pd->splitIoPerSec.key = "Split IO/Sec"; +} + +void dict_physical_disk_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct physical_disk *pd = value; + initialize_physical_disk(pd); +} + +static DICTIONARY *logicalDisks = NULL, *physicalDisks = NULL; +static void initialize(void) { + initialize_physical_disk(&system_physical_total); + + logicalDisks = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | + DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct logical_disk)); + + dictionary_register_insert_callback(logicalDisks, dict_logical_disk_insert_cb, NULL); + + physicalDisks = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | + DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct physical_disk)); + + dictionary_register_insert_callback(physicalDisks, dict_physical_disk_insert_cb, NULL); +} + +static STRING *getFileSystemType(const char* diskName) { + if (!diskName || !*diskName) return NULL; + + char fileSystemNameBuffer[128] = {0}; // Buffer for file system name + char pathBuffer[256] = {0}; // Path buffer to accommodate different formats + DWORD serialNumber = 0; + DWORD maxComponentLength = 0; + DWORD fileSystemFlags = 0; + BOOL success; + + // Check if the input is likely a drive letter (e.g., "C:") + if (isalpha((uint8_t)diskName[0]) && diskName[1] == ':' && diskName[2] == '\0') + snprintf(pathBuffer, sizeof(pathBuffer), "%s\\", diskName); // Format as "C:\\" + else + // Assume it's a Volume GUID path or a device path + snprintf(pathBuffer, sizeof(pathBuffer), "\\\\.\\%s", diskName); // Format as "\\.\HarddiskVolume1" + + // Attempt to get the volume information + success = GetVolumeInformation( + pathBuffer, // Path to the disk + NULL, // We don't need the volume name + 0, // Size of volume name buffer is 0 + &serialNumber, // Volume serial number + &maxComponentLength, // Maximum component length + &fileSystemFlags, // File system flags + fileSystemNameBuffer, // File system name buffer + sizeof(fileSystemNameBuffer) // Size of file system name buffer + ); + + if (success && fileSystemNameBuffer[0]) { + char *s = fileSystemNameBuffer; + while(*s) { *s = tolower((uint8_t)*s); s++; } + return string_strdupz(fileSystemNameBuffer); // Duplicate the file system name + } + else + return NULL; +} + +static bool do_logical_disk(PERF_DATA_BLOCK *pDataBlock, int update_every) { + DICTIONARY *dict = logicalDisks; + + PERF_OBJECT_TYPE *pObjectType = perflibFindObjectTypeByName(pDataBlock, "LogicalDisk"); + if(!pObjectType) return false; + + PERF_INSTANCE_DEFINITION *pi = NULL; + for(LONG i = 0; i < pObjectType->NumInstances ; i++) { + pi = perflibForEachInstance(pDataBlock, pObjectType, pi); + if(!pi) break; + + if(!getInstanceName(pDataBlock, pObjectType, pi, windows_shared_buffer, sizeof(windows_shared_buffer))) + strncpyz(windows_shared_buffer, "[unknown]", sizeof(windows_shared_buffer) - 1); + + if(strcasecmp(windows_shared_buffer, "_Total") == 0) + continue; + + struct logical_disk *d = dictionary_set(dict, windows_shared_buffer, NULL, sizeof(*d)); + + if(!d->collected_metadata) { + d->filesystem = getFileSystemType(windows_shared_buffer); + d->collected_metadata = true; + } + + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->percentDiskFree); + // perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->freeMegabytes); + + if(!d->st_disk_space) { + d->st_disk_space = rrdset_create_localhost( + "disk_space" + , windows_shared_buffer, NULL + , windows_shared_buffer, "disk.space" + , "Disk Space Usage" + , "GiB" + , PLUGIN_WINDOWS_NAME + , "PerflibStorage" + , NETDATA_CHART_PRIO_DISKSPACE_SPACE + , update_every + , RRDSET_TYPE_STACKED + ); + + rrdlabels_add(d->st_disk_space->rrdlabels, "mount_point", windows_shared_buffer, RRDLABEL_SRC_AUTO); + // rrdlabels_add(d->st->rrdlabels, "mount_root", name, RRDLABEL_SRC_AUTO); + + if(d->filesystem) + rrdlabels_add(d->st_disk_space->rrdlabels, "filesystem", string2str(d->filesystem), RRDLABEL_SRC_AUTO); + + d->rd_disk_space_free = rrddim_add(d->st_disk_space, "avail", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + d->rd_disk_space_used = rrddim_add(d->st_disk_space, "used", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + } + + // percentDiskFree has the free space in Data and the size of the disk in Time, in MiB. + rrddim_set_by_pointer(d->st_disk_space, d->rd_disk_space_free, (collected_number)d->percentDiskFree.current.Data); + rrddim_set_by_pointer(d->st_disk_space, d->rd_disk_space_used, (collected_number)(d->percentDiskFree.current.Time - d->percentDiskFree.current.Data)); + rrdset_done(d->st_disk_space); + } + + return true; +} + +static void physical_disk_labels(RRDSET *st, void *data) { + struct physical_disk *d = data; + + if(d->device) + rrdlabels_add(st->rrdlabels, "device", string2str(d->device), RRDLABEL_SRC_AUTO); + + if (d->mount_point) + rrdlabels_add(st->rrdlabels, "mount_point", string2str(d->mount_point), RRDLABEL_SRC_AUTO); +} + +static bool do_physical_disk(PERF_DATA_BLOCK *pDataBlock, int update_every) { + DICTIONARY *dict = physicalDisks; + + PERF_OBJECT_TYPE *pObjectType = perflibFindObjectTypeByName(pDataBlock, "PhysicalDisk"); + if(!pObjectType) return false; + + PERF_INSTANCE_DEFINITION *pi = NULL; + for (LONG i = 0; i < pObjectType->NumInstances; i++) { + pi = perflibForEachInstance(pDataBlock, pObjectType, pi); + if (!pi) + break; + + if (!getInstanceName(pDataBlock, pObjectType, pi, windows_shared_buffer, sizeof(windows_shared_buffer))) + strncpyz(windows_shared_buffer, "[unknown]", sizeof(windows_shared_buffer) - 1); + + char *device = windows_shared_buffer; + char *mount_point = NULL; + + if((mount_point = strchr(device, ' '))) { + *mount_point = '\0'; + mount_point++; + } + + struct physical_disk *d; + bool is_system; + if (strcasecmp(windows_shared_buffer, "_Total") == 0) { + d = &system_physical_total; + is_system = true; + } + else { + d = dictionary_set(dict, device, NULL, sizeof(*d)); + is_system = false; + } + + if (!d->collected_metadata) { + // TODO collect metadata - device_type, serial, id + d->device = string_strdupz(device); + d->mount_point = string_strdupz(mount_point); + d->collected_metadata = true; + } + + if (perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->diskReadBytesPerSec) && + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->diskWriteBytesPerSec)) { + if(is_system) + common_system_io(d->diskReadBytesPerSec.current.Data, d->diskWriteBytesPerSec.current.Data, update_every); + else + common_disk_io( + &d->disk_io, + device, + NULL, + d->diskReadBytesPerSec.current.Data, + d->diskWriteBytesPerSec.current.Data, + update_every, + physical_disk_labels, + d); + } + + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->percentIdleTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->percentDiskTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->percentDiskReadTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->percentDiskWriteTime); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->currentDiskQueueLength); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskQueueLength); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskReadQueueLength); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskWriteQueueLength); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskSecondsPerTransfer); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskSecondsPerRead); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskSecondsPerWrite); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->diskTransfersPerSec); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->diskReadsPerSec); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->diskWritesPerSec); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->diskBytesPerSec); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskBytesPerTransfer); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskBytesPerRead); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->averageDiskBytesPerWrite); + perflibGetInstanceCounter(pDataBlock, pObjectType, pi, &d->splitIoPerSec); + } + + return true; +} + +int do_PerflibStorage(int update_every, usec_t dt __maybe_unused) { + static bool initialized = false; + + if(unlikely(!initialized)) { + initialize(); + initialized = true; + } + + DWORD id = RegistryFindIDByName("LogicalDisk"); + if(id == PERFLIB_REGISTRY_NAME_NOT_FOUND) + return -1; + + PERF_DATA_BLOCK *pDataBlock = perflibGetPerformanceData(id); + if(!pDataBlock) return -1; + + do_logical_disk(pDataBlock, update_every); + do_physical_disk(pDataBlock, update_every); + + return 0; +} diff --git a/src/collectors/windows.plugin/perflib.c b/src/collectors/windows.plugin/perflib.c new file mode 100644 index 00000000000000..4df48acfbfa946 --- /dev/null +++ b/src/collectors/windows.plugin/perflib.c @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib.h" + +// -------------------------------------------------------------------------------- + +// Retrieve a buffer that contains the specified performance data. +// The pwszSource parameter determines the data that GetRegistryBuffer returns. +// +// Typically, when calling RegQueryValueEx, you can specify zero for the size of the buffer +// and the RegQueryValueEx will set your size variable to the required buffer size. However, +// if the source is "Global" or one or more object index values, you will need to increment +// the buffer size in a loop until RegQueryValueEx does not return ERROR_MORE_DATA. +static LPBYTE getPerformanceData(const char *pwszSource) { + static __thread DWORD size = 0; + static __thread LPBYTE buffer = NULL; + + if(pwszSource == (const char *)0x01) { + freez(buffer); + buffer = NULL; + size = 0; + return NULL; + } + + if(!size) { + size = 32 * 1024; + buffer = mallocz(size); + } + + LONG status = ERROR_SUCCESS; + while ((status = RegQueryValueEx(HKEY_PERFORMANCE_DATA, pwszSource, + NULL, NULL, buffer, &size)) == ERROR_MORE_DATA) { + size *= 2; + buffer = reallocz(buffer, size); + } + + if (status != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "RegQueryValueEx failed with 0x%x.\n", status); + return NULL; + } + + return buffer; +} + +void perflibFreePerformanceData(void) { + getPerformanceData((const char *)0x01); +} + +// -------------------------------------------------------------------------------------------------------------------- + +// Retrieve the raw counter value and any supporting data needed to calculate +// a displayable counter value. Use the counter type to determine the information +// needed to calculate the value. + +static BOOL getCounterData( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE* pObject, + PERF_COUNTER_DEFINITION* pCounter, + PERF_COUNTER_BLOCK* pCounterDataBlock, + PRAW_DATA pRawData) +{ + PVOID pData = NULL; + UNALIGNED ULONGLONG* pullData = NULL; + PERF_COUNTER_DEFINITION* pBaseCounter = NULL; + BOOL fSuccess = TRUE; + + //Point to the raw counter data. + pData = (PVOID)((LPBYTE)pCounterDataBlock + pCounter->CounterOffset); + + //Now use the PERF_COUNTER_DEFINITION.CounterType value to figure out what + //other information you need to calculate a displayable value. + switch (pCounter->CounterType) { + + case PERF_COUNTER_COUNTER: + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_SAMPLE_COUNTER: + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pRawData->Time = pDataBlock->PerfTime.QuadPart; + if (PERF_COUNTER_COUNTER == pCounter->CounterType || PERF_SAMPLE_COUNTER == pCounter->CounterType) + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + break; + + case PERF_OBJ_TIME_TIMER: + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pRawData->Time = pObject->PerfTime.QuadPart; + break; + + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + pRawData->Data = *(UNALIGNED ULONGLONG *)pData; + pRawData->Time = pDataBlock->PerfTime100nSec.QuadPart; + break; + + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + pRawData->Data = *(UNALIGNED ULONGLONG *)pData; + pRawData->Time = pObject->PerfTime.QuadPart; + break; + + case PERF_COUNTER_TIMER: + case PERF_COUNTER_TIMER_INV: + case PERF_COUNTER_BULK_COUNT: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + pullData = (UNALIGNED ULONGLONG *)pData; + pRawData->Data = *pullData; + pRawData->Time = pDataBlock->PerfTime.QuadPart; + if (pCounter->CounterType == PERF_COUNTER_BULK_COUNT) + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + break; + + case PERF_COUNTER_MULTI_TIMER: + case PERF_COUNTER_MULTI_TIMER_INV: + pullData = (UNALIGNED ULONGLONG *)pData; + pRawData->Data = *pullData; + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + pRawData->Time = pDataBlock->PerfTime.QuadPart; + + //These counter types have a second counter value that is adjacent to + //this counter value in the counter data block. The value is needed for + //the calculation. + if ((pCounter->CounterType & PERF_MULTI_COUNTER) == PERF_MULTI_COUNTER) { + ++pullData; + pRawData->MultiCounterData = *(DWORD*)pullData; + } + break; + + //These counters do not use any time reference. + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_DELTA: + // some counters in these categories, have CounterSize = sizeof(ULONGLONG) + // but the official documentation always uses them as sizeof(DWORD) + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pRawData->Time = 0; + break; + + case PERF_COUNTER_LARGE_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_DELTA: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pRawData->Time = 0; + break; + + //These counters use the 100ns time base in their calculation. + case PERF_100NSEC_TIMER: + case PERF_100NSEC_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER: + case PERF_100NSEC_MULTI_TIMER_INV: + pullData = (UNALIGNED ULONGLONG*)pData; + pRawData->Data = *pullData; + pRawData->Time = pDataBlock->PerfTime100nSec.QuadPart; + + //These counter types have a second counter value that is adjacent to + //this counter value in the counter data block. The value is needed for + //the calculation. + if ((pCounter->CounterType & PERF_MULTI_COUNTER) == PERF_MULTI_COUNTER) { + ++pullData; + pRawData->MultiCounterData = *(DWORD*)pullData; + } + break; + + //These counters use two data points, this value and one from this counter's + //base counter. The base counter should be the next counter in the object's + //list of counters. + case PERF_SAMPLE_FRACTION: + case PERF_RAW_FRACTION: + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pBaseCounter = pCounter + 1; //Get base counter + if ((pBaseCounter->CounterType & PERF_COUNTER_BASE) == PERF_COUNTER_BASE) { + pData = (PVOID)((LPBYTE)pCounterDataBlock + pBaseCounter->CounterOffset); + pRawData->Time = (LONGLONG)(*(DWORD*)pData); + } + else + fSuccess = FALSE; + break; + + case PERF_LARGE_RAW_FRACTION: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pBaseCounter = pCounter + 1; + if ((pBaseCounter->CounterType & PERF_COUNTER_BASE) == PERF_COUNTER_BASE) { + pData = (PVOID)((LPBYTE)pCounterDataBlock + pBaseCounter->CounterOffset); + pRawData->Time = *(LONGLONG*)pData; + } + else + fSuccess = FALSE; + break; + + case PERF_AVERAGE_TIMER: + case PERF_AVERAGE_BULK: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pBaseCounter = pCounter+1; + if ((pBaseCounter->CounterType & PERF_COUNTER_BASE) == PERF_COUNTER_BASE) { + pData = (PVOID)((LPBYTE)pCounterDataBlock + pBaseCounter->CounterOffset); + pRawData->Time = *(DWORD*)pData; + } + else + fSuccess = FALSE; + + if (pCounter->CounterType == PERF_AVERAGE_TIMER) + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + break; + + //These are base counters and are used in calculations for other counters. + //This case should never be entered. + case PERF_SAMPLE_BASE: + case PERF_AVERAGE_BASE: + case PERF_COUNTER_MULTI_BASE: + case PERF_RAW_BASE: + case PERF_LARGE_RAW_BASE: + pRawData->Data = 0; + pRawData->Time = 0; + fSuccess = FALSE; + break; + + case PERF_ELAPSED_TIME: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pRawData->Time = pObject->PerfTime.QuadPart; + pRawData->Frequency = pObject->PerfFreq.QuadPart; + break; + + //These counters are currently not supported. + case PERF_COUNTER_TEXT: + case PERF_COUNTER_NODATA: + case PERF_COUNTER_HISTOGRAM_TYPE: + default: // unknown counter types + pRawData->Data = 0; + pRawData->Time = 0; + fSuccess = FALSE; + break; + } + + return fSuccess; +} + +// -------------------------------------------------------------------------------------------------------------------- + +static inline BOOL isValidPointer(PERF_DATA_BLOCK *pDataBlock __maybe_unused, void *ptr __maybe_unused) { +#ifdef NETDATA_INTERNAL_CHECKS + return (PBYTE)ptr >= (PBYTE)pDataBlock + pDataBlock->TotalByteLength ? FALSE : TRUE; +#else + return TRUE; +#endif +} + +static inline BOOL isValidStructure(PERF_DATA_BLOCK *pDataBlock __maybe_unused, void *ptr __maybe_unused, size_t length __maybe_unused) { +#ifdef NETDATA_INTERNAL_CHECKS + return (PBYTE)ptr + length > (PBYTE)pDataBlock + pDataBlock->TotalByteLength ? FALSE : TRUE; +#else + return TRUE; +#endif +} + +static inline PERF_DATA_BLOCK *getDataBlock(BYTE *pBuffer) { + PERF_DATA_BLOCK *pDataBlock = (PERF_DATA_BLOCK *)pBuffer; + + static WCHAR signature[] = { 'P', 'E', 'R', 'F' }; + + if(memcmp(pDataBlock->Signature, signature, sizeof(signature)) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid data block signature."); + return NULL; + } + + if(!isValidPointer(pDataBlock, (PBYTE)pDataBlock + pDataBlock->SystemNameOffset) || + !isValidStructure(pDataBlock, (PBYTE)pDataBlock + pDataBlock->SystemNameOffset, pDataBlock->SystemNameLength)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid system name array."); + return NULL; + } + + return pDataBlock; +} + +static inline PERF_OBJECT_TYPE *getObjectType(PERF_DATA_BLOCK* pDataBlock, PERF_OBJECT_TYPE *lastObjectType) { + PERF_OBJECT_TYPE* pObjectType = NULL; + + if(!lastObjectType) + pObjectType = (PERF_OBJECT_TYPE *)((PBYTE)pDataBlock + pDataBlock->HeaderLength); + else if (lastObjectType->TotalByteLength != 0) + pObjectType = (PERF_OBJECT_TYPE *)((PBYTE)lastObjectType + lastObjectType->TotalByteLength); + + if(pObjectType && (!isValidPointer(pDataBlock, pObjectType) || !isValidStructure(pDataBlock, pObjectType, pObjectType->TotalByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid ObjectType!"); + pObjectType = NULL; + } + + return pObjectType; +} + +inline PERF_OBJECT_TYPE *getObjectTypeByIndex(PERF_DATA_BLOCK *pDataBlock, DWORD ObjectNameTitleIndex) { + PERF_OBJECT_TYPE *po = NULL; + for(DWORD o = 0; o < pDataBlock->NumObjectTypes ; o++) { + po = getObjectType(pDataBlock, po); + if(po->ObjectNameTitleIndex == ObjectNameTitleIndex) + return po; + } + + return NULL; +} + +static inline PERF_INSTANCE_DEFINITION *getInstance( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType, + PERF_COUNTER_BLOCK *lastCounterBlock +) { + PERF_INSTANCE_DEFINITION *pInstance; + + if(!lastCounterBlock) + pInstance = (PERF_INSTANCE_DEFINITION *)((PBYTE)pObjectType + pObjectType->DefinitionLength); + else + pInstance = (PERF_INSTANCE_DEFINITION *)((PBYTE)lastCounterBlock + lastCounterBlock->ByteLength); + + if(pInstance && (!isValidPointer(pDataBlock, pInstance) || !isValidStructure(pDataBlock, pInstance, pInstance->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid Instance Definition!"); + pInstance = NULL; + } + + return pInstance; +} + +static inline PERF_COUNTER_BLOCK *getObjectTypeCounterBlock( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType +) { + PERF_COUNTER_BLOCK *pCounterBlock = (PERF_COUNTER_BLOCK *)((PBYTE)pObjectType + pObjectType->DefinitionLength); + + if(pCounterBlock && (!isValidPointer(pDataBlock, pCounterBlock) || !isValidStructure(pDataBlock, pCounterBlock, pCounterBlock->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid ObjectType CounterBlock!"); + pCounterBlock = NULL; + } + + return pCounterBlock; +} + +static inline PERF_COUNTER_BLOCK *getInstanceCounterBlock( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType, + PERF_INSTANCE_DEFINITION *pInstance +) { + (void)pObjectType; + PERF_COUNTER_BLOCK *pCounterBlock = (PERF_COUNTER_BLOCK *)((PBYTE)pInstance + pInstance->ByteLength); + + if(pCounterBlock && (!isValidPointer(pDataBlock, pCounterBlock) || !isValidStructure(pDataBlock, pCounterBlock, pCounterBlock->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid Instance CounterBlock!"); + pCounterBlock = NULL; + } + + return pCounterBlock; +} + +inline PERF_INSTANCE_DEFINITION *getInstanceByPosition(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, DWORD instancePosition) { + PERF_INSTANCE_DEFINITION *pi = NULL; + PERF_COUNTER_BLOCK *pc = NULL; + for(DWORD i = 0; i <= instancePosition ;i++) { + pi = getInstance(pDataBlock, pObjectType, pc); + pc = getInstanceCounterBlock(pDataBlock, pObjectType, pi); + } + return pi; +} + +static inline PERF_COUNTER_DEFINITION *getCounterDefinition(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_COUNTER_DEFINITION *lastCounterDefinition) { + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + + if(!lastCounterDefinition) + pCounterDefinition = (PERF_COUNTER_DEFINITION *)((PBYTE)pObjectType + pObjectType->HeaderLength); + else + pCounterDefinition = (PERF_COUNTER_DEFINITION *)((PBYTE)lastCounterDefinition + lastCounterDefinition->ByteLength); + + if(pCounterDefinition && (!isValidPointer(pDataBlock, pCounterDefinition) || !isValidStructure(pDataBlock, pCounterDefinition, pCounterDefinition->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid Counter Definition!"); + pCounterDefinition = NULL; + } + + return pCounterDefinition; +} + +// -------------------------------------------------------------------------------------------------------------------- + +static inline BOOL getEncodedStringToUTF8(char *dst, size_t dst_len, DWORD CodePage, char *start, DWORD length) { + WCHAR *tempBuffer; // Temporary buffer for Unicode data + DWORD charsCopied = 0; + BOOL free_tempBuffer; + + if (CodePage == 0) { + // Input is already Unicode (UTF-16) + tempBuffer = (WCHAR *)start; + charsCopied = length / sizeof(WCHAR); // Convert byte length to number of WCHARs + free_tempBuffer = FALSE; + } + else { + // Convert the multi-byte instance name to Unicode (UTF-16) + // Calculate maximum possible characters in UTF-16 + + int charCount = MultiByteToWideChar(CodePage, 0, start, (int)length, NULL, 0); + tempBuffer = (WCHAR *)malloc(charCount * sizeof(WCHAR)); + if (!tempBuffer) return FALSE; + + charsCopied = MultiByteToWideChar(CodePage, 0, start, (int)length, tempBuffer, charCount); + if (charsCopied == 0) { + free(tempBuffer); + dst[0] = '\0'; + return FALSE; + } + + free_tempBuffer = TRUE; + } + + // Now convert from Unicode (UTF-16) to UTF-8 + int bytesCopied = WideCharToMultiByte(CP_UTF8, 0, tempBuffer, (int)charsCopied, dst, (int)dst_len, NULL, NULL); + if (bytesCopied == 0) { + if (free_tempBuffer) free(tempBuffer); + dst[0] = '\0'; // Ensure the buffer is null-terminated even on failure + return FALSE; + } + + dst[bytesCopied] = '\0'; // Ensure buffer is null-terminated + if (free_tempBuffer) free(tempBuffer); // Free temporary buffer if used + return TRUE; +} + +inline BOOL getInstanceName(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, + char *buffer, size_t bufferLen) { + (void)pDataBlock; + if (!pInstance || !buffer || !bufferLen) return FALSE; + + return getEncodedStringToUTF8(buffer, bufferLen, pObjectType->CodePage, + ((char *)pInstance + pInstance->NameOffset), pInstance->NameLength); +} + +inline BOOL getSystemName(PERF_DATA_BLOCK *pDataBlock, char *buffer, size_t bufferLen) { + return getEncodedStringToUTF8(buffer, bufferLen, 0, + ((char *)pDataBlock + pDataBlock->SystemNameOffset), pDataBlock->SystemNameLength); +} + +inline bool ObjectTypeHasInstances(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType) { + (void)pDataBlock; + return pObjectType->NumInstances != PERF_NO_INSTANCES && pObjectType->NumInstances > 0; +} + +PERF_OBJECT_TYPE *perflibFindObjectTypeByName(PERF_DATA_BLOCK *pDataBlock, const char *name) { + PERF_OBJECT_TYPE* pObjectType = NULL; + for(DWORD o = 0; o < pDataBlock->NumObjectTypes; o++) { + pObjectType = getObjectType(pDataBlock, pObjectType); + if(strcmp(name, RegistryFindNameByID(pObjectType->ObjectNameTitleIndex)) == 0) + return pObjectType; + } + + return NULL; +} + +PERF_INSTANCE_DEFINITION *perflibForEachInstance(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *lastInstance) { + if(!ObjectTypeHasInstances(pDataBlock, pObjectType)) + return NULL; + + return getInstance(pDataBlock, pObjectType, + lastInstance ? + getInstanceCounterBlock(pDataBlock, pObjectType, lastInstance) : + NULL ); +} + +bool perflibGetInstanceCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, COUNTER_DATA *cd) { + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + if(cd->id) { + if(cd->id != pCounterDefinition->CounterNameTitleIndex) + continue; + } + else { + if(strcmp(RegistryFindNameByID(pCounterDefinition->CounterNameTitleIndex), cd->key) != 0) + continue; + + cd->id = pCounterDefinition->CounterNameTitleIndex; + } + + cd->current.CounterType = cd->OverwriteCounterType ? cd->OverwriteCounterType : pCounterDefinition->CounterType; + PERF_COUNTER_BLOCK *pCounterBlock = getInstanceCounterBlock(pDataBlock, pObjectType, pInstance); + + cd->previous = cd->current; + cd->updated = getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &cd->current); + return cd->updated; + } + + cd->previous = cd->current; + cd->current = RAW_DATA_EMPTY; + cd->updated = false; + return false; +} + +bool perflibGetObjectCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, COUNTER_DATA *cd) { + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + if(cd->id) { + if(cd->id != pCounterDefinition->CounterNameTitleIndex) + continue; + } + else { + if(strcmp(RegistryFindNameByID(pCounterDefinition->CounterNameTitleIndex), cd->key) != 0) + continue; + + cd->id = pCounterDefinition->CounterNameTitleIndex; + } + + cd->current.CounterType = cd->OverwriteCounterType ? cd->OverwriteCounterType : pCounterDefinition->CounterType; + PERF_COUNTER_BLOCK *pCounterBlock = getObjectTypeCounterBlock(pDataBlock, pObjectType); + + cd->previous = cd->current; + cd->updated = getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &cd->current); + return cd->updated; + } + + cd->previous = cd->current; + cd->current = RAW_DATA_EMPTY; + cd->updated = false; + return false; +} + +PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id) { + char source[24]; + snprintfz(source, sizeof(source), "%u", id); + + LPBYTE pData = (LPBYTE)getPerformanceData((id > 0) ? source : NULL); + if (!pData) return NULL; + + PERF_DATA_BLOCK *pDataBlock = getDataBlock(pData); + if(!pDataBlock) return NULL; + + return pDataBlock; +} + +int perflibQueryAndTraverse(DWORD id, + perflib_data_cb dataCb, + perflib_object_cb objectCb, + perflib_instance_cb instanceCb, + perflib_instance_counter_cb instanceCounterCb, + perflib_counter_cb counterCb, + void *data) { + int counters = -1; + + PERF_DATA_BLOCK *pDataBlock = perflibGetPerformanceData(id); + if(!pDataBlock) goto cleanup; + + bool do_data = true; + if(dataCb) + do_data = dataCb(pDataBlock, data); + + PERF_OBJECT_TYPE* pObjectType = NULL; + for(DWORD o = 0; do_data && o < pDataBlock->NumObjectTypes; o++) { + pObjectType = getObjectType(pDataBlock, pObjectType); + if(!pObjectType) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read object type No %d (out of %d)", + o, pDataBlock->NumObjectTypes); + break; + } + + bool do_object = true; + if(objectCb) + do_object = objectCb(pDataBlock, pObjectType, data); + + if(!do_object) + continue; + + if(ObjectTypeHasInstances(pDataBlock, pObjectType)) { + PERF_INSTANCE_DEFINITION *pInstance = NULL; + PERF_COUNTER_BLOCK *pCounterBlock = NULL; + for(LONG i = 0; i < pObjectType->NumInstances ;i++) { + pInstance = getInstance(pDataBlock, pObjectType, pCounterBlock); + if(!pInstance) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read Instance No %d (out of %d)", + i, pObjectType->NumInstances); + break; + } + + pCounterBlock = getInstanceCounterBlock(pDataBlock, pObjectType, pInstance); + if(!pCounterBlock) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read CounterBlock of instance No %d (out of %d)", + i, pObjectType->NumInstances); + break; + } + + bool do_instance = true; + if(instanceCb) + do_instance = instanceCb(pDataBlock, pObjectType, pInstance, data); + + if(!do_instance) + continue; + + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + RAW_DATA sample = { + .CounterType = pCounterDefinition->CounterType, + }; + if(getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &sample)) { + // DisplayCalculatedValue(&sample, &sample); + + if(instanceCounterCb) { + instanceCounterCb(pDataBlock, pObjectType, pInstance, pCounterDefinition, &sample, data); + counters++; + } + } + } + + if(instanceCb) + instanceCb(pDataBlock, pObjectType, NULL, data); + } + } + else { + PERF_COUNTER_BLOCK *pCounterBlock = getObjectTypeCounterBlock(pDataBlock, pObjectType); + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + RAW_DATA sample = { + .CounterType = pCounterDefinition->CounterType, + }; + if(getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &sample)) { + // DisplayCalculatedValue(&sample, &sample); + + if(counterCb) { + counterCb(pDataBlock, pObjectType, pCounterDefinition, &sample, data); + counters++; + } + } + } + } + + if(objectCb) + objectCb(pDataBlock, NULL, data); + } + +cleanup: + return counters; +} diff --git a/src/collectors/windows.plugin/perflib.h b/src/collectors/windows.plugin/perflib.h new file mode 100644 index 00000000000000..deba4e9a320822 --- /dev/null +++ b/src/collectors/windows.plugin/perflib.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PERFLIB_H +#define NETDATA_PERFLIB_H + +#include "libnetdata/libnetdata.h" +#include + +const char *RegistryFindNameByID(DWORD id); +const char *RegistryFindHelpByID(DWORD id); +DWORD RegistryFindIDByName(const char *name); +#define PERFLIB_REGISTRY_NAME_NOT_FOUND (DWORD)-1 + +PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id); +void perflibFreePerformanceData(void); +PERF_OBJECT_TYPE *perflibFindObjectTypeByName(PERF_DATA_BLOCK *pDataBlock, const char *name); +PERF_INSTANCE_DEFINITION *perflibForEachInstance(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *lastInstance); + +typedef struct _rawdata { + DWORD CounterType; + DWORD MultiCounterData; // Second raw counter value for multi-valued counters + ULONGLONG Data; // Raw counter data + LONGLONG Time; // Is a time value or a base value + LONGLONG Frequency; +} RAW_DATA, *PRAW_DATA; + +typedef struct _counterdata { + DWORD id; + bool updated; + const char *key; + DWORD OverwriteCounterType; // if set, the counter type will be overwritten once read + RAW_DATA current; + RAW_DATA previous; +} COUNTER_DATA; + +#define RAW_DATA_EMPTY (RAW_DATA){ 0 } + +bool perflibGetInstanceCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, COUNTER_DATA *cd); +bool perflibGetObjectCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, COUNTER_DATA *cd); + +typedef bool (*perflib_data_cb)(PERF_DATA_BLOCK *pDataBlock, void *data); +typedef bool (*perflib_object_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, void *data); +typedef bool (*perflib_instance_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, void *data); +typedef bool (*perflib_instance_counter_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data); +typedef bool (*perflib_counter_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data); + +int perflibQueryAndTraverse(DWORD id, + perflib_data_cb dataCb, + perflib_object_cb objectCb, + perflib_instance_cb instanceCb, + perflib_instance_counter_cb instanceCounterCb, + perflib_counter_cb counterCb, + void *data); + +bool ObjectTypeHasInstances(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType); + +BOOL getInstanceName(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, + char *buffer, size_t bufferLen); + +BOOL getSystemName(PERF_DATA_BLOCK *pDataBlock, char *buffer, size_t bufferLen); + +PERF_OBJECT_TYPE *getObjectTypeByIndex(PERF_DATA_BLOCK *pDataBlock, DWORD ObjectNameTitleIndex); + +PERF_INSTANCE_DEFINITION *getInstanceByPosition( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType, + DWORD instancePosition); + +void PerflibNamesRegistryInitialize(void); +void PerflibNamesRegistryUpdate(void); + +#endif //NETDATA_PERFLIB_H diff --git a/src/collectors/windows.plugin/windows-internals.h b/src/collectors/windows.plugin/windows-internals.h new file mode 100644 index 00000000000000..1b7cccc730e725 --- /dev/null +++ b/src/collectors/windows.plugin/windows-internals.h @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_WINDOWS_INTERNALS_H +#define NETDATA_WINDOWS_INTERNALS_H + +#include + +static inline ULONGLONG FileTimeToULL(FILETIME ft) { + ULARGE_INTEGER ul; + ul.LowPart = ft.dwLowDateTime; + ul.HighPart = ft.dwHighDateTime; + return ul.QuadPart; +} + +#include "perflib.h" +#include "perflib-rrd.h" + +#endif //NETDATA_WINDOWS_INTERNALS_H diff --git a/src/collectors/windows.plugin/windows_plugin.c b/src/collectors/windows.plugin/windows_plugin.c new file mode 100644 index 00000000000000..b0a63444f40ca1 --- /dev/null +++ b/src/collectors/windows.plugin/windows_plugin.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows_plugin.h" + +char windows_shared_buffer[8192]; + +static struct proc_module { + const char *name; + const char *dim; + int enabled; + int (*func)(int update_every, usec_t dt); + RRDDIM *rd; +} win_modules[] = { + + // system metrics + {.name = "GetSystemUptime", .dim = "GetSystemUptime", .func = do_GetSystemUptime}, + {.name = "GetSystemRAM", .dim = "GetSystemRAM", .func = do_GetSystemRAM}, + + // the same is provided by PerflibProcessor, with more detailed analysis + //{.name = "GetSystemCPU", .dim = "GetSystemCPU", .func = do_GetSystemCPU}, + + {.name = "PerflibProcessor", .dim = "PerflibProcessor", .func = do_PerflibProcessor}, + {.name = "PerflibMemory", .dim = "PerflibMemory", .func = do_PerflibMemory}, + {.name = "PerflibStorage", .dim = "PerflibStorage", .func = do_PerflibStorage}, + {.name = "PerflibNetwork", .dim = "PerflibNetwork", .func = do_PerflibNetwork}, + + // the terminator of this array + {.name = NULL, .dim = NULL, .func = NULL} +}; + +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 36 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 36 +#endif + +static void windows_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + + collector_info("cleaning up..."); + + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; + + worker_unregister(); +} + +static bool log_windows_module(BUFFER *wb, void *data) { + struct proc_module *pm = data; + buffer_sprintf(wb, PLUGIN_WINDOWS_NAME "[%s]", pm->name); + return true; +} + +void *win_plugin_main(void *ptr) { + worker_register("WIN"); + + rrd_collector_started(); + PerflibNamesRegistryInitialize(); + + CLEANUP_FUNCTION_REGISTER(windows_main_cleanup) cleanup_ptr = ptr; + + // check the enabled status for each module + int i; + for(i = 0; win_modules[i].name; i++) { + struct proc_module *pm = &win_modules[i]; + + pm->enabled = config_get_boolean("plugin:windows", pm->name, CONFIG_BOOLEAN_YES); + pm->rd = NULL; + + worker_register_job_name(i, win_modules[i].dim); + } + + usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); + +#define LGS_MODULE_ID 0 + + ND_LOG_STACK lgs[] = { + [LGS_MODULE_ID] = ND_LOG_FIELD_TXT(NDF_MODULE, PLUGIN_WINDOWS_NAME), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); + + while(service_running(SERVICE_COLLECTORS)) { + worker_is_idle(); + usec_t hb_dt = heartbeat_next(&hb, step); + + if(unlikely(!service_running(SERVICE_COLLECTORS))) + break; + + PerflibNamesRegistryUpdate(); + + for(i = 0; win_modules[i].name; i++) { + if(unlikely(!service_running(SERVICE_COLLECTORS))) + break; + + struct proc_module *pm = &win_modules[i]; + if(unlikely(!pm->enabled)) + continue; + + worker_is_busy(i); + lgs[LGS_MODULE_ID] = ND_LOG_FIELD_CB(NDF_MODULE, log_windows_module, pm); + pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); + lgs[LGS_MODULE_ID] = ND_LOG_FIELD_TXT(NDF_MODULE, PLUGIN_WINDOWS_NAME); + } + } + return NULL; +} diff --git a/src/collectors/windows.plugin/windows_plugin.h b/src/collectors/windows.plugin/windows_plugin.h new file mode 100644 index 00000000000000..d6e0fa4849e1a8 --- /dev/null +++ b/src/collectors/windows.plugin/windows_plugin.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_WINDOWS_PLUGIN_H +#define NETDATA_WINDOWS_PLUGIN_H + +#include "daemon/common.h" + +#define PLUGIN_WINDOWS_NAME "windows.plugin" + +void *win_plugin_main(void *ptr); + +extern char windows_shared_buffer[8192]; + +int do_GetSystemUptime(int update_every, usec_t dt); +int do_GetSystemRAM(int update_every, usec_t dt); +int do_GetSystemCPU(int update_every, usec_t dt); +int do_PerflibStorage(int update_every, usec_t dt); +int do_PerflibNetwork(int update_every, usec_t dt); +int do_PerflibProcessor(int update_every, usec_t dt); +int do_PerflibMemory(int update_every, usec_t dt); + +#include "perflib.h" + +#endif //NETDATA_WINDOWS_PLUGIN_H diff --git a/src/daemon/analytics.c b/src/daemon/analytics.c index 015b8b2e86abbe..48681223eacd97 100644 --- a/src/daemon/analytics.c +++ b/src/daemon/analytics.c @@ -223,7 +223,7 @@ void analytics_mirrored_hosts(void) count++; } - rrd_unlock(); + rrd_rdunlock(); snprintfz(b, sizeof(b) - 1, "%zu", count); analytics_set_data(&analytics_data.netdata_mirrored_host_count, b); @@ -562,9 +562,11 @@ void analytics_gather_mutable_meta_data(void) } } -void analytics_main_cleanup(void *ptr) +void analytics_main_cleanup(void *pptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_debug(D_ANALYTICS, "Cleaning up..."); @@ -581,7 +583,7 @@ void analytics_main_cleanup(void *ptr) */ void *analytics_main(void *ptr) { - netdata_thread_cleanup_push(analytics_main_cleanup, ptr); + CLEANUP_FUNCTION_REGISTER(analytics_main_cleanup) cleanup_ptr = ptr; unsigned int sec = 0; heartbeat_t hb; heartbeat_init(&hb); @@ -626,7 +628,6 @@ void *analytics_main(void *ptr) } cleanup: - netdata_thread_cleanup_pop(1); return NULL; } @@ -777,7 +778,7 @@ void get_system_timezone(void) *d = '\0'; while (*timezone) { - if (isalnum(*timezone) || *timezone == '_' || *timezone == '/') + if (isalnum((uint8_t)*timezone) || *timezone == '_' || *timezone == '/') *d++ = *timezone++; else timezone++; @@ -816,11 +817,11 @@ void get_system_timezone(void) } else { sign[0] = zone[0] == '-' || zone[0] == '+' ? zone[0] : '0'; sign[1] = '\0'; - hh[0] = isdigit(zone[1]) ? zone[1] : '0'; - hh[1] = isdigit(zone[2]) ? zone[2] : '0'; + hh[0] = isdigit((uint8_t)zone[1]) ? zone[1] : '0'; + hh[1] = isdigit((uint8_t)zone[2]) ? zone[2] : '0'; hh[2] = '\0'; - mm[0] = isdigit(zone[3]) ? zone[3] : '0'; - mm[1] = isdigit(zone[4]) ? zone[4] : '0'; + mm[0] = isdigit((uint8_t)zone[3]) ? zone[3] : '0'; + mm[1] = isdigit((uint8_t)zone[4]) ? zone[4] : '0'; mm[2] = '\0'; netdata_configured_utc_offset = (str2i(hh) * 3600) + (str2i(mm) * 60); diff --git a/src/daemon/buildinfo.c b/src/daemon/buildinfo.c index 0b22251f2f8589..63b017e817e62f 100644 --- a/src/daemon/buildinfo.c +++ b/src/daemon/buildinfo.c @@ -1058,6 +1058,16 @@ __attribute__((constructor)) void initialize_build_info(void) { build_info_set_value(BIB_FEATURE_BUILT_FOR, "MacOS"); build_info_set_status(BIB_PLUGIN_MACOS, true); #endif +#ifdef COMPILED_FOR_WINDOWS + build_info_set_status(BIB_FEATURE_BUILT_FOR, true); +#if defined(__CYGWIN__) && defined(__MSYS__) + build_info_set_value(BIB_FEATURE_BUILT_FOR, "Windows (MSYS)"); +#elif defined(__CYGWIN__) + build_info_set_value(BIB_FEATURE_BUILT_FOR, "Windows (CYGWIN)"); +#else + build_info_set_value(BIB_FEATURE_BUILT_FOR, "Windows"); +#endif +#endif #ifdef ENABLE_ACLK build_info_set_status(BIB_FEATURE_CLOUD, true); diff --git a/src/daemon/commands.c b/src/daemon/commands.c index 8c9767a3fd1435..43123b229114ce 100644 --- a/src/daemon/commands.c +++ b/src/daemon/commands.c @@ -483,15 +483,15 @@ static void parse_commands(struct command_context *cmd_ctx) status = CMD_STATUS_FAILURE; /* Skip white-space characters */ - for (pos = cmd_ctx->command_string ; isspace(*pos) && ('\0' != *pos) ; ++pos) ; + for (pos = cmd_ctx->command_string ; isspace((uint8_t)*pos) && ('\0' != *pos) ; ++pos) ; for (i = 0 ; i < CMD_TOTAL_COMMANDS ; ++i) { if (!strncmp(pos, command_info_array[i].cmd_str, strlen(command_info_array[i].cmd_str))) { if (CMD_EXIT == i) { /* musl C does not like libuv workqueues calling exit() */ execute_command(CMD_EXIT, NULL, NULL); } - for (lstrip=pos + strlen(command_info_array[i].cmd_str); isspace(*lstrip) && ('\0' != *lstrip); ++lstrip) ; - for (rstrip=lstrip+strlen(lstrip)-1; rstrip>lstrip && isspace(*rstrip); *(rstrip--) = 0 ) ; + for (lstrip=pos + strlen(command_info_array[i].cmd_str); isspace((uint8_t)*lstrip) && ('\0' != *lstrip); ++lstrip) ; + for (rstrip=lstrip+strlen(lstrip)-1; rstrip>lstrip && isspace((uint8_t)*rstrip); *(rstrip--) = 0 ) ; cmd_ctx->work.data = cmd_ctx; cmd_ctx->idx = i; @@ -596,8 +596,9 @@ static void async_cb(uv_async_t *handle) uv_stop(handle->loop); } -static void command_thread(void *arg) -{ +static void command_thread(void *arg) { + uv_thread_set_name_np("DAEMON_COMMAND"); + int ret; uv_fs_t req; @@ -714,7 +715,6 @@ void commands_init(void) /* wait for worker thread to initialize */ completion_wait_for(&completion); completion_destroy(&completion); - uv_thread_set_name_np(thread, "DAEMON_COMMAND"); if (command_thread_error) { error = uv_thread_join(&thread); diff --git a/src/daemon/common.c b/src/daemon/common.c index d441c73b67f049..a64d53585e9b3c 100644 --- a/src/daemon/common.c +++ b/src/daemon/common.c @@ -31,9 +31,9 @@ long get_netdata_cpus(void) { if(processors) return processors; - long cores_proc_stat = get_system_cpus_with_cache(false, true); - long cores_cpuset_v1 = (long)read_cpuset_cpus("/sys/fs/cgroup/cpuset/cpuset.cpus", cores_proc_stat); - long cores_cpuset_v2 = (long)read_cpuset_cpus("/sys/fs/cgroup/cpuset.cpus", cores_proc_stat); + long cores_proc_stat = os_get_system_cpus_cached(false, true); + long cores_cpuset_v1 = (long)os_read_cpuset_cpus("/sys/fs/cgroup/cpuset/cpuset.cpus", cores_proc_stat); + long cores_cpuset_v2 = (long)os_read_cpuset_cpus("/sys/fs/cgroup/cpuset.cpus", cores_proc_stat); if(cores_cpuset_v2) processors = cores_cpuset_v2; diff --git a/src/daemon/config/dyncfg-files.c b/src/daemon/config/dyncfg-files.c index aa91c109dfa8b2..b9c4d1b4c8f1d4 100644 --- a/src/daemon/config/dyncfg-files.c +++ b/src/daemon/config/dyncfg-files.c @@ -226,16 +226,27 @@ static bool dyncfg_read_file_to_buffer(const char *filename, BUFFER *dst) { return true; } -bool dyncfg_get_schema(const char *id, BUFFER *dst) { +static bool dyncfg_get_schema_from(const char *dir, const char *id, BUFFER *dst) { char filename[FILENAME_MAX + 1]; - snprintfz(filename, sizeof(filename), "%s/schema.d/%s.json", netdata_configured_user_config_dir, id); + snprintfz(filename, sizeof(filename), "%s/schema.d/%s.json", dir, id); if(dyncfg_read_file_to_buffer(filename, dst)) return true; - snprintfz(filename, sizeof(filename), "%s/schema.d/%s.json", netdata_configured_stock_config_dir, id); + CLEAN_CHAR_P *escaped_id = dyncfg_escape_id_for_filename(id); + snprintfz(filename, sizeof(filename), "%s/schema.d/%s.json", dir, escaped_id); if(dyncfg_read_file_to_buffer(filename, dst)) return true; return false; } + +bool dyncfg_get_schema(const char *id, BUFFER *dst) { + if(dyncfg_get_schema_from(netdata_configured_user_config_dir, id, dst)) + return true; + + if(dyncfg_get_schema_from(netdata_configured_stock_config_dir, id, dst)) + return true; + + return false; +} diff --git a/src/daemon/config/dyncfg-internals.h b/src/daemon/config/dyncfg-internals.h index 181d2328fc16ad..1722ae7921ab80 100644 --- a/src/daemon/config/dyncfg-internals.h +++ b/src/daemon/config/dyncfg-internals.h @@ -10,7 +10,7 @@ #include "database/rrdcollector-internals.h" typedef struct dyncfg { - UUID host_uuid; + ND_UUID host_uuid; STRING *function; STRING *template; STRING *path; @@ -80,7 +80,7 @@ const DICTIONARY_ITEM *dyncfg_get_template_of_new_job(const char *job_id); bool dyncfg_is_user_disabled(const char *id); -RRDHOST *dyncfg_rrdhost_by_uuid(UUID *uuid); +RRDHOST *dyncfg_rrdhost_by_uuid(ND_UUID *uuid); RRDHOST *dyncfg_rrdhost(DYNCFG *df); static inline void dyncfg_copy_dyncfg_source_to_current(DYNCFG *df) { diff --git a/src/daemon/config/dyncfg-tree.c b/src/daemon/config/dyncfg-tree.c index 6af384daa273f8..285e55fd14c651 100644 --- a/src/daemon/config/dyncfg-tree.c +++ b/src/daemon/config/dyncfg-tree.c @@ -71,7 +71,7 @@ static void dyncfg_tree_for_host(RRDHOST *host, BUFFER *wb, const char *path, co if(id && *id) template = string_strdupz(id); - UUID host_uuid = uuid2UUID(host->host_uuid); + ND_UUID host_uuid = uuid2UUID(host->host_uuid); size_t path_len = strlen(path); DYNCFG *df; diff --git a/src/daemon/config/dyncfg-unittest.c b/src/daemon/config/dyncfg-unittest.c index 14a52f697b809d..775dc7cbd43714 100644 --- a/src/daemon/config/dyncfg-unittest.c +++ b/src/daemon/config/dyncfg-unittest.c @@ -165,8 +165,8 @@ static int dyncfg_unittest_action(struct dyncfg_unittest_action *a) { return rc; } -static void *dyncfg_unittest_thread_action(void *ptr __maybe_unused) { - while(1) { +static void *dyncfg_unittest_thread_action(void *ptr) { + while(!nd_thread_signaled_to_cancel()) { struct dyncfg_unittest_action *a = NULL; spinlock_lock(&dyncfg_unittest_data.spinlock); a = dyncfg_unittest_data.queue; @@ -179,6 +179,8 @@ static void *dyncfg_unittest_thread_action(void *ptr __maybe_unused) { else sleep_usec(10 * USEC_PER_MS); } + + return ptr; } static int dyncfg_unittest_execute_cb(struct rrd_function_execute *rfe, void *data) { @@ -300,8 +302,7 @@ static bool dyncfg_unittest_check(TEST *t, DYNCFG_CMDS c, const char *cmd, bool usec_t give_up_ut = now_monotonic_usec() + 2 * USEC_PER_SEC; while(!__atomic_load_n(&t->finished, __ATOMIC_RELAXED)) { - static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; - nanosleep(&ns, NULL); + tinysleep(); if(now_monotonic_usec() > give_up_ut) { fprintf(stderr, "\n - gave up waiting for the plugin to process this!"); @@ -579,9 +580,7 @@ int dyncfg_unittest(void) { // ------------------------------------------------------------------------ // create the thread for testing async communication - netdata_thread_t thread; - netdata_thread_create(&thread, "unittest", NETDATA_THREAD_OPTION_JOINABLE, - dyncfg_unittest_thread_action, NULL); + ND_THREAD *thread = nd_thread_create("unittest", NETDATA_THREAD_OPTION_JOINABLE, dyncfg_unittest_thread_action, NULL); // ------------------------------------------------------------------------ // single @@ -791,9 +790,8 @@ int dyncfg_unittest(void) { // if(rc == HTTP_RESP_OK) // fprintf(stderr, "%s\n", buffer_tostring(wb)); - void *ptr; - netdata_thread_cancel(thread); - netdata_thread_join(thread, &ptr); + nd_thread_signal_cancel(thread); + nd_thread_join(thread); dyncfg_unittest_cleanup_files(); dictionary_destroy(dyncfg_unittest_data.nodes); buffer_free(wb); diff --git a/src/daemon/config/dyncfg.c b/src/daemon/config/dyncfg.c index 80b48b6e9758b9..2f484d1ed6d5c7 100644 --- a/src/daemon/config/dyncfg.c +++ b/src/daemon/config/dyncfg.c @@ -5,7 +5,7 @@ struct dyncfg_globals dyncfg_globals = { 0 }; -RRDHOST *dyncfg_rrdhost_by_uuid(UUID *uuid) { +RRDHOST *dyncfg_rrdhost_by_uuid(ND_UUID *uuid) { char uuid_str[UUID_STR_LEN]; uuid_unparse_lower(uuid->uuid, uuid_str); @@ -438,7 +438,7 @@ void dyncfg_add_streaming(BUFFER *wb) { , 120 , "Dynamic configuration" , "config" - , HTTP_ACCESS_ANONYMOUS_DATA + , (unsigned)HTTP_ACCESS_ANONYMOUS_DATA , 1000 ); } diff --git a/src/daemon/daemon.c b/src/daemon/daemon.c index d9a4b81de52306..f77b748a84a896 100644 --- a/src/daemon/daemon.c +++ b/src/daemon/daemon.c @@ -121,11 +121,7 @@ static int become_user(const char *username, int pid_fd) { gid_t *supplementary_groups = NULL; if(ngroups > 0) { supplementary_groups = mallocz(sizeof(gid_t) * ngroups); -#ifdef __APPLE__ - if(getgrouplist(username, gid, (int *)supplementary_groups, &ngroups) == -1) { -#else - if(getgrouplist(username, gid, supplementary_groups, &ngroups) == -1) { -#endif /* __APPLE__ */ + if(os_getgrouplist(username, gid, supplementary_groups, &ngroups) == -1) { if(am_i_root) netdata_log_error("Cannot get supplementary groups of user '%s'.", username); @@ -149,20 +145,12 @@ static int become_user(const char *username, int pid_fd) { if(supplementary_groups) freez(supplementary_groups); -#ifdef __APPLE__ - if(setregid(gid, gid) != 0) { -#else - if(setresgid(gid, gid, gid) != 0) { -#endif /* __APPLE__ */ + if(os_setresgid(gid, gid, gid) != 0) { netdata_log_error("Cannot switch to user's %s group (gid: %u).", username, gid); return -1; } -#ifdef __APPLE__ - if(setreuid(uid, uid) != 0) { -#else - if(setresuid(uid, uid, uid) != 0) { -#endif /* __APPLE__ */ + if(os_setresuid(uid, uid, uid) != 0) { netdata_log_error("Cannot switch to user %s (uid: %u).", username, uid); return -1; } @@ -218,7 +206,7 @@ static void oom_score_adj(void) { // check netdata.conf configuration s = config_get(CONFIG_SECTION_GLOBAL, "OOM score", s); - if(s && *s && (isdigit(*s) || *s == '-' || *s == '+')) + if(s && *s && (isdigit((uint8_t)*s) || *s == '-' || *s == '+')) wanted_score = atoll(s); else if(s && !strcmp(s, "keep")) { netdata_log_info("Out-Of-Memory (OOM) kept as-is (running with %d)", (int) old_score); diff --git a/src/daemon/event_loop.c b/src/daemon/event_loop.c index 93bac97d0a6812..d1908ec15d7e5f 100644 --- a/src/daemon/event_loop.c +++ b/src/daemon/event_loop.c @@ -62,5 +62,5 @@ void register_libuv_worker_jobs() { char buf[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(buf, NETDATA_THREAD_TAG_MAX, "UV_WORKER[%d]", worker_id); - uv_thread_set_name_np(pthread_self(), buf); + uv_thread_set_name_np(buf); } diff --git a/src/daemon/global_statistics.c b/src/daemon/global_statistics.c index 7a8fc4c385d164..429f68c0de3930 100644 --- a/src/daemon/global_statistics.c +++ b/src/daemon/global_statistics.c @@ -2576,7 +2576,7 @@ static void dbengine2_statistics_charts(void) { } } } - rrd_unlock(); + rrd_rdunlock(); if (dbengine_contexts) { /* deduplicate global statistics by getting the ones from the last context */ @@ -3520,6 +3520,7 @@ static struct worker_utilization all_workers_utilization[] = { { .name = "STATSD", .family = "workers plugin statsd", .priority = 1000000 }, { .name = "STATSDFLUSH", .family = "workers plugin statsd flush", .priority = 1000000 }, { .name = "PROC", .family = "workers plugin proc", .priority = 1000000 }, + { .name = "WIN", .family = "workers plugin windows", .priority = 1000000 }, { .name = "NETDEV", .family = "workers plugin proc netdev", .priority = 1000000 }, { .name = "FREEBSD", .family = "workers plugin freebsd", .priority = 1000000 }, { .name = "MACOS", .family = "workers plugin macos", .priority = 1000000 }, @@ -4155,17 +4156,13 @@ static void worker_utilization_charts(void) { for(int i = 0; all_workers_utilization[i].name ;i++) { workers_utilization_reset_statistics(&all_workers_utilization[i]); - netdata_thread_disable_cancelability(); workers_foreach(all_workers_utilization[i].name, worker_utilization_charts_callback, &all_workers_utilization[i]); - netdata_thread_enable_cancelability(); // skip the first iteration, so that we don't accumulate startup utilization to our charts if(likely(iterations > 1)) workers_utilization_update_chart(&all_workers_utilization[i]); - netdata_thread_disable_cancelability(); workers_threads_cleanup(&all_workers_utilization[i]); - netdata_thread_enable_cancelability(); } workers_total_cpu_utilization_chart(); @@ -4215,13 +4212,14 @@ static void global_statistics_register_workers(void) { worker_register_job_name(WORKER_JOB_SQLITE3, "sqlite3"); } -static void global_statistics_cleanup(void *ptr) +static void global_statistics_cleanup(void *pptr) { - worker_unregister(); + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + worker_unregister(); netdata_log_info("cleaning up..."); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; @@ -4229,9 +4227,9 @@ static void global_statistics_cleanup(void *ptr) void *global_statistics_main(void *ptr) { - global_statistics_register_workers(); + CLEANUP_FUNCTION_REGISTER(global_statistics_cleanup) cleanup_ptr = ptr; - netdata_thread_cleanup_push(global_statistics_cleanup, ptr); + global_statistics_register_workers(); int update_every = (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); @@ -4280,7 +4278,6 @@ void *global_statistics_main(void *ptr) #endif } - netdata_thread_cleanup_pop(1); return NULL; } @@ -4288,15 +4285,16 @@ void *global_statistics_main(void *ptr) // --------------------------------------------------------------------------------------------------------------------- // workers thread -static void global_statistics_workers_cleanup(void *ptr) +static void global_statistics_workers_cleanup(void *pptr) { - worker_unregister(); + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_info("cleaning up..."); + worker_unregister(); worker_utilization_finish(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; @@ -4304,41 +4302,41 @@ static void global_statistics_workers_cleanup(void *ptr) void *global_statistics_workers_main(void *ptr) { + CLEANUP_FUNCTION_REGISTER(global_statistics_workers_cleanup) cleanup_ptr = ptr; + global_statistics_register_workers(); - netdata_thread_cleanup_push(global_statistics_workers_cleanup, ptr) - { - int update_every = - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); - if (update_every < localhost->rrd_update_every) - update_every = localhost->rrd_update_every; + int update_every = + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); + if (update_every < localhost->rrd_update_every) + update_every = localhost->rrd_update_every; - usec_t step = update_every * USEC_PER_SEC; - heartbeat_t hb; - heartbeat_init(&hb); + usec_t step = update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); - while (service_running(SERVICE_COLLECTORS)) { - worker_is_idle(); - heartbeat_next(&hb, step); + while (service_running(SERVICE_COLLECTORS)) { + worker_is_idle(); + heartbeat_next(&hb, step); - worker_is_busy(WORKER_JOB_WORKERS); - worker_utilization_charts(); - } + worker_is_busy(WORKER_JOB_WORKERS); + worker_utilization_charts(); } - netdata_thread_cleanup_pop(1); + return NULL; } // --------------------------------------------------------------------------------------------------------------------- // sqlite3 thread -static void global_statistics_sqlite3_cleanup(void *ptr) +static void global_statistics_sqlite3_cleanup(void *pptr) { - worker_unregister(); + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + worker_unregister(); netdata_log_info("cleaning up..."); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; @@ -4346,29 +4344,27 @@ static void global_statistics_sqlite3_cleanup(void *ptr) void *global_statistics_sqlite3_main(void *ptr) { - global_statistics_register_workers(); + CLEANUP_FUNCTION_REGISTER(global_statistics_sqlite3_cleanup) cleanup_ptr = ptr; - netdata_thread_cleanup_push(global_statistics_sqlite3_cleanup, ptr) - { + global_statistics_register_workers(); - int update_every = - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); - if (update_every < localhost->rrd_update_every) - update_every = localhost->rrd_update_every; + int update_every = + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); + if (update_every < localhost->rrd_update_every) + update_every = localhost->rrd_update_every; - usec_t step = update_every * USEC_PER_SEC; - heartbeat_t hb; - heartbeat_init(&hb); + usec_t step = update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); - while (service_running(SERVICE_COLLECTORS)) { - worker_is_idle(); - heartbeat_next(&hb, step); + while (service_running(SERVICE_COLLECTORS)) { + worker_is_idle(); + heartbeat_next(&hb, step); - worker_is_busy(WORKER_JOB_SQLITE3); - sqlite3_statistics_charts(); - } + worker_is_busy(WORKER_JOB_SQLITE3); + sqlite3_statistics_charts(); } - netdata_thread_cleanup_pop(1); + return NULL; } diff --git a/src/daemon/main.c b/src/daemon/main.c index 8f11b2169176e5..f7cf7210cdfef4 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -42,12 +42,12 @@ typedef struct service_thread { pid_t tid; SERVICE_THREAD_TYPE type; SERVICE_TYPE services; - char name[NETDATA_THREAD_NAME_MAX + 1]; + char name[ND_THREAD_TAG_MAX + 1]; bool stop_immediately; bool cancelled; union { - netdata_thread_t netdata_thread; + ND_THREAD *netdata_thread; uv_thread_t uv_thread; }; @@ -65,7 +65,7 @@ struct service_globals { SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t request_quit_callback, force_quit_t force_quit_callback, void *data, bool update __maybe_unused) { SERVICE_THREAD *sth = NULL; - pid_t tid = gettid(); + pid_t tid = gettid_cached(); spinlock_lock(&service_globals.lock); Pvoid_t *PValue = JudyLIns(&service_globals.pid_judy, tid, PJE0); @@ -76,13 +76,12 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t sth->request_quit_callback = request_quit_callback; sth->force_quit_callback = force_quit_callback; sth->data = data; - os_thread_get_current_name_np(sth->name); *PValue = sth; switch(thread_type) { default: case SERVICE_THREAD_TYPE_NETDATA: - sth->netdata_thread = netdata_thread_self(); + sth->netdata_thread = nd_thread_self(); break; case SERVICE_THREAD_TYPE_EVENT_LOOP: @@ -90,6 +89,10 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t sth->uv_thread = uv_thread_self(); break; } + + const char *name = nd_thread_tag(); + if(!name) name = ""; + strncpyz(sth->name, name, sizeof(sth->name) - 1); } else { sth = *PValue; @@ -100,7 +103,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t } void service_exits(void) { - pid_t tid = gettid(); + pid_t tid = gettid_cached(); spinlock_lock(&service_globals.lock); Pvoid_t *PValue = JudyLGet(service_globals.pid_judy, tid, PJE0); @@ -119,7 +122,7 @@ bool service_running(SERVICE_TYPE service) { sth->services |= service; - return !(sth->stop_immediately || netdata_exit); + return !sth->stop_immediately && !netdata_exit && !nd_thread_signaled_to_cancel(); } void service_signal_exit(SERVICE_TYPE service) { @@ -134,6 +137,9 @@ void service_signal_exit(SERVICE_TYPE service) { if((sth->services & service)) { sth->stop_immediately = true; + // this does not harm - it just raises a flag + nd_thread_signal_cancel(sth->netdata_thread); + if(sth->request_quit_callback) { spinlock_unlock(&service_globals.lock); sth->request_quit_callback(sth->data); @@ -196,13 +202,13 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) { bool first = true; while((PValue = JudyLFirstThenNext(service_globals.pid_judy, &tid, &first))) { SERVICE_THREAD *sth = *PValue; - if(sth->services & service && sth->tid != gettid() && !sth->cancelled) { + if(sth->services & service && sth->tid != gettid_cached() && !sth->cancelled) { sth->cancelled = true; switch(sth->type) { default: case SERVICE_THREAD_TYPE_NETDATA: - netdata_thread_cancel(sth->netdata_thread); + nd_thread_signal_cancel(sth->netdata_thread); break; case SERVICE_THREAD_TYPE_EVENT_LOOP: @@ -253,7 +259,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) { bool first = true; while((PValue = JudyLFirstThenNext(service_globals.pid_judy, &tid, &first))) { SERVICE_THREAD *sth = *PValue; - if(sth->services & service && sth->tid != gettid()) { + if(sth->services & service && sth->tid != gettid_cached()) { if(running) buffer_strcat(thread_list, ", "); @@ -667,7 +673,7 @@ void cancel_main_threads() { if (static_threads[i].enabled == NETDATA_MAIN_THREAD_RUNNING) { if (static_threads[i].thread) { netdata_log_info("EXIT: Stopping main thread: %s", static_threads[i].name); - netdata_thread_cancel(*static_threads[i].thread); + nd_thread_signal_cancel(static_threads[i].thread); } else { netdata_log_info("EXIT: No thread running (marking as EXITED): %s", static_threads[i].name); static_threads[i].enabled = NETDATA_MAIN_THREAD_EXITED; @@ -688,7 +694,7 @@ void cancel_main_threads() { continue; // Don't wait ourselves. - if (static_threads[i].thread && (*static_threads[i].thread == pthread_self())) + if (nd_thread_is_me(static_threads[i].thread)) continue; found++; @@ -704,9 +710,6 @@ void cancel_main_threads() { else netdata_log_info("All threads finished."); - for (i = 0; static_threads[i].name != NULL ; i++) - freez(static_threads[i].thread); - freez(static_threads); static_threads = NULL; } @@ -821,6 +824,10 @@ int help(int exitcode) { " Check if string matches pattern and exit.\n\n" " -W \"claim -token=TOKEN -rooms=ROOM1,ROOM2\"\n" " Claim the agent to the workspace rooms pointed to by TOKEN and ROOM*.\n\n" +#ifdef COMPILED_FOR_WINDOWS + " -W perflibdump [key]\n" + " Dump the Windows Performance Counters Registry in JSON.\n\n" +#endif ); fprintf(stream, "\n Signals netdata handles:\n\n" @@ -1254,9 +1261,9 @@ static void get_netdata_configured_variables() { // -------------------------------------------------------------------- // get various system parameters - get_system_HZ(); - get_system_cpus_uncached(); - get_system_pid_max(); + os_get_system_HZ(); + os_get_system_cpus_uncached(); + os_get_system_pid_max(); } @@ -1383,6 +1390,10 @@ int uuid_unittest(void); int progress_unittest(void); int dyncfg_unittest(void); +#ifdef COMPILED_FOR_WINDOWS +int windows_perflib_dump(const char *key); +#endif + int unittest_prepare_rrd(char **user) { post_conf_load(user); get_netdata_configured_variables(); @@ -1590,6 +1601,11 @@ int main(int argc, char **argv) { unittest_running = true; return uuid_unittest(); } +#ifdef COMPILED_FOR_WINDOWS + else if(strcmp(optarg, "perflibdump") == 0) { + return windows_perflib_dump(optind + 1 > argc ? NULL : argv[optind]); + } +#endif #ifdef ENABLE_DBENGINE else if(strcmp(optarg, "mctest") == 0) { unittest_running = true; @@ -2110,9 +2126,13 @@ int main(int argc, char **argv) { delta_startup_time("become daemon"); +#if defined(COMPILED_FOR_LINUX) || defined(COMPILED_FOR_MACOS) || defined(COMPILED_FOR_FREEBSD) // fork, switch user, create pid file, set process priority if(become_daemon(dont_fork, user) == -1) fatal("Cannot daemonize myself."); +#else + (void)dont_fork; +#endif watcher_thread_start(); @@ -2228,9 +2248,8 @@ int main(int argc, char **argv) { struct netdata_static_thread *st = &static_threads[i]; if(st->enabled) { - st->thread = mallocz(sizeof(netdata_thread_t)); netdata_log_debug(D_SYSTEM, "Starting thread %s.", st->name); - netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, st); + st->thread = nd_thread_create(st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, st); } else netdata_log_debug(D_SYSTEM, "Not starting thread %s.", st->name); @@ -2266,10 +2285,9 @@ int main(int argc, char **argv) { for (i = 0; static_threads[i].name != NULL; i++) { if (!strncmp(static_threads[i].name, "ANALYTICS", 9)) { struct netdata_static_thread *st = &static_threads[i]; - st->thread = mallocz(sizeof(netdata_thread_t)); st->enabled = 1; netdata_log_debug(D_SYSTEM, "Starting thread %s.", st->name); - netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, st); + st->thread = nd_thread_create(st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, st); } } } diff --git a/src/daemon/service.c b/src/daemon/service.c index ff966c57d24ad3..41f40aab154bcd 100644 --- a/src/daemon/service.c +++ b/src/daemon/service.c @@ -218,7 +218,7 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() { netdata_mutex_unlock(&host->receiver_lock); } - rrd_unlock(); + rrd_rdunlock(); } static void svc_rrdhost_cleanup_orphan_hosts(RRDHOST *protected_host) { @@ -259,12 +259,14 @@ static void svc_rrdhost_cleanup_orphan_hosts(RRDHOST *protected_host) { goto restart_after_removal; } - rrd_unlock(); + rrd_wrunlock(); } -static void service_main_cleanup(void *ptr) +static void service_main_cleanup(void *pptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_debug(D_SYSTEM, "Cleaning up..."); @@ -294,7 +296,8 @@ void *service_main(void *ptr) worker_register_job_name(WORKER_JOB_PGC_OPEN_EVICT, "open cache evictions"); worker_register_job_name(WORKER_JOB_PGC_OPEN_FLUSH, "open cache flushes"); - netdata_thread_cleanup_push(service_main_cleanup, ptr); + CLEANUP_FUNCTION_REGISTER(service_main_cleanup) cleanup_ptr = ptr; + heartbeat_t hb; heartbeat_init(&hb); usec_t step = USEC_PER_SEC * SERVICE_HEARTBEAT; @@ -317,6 +320,5 @@ void *service_main(void *ptr) svc_rrdhost_cleanup_orphan_hosts(localhost); } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/daemon/static_threads.c b/src/daemon/static_threads.c index 9f0db6e1af6adc..c655a4f1965a07 100644 --- a/src/daemon/static_threads.c +++ b/src/daemon/static_threads.c @@ -105,6 +105,8 @@ const struct netdata_static_thread static_threads_common[] = { .init_routine = NULL, .start_routine = statsd_main }, +#ifndef COMPILED_FOR_WINDOWS + // this crashes the debugger under windows { .name = "EXPORTING", .config_section = NULL, @@ -114,6 +116,7 @@ const struct netdata_static_thread static_threads_common[] = { .init_routine = NULL, .start_routine = exporting_main }, +#endif { .name = "SNDR[localhost]", .config_section = NULL, diff --git a/src/daemon/static_threads.h b/src/daemon/static_threads.h index 46195cf469a8fd..9c9be7620257e9 100644 --- a/src/daemon/static_threads.h +++ b/src/daemon/static_threads.h @@ -6,9 +6,6 @@ #include "common.h" extern const struct netdata_static_thread static_threads_common[]; -extern const struct netdata_static_thread static_threads_linux[]; -extern const struct netdata_static_thread static_threads_freebsd[]; -extern const struct netdata_static_thread static_threads_macos[]; struct netdata_static_thread * static_threads_concat(const struct netdata_static_thread *lhs, diff --git a/src/daemon/static_threads_freebsd.c b/src/daemon/static_threads_freebsd.c index b354b9c2a53846..1bb671a68c93ec 100644 --- a/src/daemon/static_threads_freebsd.c +++ b/src/daemon/static_threads_freebsd.c @@ -2,10 +2,10 @@ #include "common.h" -extern void *freebsd_main(void *ptr); -extern void *timex_main(void *ptr); +void *freebsd_main(void *ptr); +void *timex_main(void *ptr); -const struct netdata_static_thread static_threads_freebsd[] = { +static const struct netdata_static_thread static_threads_freebsd[] = { { .name = "P[freebsd]", .config_section = CONFIG_SECTION_PLUGINS, @@ -28,14 +28,6 @@ const struct netdata_static_thread static_threads_freebsd[] = { {NULL, NULL, NULL, 0, NULL, NULL, NULL} }; -const struct netdata_static_thread static_threads_linux[] = { - {NULL, NULL, NULL, 0, NULL, NULL, NULL} -}; - -const struct netdata_static_thread static_threads_macos[] = { - {NULL, NULL, NULL, 0, NULL, NULL, NULL} -}; - struct netdata_static_thread *static_threads_get() { return static_threads_concat(static_threads_common, static_threads_freebsd); } diff --git a/src/daemon/static_threads_linux.c b/src/daemon/static_threads_linux.c index 6939189978c031..1efd63755528ce 100644 --- a/src/daemon/static_threads_linux.c +++ b/src/daemon/static_threads_linux.c @@ -2,13 +2,13 @@ #include "common.h" -extern void *cgroups_main(void *ptr); -extern void *proc_main(void *ptr); -extern void *diskspace_main(void *ptr); -extern void *tc_main(void *ptr); -extern void *timex_main(void *ptr); +void *cgroups_main(void *ptr); +void *proc_main(void *ptr); +void *diskspace_main(void *ptr); +void *tc_main(void *ptr); +void *timex_main(void *ptr); -const struct netdata_static_thread static_threads_linux[] = { +static const struct netdata_static_thread static_threads_linux[] = { { .name = "P[tc]", .config_section = CONFIG_SECTION_PLUGINS, @@ -68,34 +68,6 @@ const struct netdata_static_thread static_threads_linux[] = { } }; -const struct netdata_static_thread static_threads_freebsd[] = { - // terminator - { - .name = NULL, - .config_section = NULL, - .config_name = NULL, - .env_name = NULL, - .enabled = 0, - .thread = NULL, - .init_routine = NULL, - .start_routine = NULL - } -}; - -const struct netdata_static_thread static_threads_macos[] = { - // terminator - { - .name = NULL, - .config_section = NULL, - .config_name = NULL, - .env_name = NULL, - .enabled = 0, - .thread = NULL, - .init_routine = NULL, - .start_routine = NULL - } -}; - struct netdata_static_thread *static_threads_get() { return static_threads_concat(static_threads_common, static_threads_linux); } diff --git a/src/daemon/static_threads_macos.c b/src/daemon/static_threads_macos.c index f4a33e85538655..3b417c0b2653fa 100644 --- a/src/daemon/static_threads_macos.c +++ b/src/daemon/static_threads_macos.c @@ -2,10 +2,10 @@ #include "common.h" -extern void *macos_main(void *ptr); -extern void *timex_main(void *ptr); +void *macos_main(void *ptr); +void *timex_main(void *ptr); -const struct netdata_static_thread static_threads_macos[] = { +static const struct netdata_static_thread static_threads_macos[] = { { .name = "P[timex]", .config_section = CONFIG_SECTION_PLUGINS, @@ -30,14 +30,6 @@ const struct netdata_static_thread static_threads_macos[] = { {NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL} }; -const struct netdata_static_thread static_threads_freebsd[] = { - {NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL} -}; - -const struct netdata_static_thread static_threads_linux[] = { - {NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL} -}; - struct netdata_static_thread *static_threads_get() { return static_threads_concat(static_threads_common, static_threads_macos); } diff --git a/src/daemon/static_threads_windows.c b/src/daemon/static_threads_windows.c new file mode 100644 index 00000000000000..57c47be0c03307 --- /dev/null +++ b/src/daemon/static_threads_windows.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "common.h" + +void *win_plugin_main(void *ptr); + +static const struct netdata_static_thread static_threads_windows[] = { + { + .name = "P[windows]", + .config_section = CONFIG_SECTION_PLUGINS, + .config_name = "windows", + .enabled = 1, + .thread = NULL, + .init_routine = NULL, + .start_routine = win_plugin_main + }, + + // terminator + { + .name = NULL, + .config_section = NULL, + .config_name = NULL, + .env_name = NULL, + .enabled = 0, + .thread = NULL, + .init_routine = NULL, + .start_routine = NULL + } +}; + +struct netdata_static_thread *static_threads_get() { + return static_threads_concat(static_threads_common, static_threads_windows); +} diff --git a/src/daemon/watcher.c b/src/daemon/watcher.c index 3eea22019d60e4..1e0090e241970c 100644 --- a/src/daemon/watcher.c +++ b/src/daemon/watcher.c @@ -6,7 +6,7 @@ watcher_step_t *watcher_steps; static struct completion shutdown_begin_completion; static struct completion shutdown_end_completion; -static netdata_thread_t watcher_thread; +static ND_THREAD *watcher_thread; void watcher_shutdown_begin(void) { completion_mark_complete(&shutdown_begin_completion); @@ -39,13 +39,13 @@ static void watcher_wait_for_step(const watcher_step_id_t step_id) if (ok) { netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds", - step_id + 1, WATCHER_STEP_ID_MAX, + (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, watcher_steps[step_id].msg, step_duration / USEC_PER_MS); } else { // Do not call fatal() because it will try to execute the exit // sequence twice. netdata_log_error("shutdown step: [%d/%d] - '%s' took more than %u seconds (ie. %llu milliseconds)", - step_id + 1, WATCHER_STEP_ID_MAX, watcher_steps[step_id].msg, + (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, watcher_steps[step_id].msg, timeout, step_duration / USEC_PER_MS); abort(); @@ -161,11 +161,11 @@ void watcher_thread_start() { completion_init(&shutdown_begin_completion); completion_init(&shutdown_end_completion); - netdata_thread_create(&watcher_thread, "P[WATCHER]", NETDATA_THREAD_OPTION_JOINABLE, watcher_main, NULL); + watcher_thread = nd_thread_create("P[WATCHER]", NETDATA_THREAD_OPTION_JOINABLE, watcher_main, NULL); } void watcher_thread_stop() { - netdata_thread_join(watcher_thread, NULL); + nd_thread_join(watcher_thread); for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) { completion_destroy(&watcher_steps[i].p); diff --git a/src/database/contexts/api_v2.c b/src/database/contexts/api_v2.c index edfeab1d636b9d..a5c759d92fcfae 100644 --- a/src/database/contexts/api_v2.c +++ b/src/database/contexts/api_v2.c @@ -1580,9 +1580,9 @@ static void contexts_v2_alerts_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_ struct sql_alert_transition_fixed_size { usec_t global_id; - uuid_t transition_id; - uuid_t host_id; - uuid_t config_hash_id; + nd_uuid_t transition_id; + nd_uuid_t host_id; + nd_uuid_t config_hash_id; uint32_t alarm_id; char alert_name[SQL_TRANSITION_DATA_SMALL_STRING]; char chart[RRD_ID_LENGTH_MAX]; diff --git a/src/database/contexts/instance.c b/src/database/contexts/instance.c index 117953d3898624..5d841bc82f07fd 100644 --- a/src/database/contexts/instance.c +++ b/src/database/contexts/instance.c @@ -137,7 +137,7 @@ static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_un "RRDINSTANCE: '%s' cannot change id to '%s'", string2str(ri->id), string2str(ri_new->id)); - if(uuid_memcmp(&ri->uuid, &ri_new->uuid) != 0) { + if(!uuid_eq(ri->uuid, ri_new->uuid)) { #ifdef NETDATA_INTERNAL_CHECKS char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(ri->uuid, uuid1); @@ -156,7 +156,7 @@ static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_un } #ifdef NETDATA_INTERNAL_CHECKS - if(ri->rrdset && uuid_memcmp(&ri->uuid, &ri->rrdset->chart_uuid) != 0) { + if(ri->rrdset && !uuid_eq(ri->uuid, ri->rrdset->chart_uuid)) { char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(ri->uuid, uuid1); uuid_unparse(ri->rrdset->chart_uuid, uuid2); diff --git a/src/database/contexts/internal.h b/src/database/contexts/internal.h index 293659fdd0a00d..270c59649663c2 100644 --- a/src/database/contexts/internal.h +++ b/src/database/contexts/internal.h @@ -198,7 +198,7 @@ rrd_flag_add_remove_atomic(RRD_FLAGS *flags, RRD_FLAGS check, RRD_FLAGS conditio typedef struct rrdmetric { - uuid_t uuid; + nd_uuid_t uuid; STRING *id; STRING *name; @@ -213,7 +213,7 @@ typedef struct rrdmetric { } RRDMETRIC; typedef struct rrdinstance { - uuid_t uuid; + nd_uuid_t uuid; STRING *id; STRING *name; diff --git a/src/database/contexts/metric.c b/src/database/contexts/metric.c index 0f0785972dfe1a..be29f33eadbb8a 100644 --- a/src/database/contexts/metric.c +++ b/src/database/contexts/metric.c @@ -108,7 +108,7 @@ static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus "RRDMETRIC: '%s' cannot change id to '%s'", string2str(rm->id), string2str(rm_new->id)); - if(uuid_memcmp(&rm->uuid, &rm_new->uuid) != 0) { + if(!uuid_eq(rm->uuid, rm_new->uuid)) { #ifdef NETDATA_INTERNAL_CHECKS char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(rm->uuid, uuid1); @@ -150,7 +150,7 @@ static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus } #ifdef NETDATA_INTERNAL_CHECKS - if(rm->rrddim && uuid_memcmp(&rm->uuid, &rm->rrddim->metric_uuid) != 0) { + if(rm->rrddim && !uuid_eq(rm->uuid, rm->rrddim->metric_uuid)) { char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(rm->uuid, uuid1); uuid_unparse(rm_new->uuid, uuid2); diff --git a/src/database/contexts/rrdcontext.c b/src/database/contexts/rrdcontext.c index 9dee39be2a437d..f755e1f7ed1f05 100644 --- a/src/database/contexts/rrdcontext.c +++ b/src/database/contexts/rrdcontext.c @@ -105,7 +105,7 @@ void rrdcontext_db_rotation(void) { rrdcontext_next_db_rotation_ut = now_realtime_usec() + FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS * USEC_PER_SEC; } -int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, uuid_t *store_uuid) { +int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, nd_uuid_t *store_uuid) { if(!st->rrdhost) return 1; if(!st->context) return 2; @@ -139,7 +139,7 @@ int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, uuid_t *store_uui return 0; } -int rrdcontext_find_chart_uuid(RRDSET *st, uuid_t *store_uuid) { +int rrdcontext_find_chart_uuid(RRDSET *st, nd_uuid_t *store_uuid) { if(!st->rrdhost) return 1; if(!st->context) return 2; @@ -203,23 +203,6 @@ static bool rrdhost_check_our_claim_id(const char *claim_id) { return (strcasecmp(claim_id, localhost->aclk_state.claimed_id) == 0) ? true : false; } -static RRDHOST *rrdhost_find_by_node_id(const char *node_id) { - uuid_t uuid; - if (uuid_parse(node_id, uuid)) - return NULL; - - RRDHOST *host = NULL; - dfe_start_read(rrdhost_root_index, host) { - if(!host->node_id) continue; - - if(uuid_memcmp(&uuid, host->node_id) == 0) - break; - } - dfe_done(host); - - return host; -} - void rrdcontext_hub_checkpoint_command(void *ptr) { struct ctxs_checkpoint *cmd = ptr; @@ -234,7 +217,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { return; } - RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id); + RRDHOST *host = find_host_by_node_id(cmd->node_id); if(!host) { nd_log(NDLS_DAEMON, NDLP_WARNING, "RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', " @@ -308,7 +291,7 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) { return; } - RRDHOST *host = rrdhost_find_by_node_id(cmd->node_id); + RRDHOST *host = find_host_by_node_id(cmd->node_id); if(!host) { nd_log(NDLS_DAEMON, NDLP_WARNING, "RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', " diff --git a/src/database/contexts/rrdcontext.h b/src/database/contexts/rrdcontext.h index 08a5760b55343f..9fea55d38cdca5 100644 --- a/src/database/contexts/rrdcontext.h +++ b/src/database/contexts/rrdcontext.h @@ -99,7 +99,7 @@ void rrdcontext_updated_rrddim_multiplier(RRDDIM *rd); void rrdcontext_updated_rrddim_divisor(RRDDIM *rd); void rrdcontext_updated_rrddim_flags(RRDDIM *rd); void rrdcontext_collected_rrddim(RRDDIM *rd); -int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, uuid_t *store_uuid); +int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, nd_uuid_t *store_uuid); // ---------------------------------------------------------------------------- // public API for rrdsets @@ -110,7 +110,7 @@ void rrdcontext_updated_rrdset_name(RRDSET *st); void rrdcontext_updated_rrdset_flags(RRDSET *st); void rrdcontext_updated_retention_rrdset(RRDSET *st); void rrdcontext_collected_rrdset(RRDSET *st); -int rrdcontext_find_chart_uuid(RRDSET *st, uuid_t *store_uuid); +int rrdcontext_find_chart_uuid(RRDSET *st, nd_uuid_t *store_uuid); // ---------------------------------------------------------------------------- // public API for ACLK @@ -165,7 +165,7 @@ typedef struct query_alerts_counts { // counts the number of alerts related t size_t other; // number of alerts in any other state } QUERY_ALERTS_COUNTS; -typedef struct query_node { +typedef struct _query_node { uint32_t slot; RRDHOST *rrdhost; char node_id[UUID_STR_LEN]; @@ -177,7 +177,7 @@ typedef struct query_node { QUERY_ALERTS_COUNTS alerts; } QUERY_NODE; -typedef struct query_context { +typedef struct _query_context { uint32_t slot; RRDCONTEXT_ACQUIRED *rca; @@ -187,7 +187,7 @@ typedef struct query_context { QUERY_ALERTS_COUNTS alerts; } QUERY_CONTEXT; -typedef struct query_instance { +typedef struct _query_instance { uint32_t slot; uint32_t query_host_id; RRDINSTANCE_ACQUIRED *ria; @@ -199,14 +199,14 @@ typedef struct query_instance { QUERY_ALERTS_COUNTS alerts; } QUERY_INSTANCE; -typedef struct query_dimension { +typedef struct _query_dimension { uint32_t slot; uint32_t priority; RRDMETRIC_ACQUIRED *rma; QUERY_STATUS status; } QUERY_DIMENSION; -typedef struct query_metric { +typedef struct _query_metric { RRDR_DIMENSION_FLAGS status; struct query_metric_tier { @@ -300,7 +300,7 @@ typedef struct query_target_request { qt_interrupt_callback_t interrupt_callback; void *interrupt_callback_data; - uuid_t *transaction; + nd_uuid_t *transaction; } QUERY_TARGET_REQUEST; #define GROUP_BY_MAX_LABEL_KEYS 10 @@ -421,9 +421,9 @@ typedef struct query_target { struct sql_alert_transition_data { usec_t global_id; - uuid_t *transition_id; - uuid_t *host_id; - uuid_t *config_hash_id; + nd_uuid_t *transition_id; + nd_uuid_t *host_id; + nd_uuid_t *config_hash_id; uint32_t alarm_id; const char *alert_name; const char *chart; @@ -454,7 +454,7 @@ struct sql_alert_transition_data { }; struct sql_alert_config_data { - uuid_t *config_hash_id; + nd_uuid_t *config_hash_id; const char *name; struct { @@ -539,9 +539,9 @@ struct sql_alert_instance_v2_entry { time_t last_updated; time_t last_status_change; NETDATA_DOUBLE last_status_change_value; - uuid_t config_hash_id; + nd_uuid_t config_hash_id; usec_t global_id; - uuid_t last_transition_id; + nd_uuid_t last_transition_id; uint32_t alarm_id; RRDHOST *host; size_t ni; diff --git a/src/database/contexts/worker.c b/src/database/contexts/worker.c index 2aae8363dbbf02..71af3c44dff4f4 100644 --- a/src/database/contexts/worker.c +++ b/src/database/contexts/worker.c @@ -1060,8 +1060,10 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now // ---------------------------------------------------------------------------- // worker thread -static void rrdcontext_main_cleanup(void *ptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; +static void rrdcontext_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; // custom code @@ -1071,83 +1073,82 @@ static void rrdcontext_main_cleanup(void *ptr) { } void *rrdcontext_main(void *ptr) { - netdata_thread_cleanup_push(rrdcontext_main_cleanup, ptr); - - worker_register("RRDCONTEXT"); - worker_register_job_name(WORKER_JOB_HOSTS, "hosts"); - worker_register_job_name(WORKER_JOB_CHECK, "dedup checks"); - worker_register_job_name(WORKER_JOB_SEND, "sent contexts"); - worker_register_job_name(WORKER_JOB_DEQUEUE, "deduplicated contexts"); - worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention"); - worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts"); - worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups"); - worker_register_job_name(WORKER_JOB_CLEANUP_DELETE, "deletes"); - worker_register_job_name(WORKER_JOB_PP_METRIC, "check metrics"); - worker_register_job_name(WORKER_JOB_PP_INSTANCE, "check instances"); - worker_register_job_name(WORKER_JOB_PP_CONTEXT, "check contexts"); - - worker_register_job_custom_metric(WORKER_JOB_HUB_QUEUE_SIZE, "hub queue size", "contexts", WORKER_METRIC_ABSOLUTE); - worker_register_job_custom_metric(WORKER_JOB_PP_QUEUE_SIZE, "post processing queue size", "contexts", WORKER_METRIC_ABSOLUTE); - - heartbeat_t hb; - heartbeat_init(&hb); - usec_t step = RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC; - - while (service_running(SERVICE_CONTEXT)) { - worker_is_idle(); - heartbeat_next(&hb, step); - - if(unlikely(!service_running(SERVICE_CONTEXT))) break; - - usec_t now_ut = now_realtime_usec(); - - if(rrdcontext_next_db_rotation_ut && now_ut > rrdcontext_next_db_rotation_ut) { - rrdcontext_recalculate_retention_all_hosts(); - rrdcontext_garbage_collect_for_all_hosts(); - rrdcontext_next_db_rotation_ut = 0; - } - - size_t hub_queued_contexts_for_all_hosts = 0; - size_t pp_queued_contexts_for_all_hosts = 0; + CLEANUP_FUNCTION_REGISTER(rrdcontext_main_cleanup) cleanup_ptr = ptr; + + worker_register("RRDCONTEXT"); + worker_register_job_name(WORKER_JOB_HOSTS, "hosts"); + worker_register_job_name(WORKER_JOB_CHECK, "dedup checks"); + worker_register_job_name(WORKER_JOB_SEND, "sent contexts"); + worker_register_job_name(WORKER_JOB_DEQUEUE, "deduplicated contexts"); + worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention"); + worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts"); + worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups"); + worker_register_job_name(WORKER_JOB_CLEANUP_DELETE, "deletes"); + worker_register_job_name(WORKER_JOB_PP_METRIC, "check metrics"); + worker_register_job_name(WORKER_JOB_PP_INSTANCE, "check instances"); + worker_register_job_name(WORKER_JOB_PP_CONTEXT, "check contexts"); + + worker_register_job_custom_metric(WORKER_JOB_HUB_QUEUE_SIZE, "hub queue size", "contexts", WORKER_METRIC_ABSOLUTE); + worker_register_job_custom_metric(WORKER_JOB_PP_QUEUE_SIZE, "post processing queue size", "contexts", WORKER_METRIC_ABSOLUTE); + + heartbeat_t hb; + heartbeat_init(&hb); + usec_t step = RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC; + + while (service_running(SERVICE_CONTEXT)) { + worker_is_idle(); + heartbeat_next(&hb, step); + + if(unlikely(!service_running(SERVICE_CONTEXT))) break; + + usec_t now_ut = now_realtime_usec(); + + if(rrdcontext_next_db_rotation_ut && now_ut > rrdcontext_next_db_rotation_ut) { + rrdcontext_recalculate_retention_all_hosts(); + rrdcontext_garbage_collect_for_all_hosts(); + rrdcontext_next_db_rotation_ut = 0; + } - RRDHOST *host; - dfe_start_reentrant(rrdhost_root_index, host) { - if(unlikely(!service_running(SERVICE_CONTEXT))) break; + size_t hub_queued_contexts_for_all_hosts = 0; + size_t pp_queued_contexts_for_all_hosts = 0; - worker_is_busy(WORKER_JOB_HOSTS); + RRDHOST *host; + dfe_start_reentrant(rrdhost_root_index, host) { + if(unlikely(!service_running(SERVICE_CONTEXT))) break; - if(host->rrdctx.pp_queue) { - pp_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.pp_queue); - rrdcontext_post_process_queued_contexts(host); - dictionary_garbage_collect(host->rrdctx.pp_queue); - } + worker_is_busy(WORKER_JOB_HOSTS); - if(host->rrdctx.hub_queue) { - hub_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.hub_queue); - rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut); - dictionary_garbage_collect(host->rrdctx.hub_queue); - } + if(host->rrdctx.pp_queue) { + pp_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.pp_queue); + rrdcontext_post_process_queued_contexts(host); + dictionary_garbage_collect(host->rrdctx.pp_queue); + } - if (host->rrdctx.contexts) - dictionary_garbage_collect(host->rrdctx.contexts); + if(host->rrdctx.hub_queue) { + hub_queued_contexts_for_all_hosts += dictionary_entries(host->rrdctx.hub_queue); + rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut); + dictionary_garbage_collect(host->rrdctx.hub_queue); + } - // calculate the number of metrics and instances in the host - RRDCONTEXT *rc; - uint32_t metrics = 0, instances = 0; - dfe_start_read(host->rrdctx.contexts, rc) { - metrics += rc->stats.metrics; - instances += dictionary_entries(rc->rrdinstances); - } - dfe_done(rc); - host->rrdctx.metrics = metrics; - host->rrdctx.instances = instances; - } - dfe_done(host); + if (host->rrdctx.contexts) + dictionary_garbage_collect(host->rrdctx.contexts); - worker_set_metric(WORKER_JOB_HUB_QUEUE_SIZE, (NETDATA_DOUBLE)hub_queued_contexts_for_all_hosts); - worker_set_metric(WORKER_JOB_PP_QUEUE_SIZE, (NETDATA_DOUBLE)pp_queued_contexts_for_all_hosts); + // calculate the number of metrics and instances in the host + RRDCONTEXT *rc; + uint32_t metrics = 0, instances = 0; + dfe_start_read(host->rrdctx.contexts, rc) { + metrics += rc->stats.metrics; + instances += dictionary_entries(rc->rrdinstances); } + dfe_done(rc); + host->rrdctx.metrics = metrics; + host->rrdctx.instances = instances; + } + dfe_done(host); + + worker_set_metric(WORKER_JOB_HUB_QUEUE_SIZE, (NETDATA_DOUBLE)hub_queued_contexts_for_all_hosts); + worker_set_metric(WORKER_JOB_PP_QUEUE_SIZE, (NETDATA_DOUBLE)pp_queued_contexts_for_all_hosts); + } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/database/engine/cache.c b/src/database/engine/cache.c index 49a9b6b964d482..a03df4676f0a71 100644 --- a/src/database/engine/cache.c +++ b/src/database/engine/cache.c @@ -1300,8 +1300,7 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) { if(unlikely(!page)) { // now that we don't have the lock, // give it some time for the old page to go away - struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; - nanosleep(&ns, NULL); + tinysleep(); } } @@ -2026,8 +2025,6 @@ void pgc_page_hot_set_end_time_s(PGC *cache __maybe_unused, PGC_PAGE *page, time } PGC_PAGE *pgc_page_get_and_acquire(PGC *cache, Word_t section, Word_t metric_id, time_t start_time_s, PGC_SEARCH method) { - static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; - PGC_PAGE *page = NULL; __atomic_add_fetch(&cache->stats.workers_search, 1, __ATOMIC_RELAXED); @@ -2053,7 +2050,7 @@ PGC_PAGE *pgc_page_get_and_acquire(PGC *cache, Word_t section, Word_t metric_id, if(page || !retry) break; - nanosleep(&ns, NULL); + tinysleep(); } if(page) { @@ -2374,8 +2371,6 @@ void *unittest_stress_test_collector(void *ptr) { while(!__atomic_load_n(&pgc_uts.stop, __ATOMIC_RELAXED)) { // netdata_log_info("COLLECTOR %zu: collecting metrics %zu to %zu, from %ld to %lu", id, metric_start, metric_end, start_time_t, start_time_t + pgc_uts.points_per_page); - netdata_thread_disable_cancelability(); - for (size_t i = metric_start; i < metric_end; i++) { bool added; @@ -2416,8 +2411,6 @@ void *unittest_stress_test_collector(void *ptr) { pgc_page_hot_to_dirty_and_release(pgc_uts.cache, pgc_uts.metrics[i], false); } } - - netdata_thread_enable_cancelability(); } return ptr; @@ -2431,8 +2424,6 @@ void *unittest_stress_test_queries(void *ptr) { size_t end = pgc_uts.clean_metrics + pgc_uts.hot_metrics; while(!__atomic_load_n(&pgc_uts.stop, __ATOMIC_RELAXED)) { - netdata_thread_disable_cancelability(); - int32_t random_number; random_r(random_data, &random_number); @@ -2482,8 +2473,6 @@ void *unittest_stress_test_queries(void *ptr) { pgc_page_release(pgc_uts.cache, array[i]); array[i] = NULL; } - - netdata_thread_enable_cancelability(); } return ptr; @@ -2527,7 +2516,7 @@ void unittest_stress_test(void) { pgc_uts.metrics = callocz(pgc_uts.clean_metrics + pgc_uts.hot_metrics, sizeof(PGC_PAGE *)); pthread_t service_thread; - netdata_thread_create(&service_thread, "SERVICE", + nd_thread_create(&service_thread, "SERVICE", NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, unittest_stress_test_service, NULL); @@ -2537,7 +2526,7 @@ void unittest_stress_test(void) { collect_thread_ids[i] = i; char buffer[100 + 1]; snprintfz(buffer, sizeof(buffer) - 1, "COLLECT_%zu", i); - netdata_thread_create(&collect_threads[i], buffer, + nd_thread_create(&collect_threads[i], buffer, NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, unittest_stress_test_collector, &collect_thread_ids[i]); } @@ -2550,7 +2539,7 @@ void unittest_stress_test(void) { char buffer[100 + 1]; snprintfz(buffer, sizeof(buffer) - 1, "QUERY_%zu", i); initstate_r(1, pgc_uts.rand_statebufs, 1024, &pgc_uts.random_data[i]); - netdata_thread_create(&queries_threads[i], buffer, + nd_thread_create(&queries_threads[i], buffer, NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, unittest_stress_test_queries, &query_thread_ids[i]); } @@ -2671,13 +2660,13 @@ void unittest_stress_test(void) { netdata_log_info("Waiting for threads to stop..."); __atomic_store_n(&pgc_uts.stop, true, __ATOMIC_RELAXED); - netdata_thread_join(service_thread, NULL); + nd_thread_join(service_thread, NULL); for(size_t i = 0; i < pgc_uts.collect_threads ;i++) - netdata_thread_join(collect_threads[i],NULL); + nd_thread_join(collect_threads[i],NULL); for(size_t i = 0; i < pgc_uts.query_threads ;i++) - netdata_thread_join(queries_threads[i],NULL); + nd_thread_join(queries_threads[i],NULL); pgc_destroy(pgc_uts.cache); diff --git a/src/database/engine/datafile.c b/src/database/engine/datafile.c index 1ec2dea799e8e1..39694f1599a5e0 100644 --- a/src/database/engine/datafile.c +++ b/src/database/engine/datafile.c @@ -526,7 +526,6 @@ int init_data_files(struct rrdengine_instance *ctx) { int ret; - fatal_assert(0 == uv_rwlock_init(&ctx->datafiles.rwlock)); ret = scan_data_files(ctx); if (ret < 0) { netdata_log_error("DBENGINE: failed to scan path \"%s\".", ctx->config.dbfiles_path); diff --git a/src/database/engine/dbengine-stresstest.c b/src/database/engine/dbengine-stresstest.c index 86d09c4ab23554..1be114e231b51e 100644 --- a/src/database/engine/dbengine-stresstest.c +++ b/src/database/engine/dbengine-stresstest.c @@ -186,7 +186,7 @@ void generate_dbengine_dataset(unsigned history_seconds) freez(thread_info); rrd_wrlock(); rrdhost_free___while_having_rrd_wrlock(localhost, true); - rrd_unlock(); + rrd_wrunlock(); } struct dbengine_query_thread { @@ -450,7 +450,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].si); rrdeng_exit((struct rrdengine_instance *)host->db[0].si); rrdeng_enq_cmd(NULL, RRDENG_OPCODE_SHUTDOWN_EVLOOP, NULL, NULL, STORAGE_PRIORITY_BEST_EFFORT, NULL, NULL); - rrd_unlock(); + rrd_wrunlock(); } #endif \ No newline at end of file diff --git a/src/database/engine/dbengine-unittest.c b/src/database/engine/dbengine-unittest.c index 4c4d312c08c47c..daa5fe57fca147 100644 --- a/src/database/engine/dbengine-unittest.c +++ b/src/database/engine/dbengine-unittest.c @@ -411,7 +411,7 @@ int test_dbengine(void) { rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].si); rrdeng_exit((struct rrdengine_instance *)host->db[0].si); rrdeng_enq_cmd(NULL, RRDENG_OPCODE_SHUTDOWN_EVLOOP, NULL, NULL, STORAGE_PRIORITY_BEST_EFFORT, NULL, NULL); - rrd_unlock(); + rrd_wrunlock(); return (int)(errors + value_errors + time_errors); } diff --git a/src/database/engine/journalfile.c b/src/database/engine/journalfile.c index 1e25ad5fd7790f..670c142cef01ca 100644 --- a/src/database/engine/journalfile.c +++ b/src/database/engine/journalfile.c @@ -670,7 +670,7 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx, time_t now_s = max_acceptable_collected_time(); for (i = 0; i < count ; ++i) { - uuid_t *temp_id; + nd_uuid_t *temp_id; uint8_t page_type = jf_metric_data->descr[i].type; if (page_type > RRDENG_PAGE_TYPE_MAX) { @@ -681,7 +681,7 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx, continue; } - temp_id = (uuid_t *)jf_metric_data->descr[i].uuid; + temp_id = (nd_uuid_t *)jf_metric_data->descr[i].uuid; METRIC *metric = mrg_metric_get_and_acquire(main_mrg, temp_id, (Word_t) ctx); struct rrdeng_extent_page_descr *descr = &jf_metric_data->descr[i]; @@ -1145,7 +1145,7 @@ static int journalfile_metric_compare (const void *item1, const void *item2) const struct jv2_metrics_info *metric1 = ((struct journal_metric_list_to_sort *) item1)->metric_info; const struct jv2_metrics_info *metric2 = ((struct journal_metric_list_to_sort *) item2)->metric_info; - return memcmp(metric1->uuid, metric2->uuid, sizeof(uuid_t)); + return memcmp(metric1->uuid, metric2->uuid, sizeof(nd_uuid_t)); } diff --git a/src/database/engine/journalfile.h b/src/database/engine/journalfile.h index 3f881ee16098ac..5bb38b00aa445d 100644 --- a/src/database/engine/journalfile.h +++ b/src/database/engine/journalfile.h @@ -84,7 +84,7 @@ struct journal_page_header { }; uint32_t uuid_offset; // Points back to the UUID list which should point here (UUIDs should much) uint32_t entries; // Entries - uuid_t uuid; // Which UUID this is + nd_uuid_t uuid; // Which UUID this is }; // 20 bytes @@ -100,7 +100,7 @@ struct journal_page_list { // UUID_LIST // 36 bytes struct journal_metric_list { - uuid_t uuid; + nd_uuid_t uuid; uint32_t entries; // Number of entries uint32_t page_offset; // OFFSET that contains entries * struct( journal_page_list ) uint32_t delta_start_s; // Min time of metric diff --git a/src/database/engine/metric.c b/src/database/engine/metric.c index 01eb22fbc2fb87..9b2ce6374319a8 100644 --- a/src/database/engine/metric.c +++ b/src/database/engine/metric.c @@ -8,7 +8,7 @@ typedef int32_t REFCOUNT; #define REFCOUNT_DELETING (-100) struct metric { - uuid_t uuid; // never changes + nd_uuid_t uuid; // never changes Word_t section; // never changes time_t first_time_s; // the timestamp of the oldest point in the database @@ -98,14 +98,14 @@ static inline void mrg_stats_size_judyl_change(MRG *mrg, size_t mem_before_judyl } static inline void mrg_stats_size_judyhs_added_uuid(MRG *mrg, size_t partition) { - __atomic_add_fetch(&mrg->index[partition].stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED); + __atomic_add_fetch(&mrg->index[partition].stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(nd_uuid_t)), __ATOMIC_RELAXED); } static inline void mrg_stats_size_judyhs_removed_uuid(MRG *mrg, size_t partition) { - __atomic_sub_fetch(&mrg->index[partition].stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED); + __atomic_sub_fetch(&mrg->index[partition].stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(nd_uuid_t)), __ATOMIC_RELAXED); } -static inline size_t uuid_partition(MRG *mrg __maybe_unused, uuid_t *uuid) { +static inline size_t uuid_partition(MRG *mrg __maybe_unused, nd_uuid_t *uuid) { uint8_t *u = (uint8_t *)uuid; size_t n; @@ -167,7 +167,7 @@ static inline void acquired_for_deletion_metric_delete(MRG *mrg, METRIC *metric) mrg_index_write_lock(mrg, partition); - Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, &metric->uuid, sizeof(uuid_t)); + Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, &metric->uuid, sizeof(nd_uuid_t)); if(unlikely(!sections_judy_pptr || !*sections_judy_pptr)) { MRG_STATS_DELETE_MISS(mrg, partition); mrg_index_write_unlock(mrg, partition); @@ -186,7 +186,7 @@ static inline void acquired_for_deletion_metric_delete(MRG *mrg, METRIC *metric) } if(!*sections_judy_pptr) { - rc = JudyHSDel(&mrg->index[partition].uuid_judy, &metric->uuid, sizeof(uuid_t), PJE0); + rc = JudyHSDel(&mrg->index[partition].uuid_judy, &metric->uuid, sizeof(nd_uuid_t), PJE0); if(unlikely(!rc)) fatal("DBENGINE METRIC: cannot delete UUID from JudyHS"); mrg_stats_size_judyhs_removed_uuid(mrg, partition); @@ -264,7 +264,7 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r size_t mem_before_judyl, mem_after_judyl; - Pvoid_t *sections_judy_pptr = JudyHSIns(&mrg->index[partition].uuid_judy, entry->uuid, sizeof(uuid_t), PJE0); + Pvoid_t *sections_judy_pptr = JudyHSIns(&mrg->index[partition].uuid_judy, entry->uuid, sizeof(nd_uuid_t), PJE0); if (unlikely(!sections_judy_pptr || sections_judy_pptr == PJERR)) fatal("DBENGINE METRIC: corrupted UUIDs JudyHS array"); @@ -323,13 +323,13 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r return metric; } -static inline METRIC *metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section) { +static inline METRIC *metric_get_and_acquire(MRG *mrg, nd_uuid_t *uuid, Word_t section) { size_t partition = uuid_partition(mrg, uuid); while(1) { mrg_index_read_lock(mrg, partition); - Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, uuid, sizeof(uuid_t)); + Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, uuid, sizeof(nd_uuid_t)); if (unlikely(!sections_judy_pptr)) { mrg_index_read_unlock(mrg, partition); MRG_STATS_SEARCH_MISS(mrg, partition); @@ -404,7 +404,7 @@ inline METRIC *mrg_metric_add_and_acquire(MRG *mrg, MRG_ENTRY entry, bool *ret) return metric_add_and_acquire(mrg, &entry, ret); } -inline METRIC *mrg_metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section) { +inline METRIC *mrg_metric_get_and_acquire(MRG *mrg, nd_uuid_t *uuid, Word_t section) { return metric_get_and_acquire(mrg, uuid, section); } @@ -425,7 +425,7 @@ inline Word_t mrg_metric_id(MRG *mrg __maybe_unused, METRIC *metric) { return (Word_t)metric; } -inline uuid_t *mrg_metric_uuid(MRG *mrg __maybe_unused, METRIC *metric) { +inline nd_uuid_t *mrg_metric_uuid(MRG *mrg __maybe_unused, METRIC *metric) { return &metric->uuid; } @@ -600,7 +600,7 @@ inline uint32_t mrg_metric_get_update_every_s(MRG *mrg __maybe_unused, METRIC *m inline bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) { pid_t expected = __atomic_load_n(&metric->writer, __ATOMIC_RELAXED); - pid_t wanted = gettid(); + pid_t wanted = gettid_cached(); bool done = true; do { @@ -639,7 +639,7 @@ inline bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric) { } inline void mrg_update_metric_retention_and_granularity_by_uuid( - MRG *mrg, Word_t section, uuid_t *uuid, + MRG *mrg, Word_t section, nd_uuid_t *uuid, time_t first_time_s, time_t last_time_s, uint32_t update_every_s, time_t now_s) { @@ -736,7 +736,7 @@ inline void mrg_get_statistics(MRG *mrg, struct mrg_statistics *s) { // unit test struct mrg_stress_entry { - uuid_t uuid; + nd_uuid_t uuid; time_t after; time_t before; }; @@ -757,13 +757,13 @@ static void *mrg_stress(void *ptr) { ssize_t end = (ssize_t)t->entries; ssize_t step = 1; - if(gettid() % 2) { + if(gettid_cached() % 2) { start = (ssize_t)t->entries - 1; end = -1; step = -1; } - while(!__atomic_load_n(&t->stop, __ATOMIC_RELAXED)) { + while(!__atomic_load_n(&t->stop, __ATOMIC_RELAXED) && !nd_thread_signaled_to_cancel()) { for (ssize_t i = start; i != end; i += step) { struct mrg_stress_entry *e = &t->array[i]; @@ -791,7 +791,7 @@ int mrg_unittest(void) { METRIC *m1_t1, *m2_t1, *m3_t1, *m4_t1; bool ret; - uuid_t test_uuid; + nd_uuid_t test_uuid; uuid_generate(test_uuid); MRG_ENTRY entry = { .uuid = &test_uuid, @@ -902,23 +902,21 @@ int mrg_unittest(void) { usec_t started_ut = now_monotonic_usec(); - pthread_t th[threads]; + ND_THREAD *th[threads]; for(size_t i = 0; i < threads ; i++) { char buf[15 + 1]; snprintfz(buf, sizeof(buf) - 1, "TH[%zu]", i); - netdata_thread_create(&th[i], buf, - NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, - mrg_stress, &t); + th[i] = nd_thread_create(buf, NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, mrg_stress, &t); } sleep_usec(run_for_secs * USEC_PER_SEC); __atomic_store_n(&t.stop, true, __ATOMIC_RELAXED); for(size_t i = 0; i < threads ; i++) - netdata_thread_cancel(th[i]); + nd_thread_signal_cancel(th[i]); for(size_t i = 0; i < threads ; i++) - netdata_thread_join(th[i], NULL); + nd_thread_join(th[i]); usec_t ended_ut = now_monotonic_usec(); diff --git a/src/database/engine/metric.h b/src/database/engine/metric.h index 3bace90574e25f..038a90e02f3b8e 100644 --- a/src/database/engine/metric.h +++ b/src/database/engine/metric.h @@ -10,7 +10,7 @@ typedef struct metric METRIC; typedef struct mrg MRG; typedef struct mrg_entry { - uuid_t *uuid; + nd_uuid_t *uuid; Word_t section; time_t first_time_s; time_t last_time_s; @@ -55,11 +55,11 @@ METRIC *mrg_metric_dup(MRG *mrg, METRIC *metric); void mrg_metric_release(MRG *mrg, METRIC *metric); METRIC *mrg_metric_add_and_acquire(MRG *mrg, MRG_ENTRY entry, bool *ret); -METRIC *mrg_metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section); +METRIC *mrg_metric_get_and_acquire(MRG *mrg, nd_uuid_t *uuid, Word_t section); bool mrg_metric_release_and_delete(MRG *mrg, METRIC *metric); Word_t mrg_metric_id(MRG *mrg, METRIC *metric); -uuid_t *mrg_metric_uuid(MRG *mrg, METRIC *metric); +nd_uuid_t *mrg_metric_uuid(MRG *mrg, METRIC *metric); Word_t mrg_metric_section(MRG *mrg, METRIC *metric); bool mrg_metric_set_first_time_s(MRG *mrg, METRIC *metric, time_t first_time_s); @@ -88,7 +88,7 @@ size_t mrg_aral_overhead(void); void mrg_update_metric_retention_and_granularity_by_uuid( - MRG *mrg, Word_t section, uuid_t *uuid, + MRG *mrg, Word_t section, nd_uuid_t *uuid, time_t first_time_s, time_t last_time_s, uint32_t update_every_s, time_t now_s); diff --git a/src/database/engine/page.c b/src/database/engine/page.c index 13fe90f7f26228..5c4ac14e773246 100644 --- a/src/database/engine/page.c +++ b/src/database/engine/page.c @@ -183,7 +183,7 @@ PGD *pgd_create(uint8_t type, uint32_t slots) pg->slots = 8 * RRDENG_GORILLA_32BIT_BUFFER_SLOTS; // allocate new gorilla writer - pg->gorilla.aral_index = gettid() % 4; + pg->gorilla.aral_index = gettid_cached() % 4; pg->gorilla.writer = aral_mallocz(pgd_alloc_globals.aral_gorilla_writer[pg->gorilla.aral_index]); // allocate new gorilla buffer diff --git a/src/database/engine/pagecache.c b/src/database/engine/pagecache.c index 452fdc50ba03b4..a88992223c91f8 100644 --- a/src/database/engine/pagecache.c +++ b/src/database/engine/pagecache.c @@ -491,7 +491,7 @@ static size_t list_has_time_gaps( typedef void (*page_found_callback_t)(PGC_PAGE *page, void *data); static size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METRIC *metric, usec_t start_time_ut, usec_t end_time_ut, page_found_callback_t callback, void *callback_data) { - uuid_t *uuid = mrg_metric_uuid(main_mrg, metric); + nd_uuid_t *uuid = mrg_metric_uuid(main_mrg, metric); Word_t metric_id = mrg_metric_id(main_mrg, metric); time_t wanted_start_time_s = (time_t)(start_time_ut / USEC_PER_SEC); diff --git a/src/database/engine/pagecache.h b/src/database/engine/pagecache.h index 103d3648435f60..f8b86c3dc43b80 100644 --- a/src/database/engine/pagecache.h +++ b/src/database/engine/pagecache.h @@ -18,7 +18,7 @@ struct rrdengine_instance; extern struct rrdeng_cache_efficiency_stats rrdeng_cache_efficiency_stats; struct page_descr_with_data { - uuid_t *id; + nd_uuid_t *id; Word_t metric_id; usec_t start_time_ut; usec_t end_time_ut; diff --git a/src/database/engine/pdc.c b/src/database/engine/pdc.c index 79a424b773b03f..f22d63b45a6d92 100644 --- a/src/database/engine/pdc.c +++ b/src/database/engine/pdc.c @@ -651,7 +651,7 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_ } return validate_page( - (uuid_t *)descr->uuid, + (nd_uuid_t *)descr->uuid, start_time_s, end_time_s, 0, @@ -665,7 +665,7 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_ } VALIDATED_PAGE_DESCRIPTOR validate_page( - uuid_t *uuid, + nd_uuid_t *uuid, time_t start_time_s, time_t end_time_s, uint32_t update_every_s, // can be zero, if unknown @@ -898,7 +898,7 @@ static void epdl_extent_loading_error_log(struct rrdengine_instance *ctx, EPDL * start_time_s = pd->first_time_s; end_time_s = pd->last_time_s; METRIC *metric = (METRIC *)pd->metric_id; - uuid_t *u = mrg_metric_uuid(main_mrg, metric); + nd_uuid_t *u = mrg_metric_uuid(main_mrg, metric); uuid_unparse_lower(*u, uuid); used_epdl = true; } diff --git a/src/database/engine/rrdengine.c b/src/database/engine/rrdengine.c index 87c1e3ca0b34f5..f6759b6b6f49ce 100644 --- a/src/database/engine/rrdengine.c +++ b/src/database/engine/rrdengine.c @@ -111,7 +111,7 @@ static void sanity_check(void) /* Data file super-block cannot be larger than RRDENG_BLOCK_SIZE */ BUILD_BUG_ON(RRDENG_DF_SB_PADDING_SZ < 0); - BUILD_BUG_ON(sizeof(uuid_t) != UUID_SZ); /* check UUID size */ + BUILD_BUG_ON(sizeof(nd_uuid_t) != UUID_SZ); /* check UUID size */ /* page count must fit in 8 bits */ BUILD_BUG_ON(MAX_PAGES_PER_EXTENT > 255); @@ -233,7 +233,7 @@ static void after_work_standard_callback(uv_work_t* req, int status) { static bool work_dispatch(struct rrdengine_instance *ctx, void *data, struct completion *completion, enum rrdeng_opcode opcode, work_cb do_work_cb, after_work_cb do_after_work_cb) { struct rrdeng_work *work_request = NULL; - internal_fatal(rrdeng_main.tid != gettid(), "work_dispatch() can only be run from the event loop thread"); + internal_fatal(rrdeng_main.tid != gettid_cached(), "work_dispatch() can only be run from the event loop thread"); work_request = aral_mallocz(rrdeng_main.work_cmd.ar); memset(work_request, 0, sizeof(struct rrdeng_work)); @@ -824,7 +824,7 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta for (i = 0 ; i < count ; ++i) { descr = xt_io_descr->descr_array[i]; header->descr[i].type = descr->type; - uuid_copy(*(uuid_t *)header->descr[i].uuid, *descr->id); + uuid_copy(*(nd_uuid_t *)header->descr[i].uuid, *descr->id); header->descr[i].page_length = descr->page_length; header->descr[i].start_time_ut = descr->start_time_ut; @@ -935,7 +935,7 @@ static void after_database_rotate(struct rrdengine_instance *ctx __maybe_unused, } struct uuid_first_time_s { - uuid_t *uuid; + nd_uuid_t *uuid; time_t first_time_s; METRIC *metric; size_t pages_found; @@ -1690,7 +1690,7 @@ static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_w void dbengine_event_loop(void* arg) { sanity_check(); - uv_thread_set_name_np(pthread_self(), "DBENGINE"); + uv_thread_set_name_np("DBENGINE"); service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true); worker_register("DBENGINE"); @@ -1734,7 +1734,7 @@ void dbengine_event_loop(void* arg) { struct rrdeng_main *main = arg; enum rrdeng_opcode opcode; struct rrdeng_cmd cmd; - main->tid = gettid(); + main->tid = gettid_cached(); fatal_assert(0 == uv_timer_start(&main->timer, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS)); diff --git a/src/database/engine/rrdengine.h b/src/database/engine/rrdengine.h index ab30ce8770f40d..c594efe992d905 100644 --- a/src/database/engine/rrdengine.h +++ b/src/database/engine/rrdengine.h @@ -3,9 +3,6 @@ #ifndef NETDATA_RRDENGINE_H #define NETDATA_RRDENGINE_H -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif #include #include #include @@ -143,7 +140,7 @@ struct jv2_extents_info { }; struct jv2_metrics_info { - uuid_t *uuid; + nd_uuid_t *uuid; uint32_t page_list_header; time_t first_time_s; time_t last_time_s; @@ -497,7 +494,7 @@ typedef struct validated_page_descriptor { #define page_entries_by_size(page_length_in_bytes, point_size_in_bytes) \ ((page_length_in_bytes) / (point_size_in_bytes)) -VALIDATED_PAGE_DESCRIPTOR validate_page(uuid_t *uuid, +VALIDATED_PAGE_DESCRIPTOR validate_page(nd_uuid_t *uuid, time_t start_time_s, time_t end_time_s, uint32_t update_every_s, @@ -526,8 +523,18 @@ static inline time_t max_acceptable_collected_time(void) { void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool update_retention, bool worker); +// -------------------------------------------------------------------------------------------------------------------- +// the following functions are used to sort UUIDs in the journal files +// DO NOT CHANGE, as this will break backwards compatibility with the data files users have. + +static inline int journal_uuid_memcmp(const nd_uuid_t *uu1, const nd_uuid_t *uu2) { + return memcmp(uu1, uu2, sizeof(nd_uuid_t)); +} + static inline int journal_metric_uuid_compare(const void *key, const void *metric) { - return uuid_memcmp((uuid_t *)key, &(((struct journal_metric_list *) metric)->uuid)); + return journal_uuid_memcmp((const nd_uuid_t *)key, (const nd_uuid_t *)&(((struct journal_metric_list *) metric)->uuid)); } +// -------------------------------------------------------------------------------------------------------------------- + #endif /* NETDATA_RRDENGINE_H */ diff --git a/src/database/engine/rrdengineapi.c b/src/database/engine/rrdengineapi.c index f5967984091e32..313b5463270b6c 100755 --- a/src/database/engine/rrdengineapi.c +++ b/src/database/engine/rrdengineapi.c @@ -5,18 +5,18 @@ #include "dbengine-compression.h" /* Default global database instance */ -struct rrdengine_instance multidb_ctx_storage_tier0; -struct rrdengine_instance multidb_ctx_storage_tier1; -struct rrdengine_instance multidb_ctx_storage_tier2; -struct rrdengine_instance multidb_ctx_storage_tier3; -struct rrdengine_instance multidb_ctx_storage_tier4; +struct rrdengine_instance multidb_ctx_storage_tier0 = { 0 }; +struct rrdengine_instance multidb_ctx_storage_tier1 = { 0 }; +struct rrdengine_instance multidb_ctx_storage_tier2 = { 0 }; +struct rrdengine_instance multidb_ctx_storage_tier3 = { 0 }; +struct rrdengine_instance multidb_ctx_storage_tier4 = { 0 }; #define mrg_metric_ctx(metric) (struct rrdengine_instance *)mrg_metric_section(main_mrg, metric) #if RRD_STORAGE_TIERS != 5 #error RRD_STORAGE_TIERS is not 5 - you need to add allocations here #endif -struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS]; +struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS] = { 0 }; uint8_t tier_page_type[RRD_STORAGE_TIERS] = { RRDENG_PAGE_TYPE_GORILLA_32BIT, RRDENG_PAGE_TYPE_ARRAY_TIER1, @@ -40,12 +40,21 @@ size_t page_type_size[256] = { [RRDENG_PAGE_TYPE_GORILLA_32BIT] = sizeof(storage_number) }; +static inline void initialize_single_ctx(struct rrdengine_instance *ctx) { + memset(ctx, 0, sizeof(*ctx)); + uv_rwlock_init(&ctx->datafiles.rwlock); + rw_spinlock_init(&ctx->njfv2idx.spinlock); +} + __attribute__((constructor)) void initialize_multidb_ctx(void) { multidb_ctx[0] = &multidb_ctx_storage_tier0; multidb_ctx[1] = &multidb_ctx_storage_tier1; multidb_ctx[2] = &multidb_ctx_storage_tier2; multidb_ctx[3] = &multidb_ctx_storage_tier3; multidb_ctx[4] = &multidb_ctx_storage_tier4; + + for(int i = 0; i < RRD_STORAGE_TIERS ; i++) + initialize_single_ctx(multidb_ctx[i]); } int db_engine_journal_check = 0; @@ -80,7 +89,7 @@ static inline bool rrdeng_page_alignment_release(struct pg_alignment *pa) { } // charts call this -STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si __maybe_unused, uuid_t *uuid __maybe_unused) { +STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si __maybe_unused, nd_uuid_t *uuid __maybe_unused) { struct pg_alignment *pa = callocz(1, sizeof(struct pg_alignment)); rrdeng_page_alignment_acquire(pa); return (STORAGE_METRICS_GROUP *)pa; @@ -98,17 +107,17 @@ void rrdeng_metrics_group_release(STORAGE_INSTANCE *si __maybe_unused, STORAGE_M // metric handle for legacy dbs /* This UUID is not unique across hosts */ -void rrdeng_generate_unittest_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid) +void rrdeng_generate_unittest_uuid(const char *dim_id, const char *chart_id, nd_uuid_t *ret_uuid) { CLEAN_BUFFER *wb = buffer_create(100, NULL); buffer_sprintf(wb,"%s.%s", dim_id, chart_id); - UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb)); + ND_UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb)); uuid_copy(*ret_uuid, uuid.uuid); } static METRIC *rrdeng_metric_unittest(STORAGE_INSTANCE *si, const char *rd_id, const char *st_id) { struct rrdengine_instance *ctx = (struct rrdengine_instance *)si; - uuid_t legacy_uuid; + nd_uuid_t legacy_uuid; rrdeng_generate_unittest_uuid(rd_id, st_id, &legacy_uuid); return mrg_metric_get_and_acquire(main_mrg, &legacy_uuid, (Word_t) ctx); } @@ -126,12 +135,12 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *smh) { return (STORAGE_METRIC_HANDLE *) mrg_metric_dup(main_mrg, metric); } -STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *si, uuid_t *uuid) { +STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid) { struct rrdengine_instance *ctx = (struct rrdengine_instance *)si; return (STORAGE_METRIC_HANDLE *) mrg_metric_get_and_acquire(main_mrg, uuid, (Word_t) ctx); } -static METRIC *rrdeng_metric_create(STORAGE_INSTANCE *si, uuid_t *uuid) { +static METRIC *rrdeng_metric_create(STORAGE_INSTANCE *si, nd_uuid_t *uuid) { internal_fatal(!si, "DBENGINE: STORAGE_INSTANCE is NULL"); struct rrdengine_instance *ctx = (struct rrdengine_instance *)si; @@ -168,7 +177,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE } #ifdef NETDATA_INTERNAL_CHECKS - if(uuid_memcmp(&rd->metric_uuid, mrg_metric_uuid(main_mrg, metric)) != 0) { + if(!uuid_eq(rd->metric_uuid, *mrg_metric_uuid(main_mrg, metric))) { char uuid1[UUID_STR_LEN + 1]; char uuid2[UUID_STR_LEN + 1]; @@ -208,7 +217,7 @@ static inline bool check_completed_page_consistency(struct rrdeng_collect_handle struct rrdengine_instance *ctx = mrg_metric_ctx(handle->metric); - uuid_t *uuid = mrg_metric_uuid(main_mrg, handle->metric); + nd_uuid_t *uuid = mrg_metric_uuid(main_mrg, handle->metric); time_t start_time_s = pgc_page_start_time_s(handle->pgc_page); time_t end_time_s = pgc_page_end_time_s(handle->pgc_page); uint32_t update_every_s = pgc_page_update_every_s(handle->pgc_page); @@ -687,7 +696,7 @@ void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *sch SPINLOCK global_query_handle_spinlock = NETDATA_SPINLOCK_INITIALIZER; static struct rrdeng_query_handle *global_query_handle_ll = NULL; static void register_query_handle(struct rrdeng_query_handle *handle) { - handle->query_pid = gettid(); + handle->query_pid = gettid_cached(); handle->started_time_s = now_realtime_sec(); spinlock_lock(&global_query_handle_spinlock); @@ -720,8 +729,6 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *smh, { usec_t started_ut = now_monotonic_usec(); - netdata_thread_disable_cancelability(); - METRIC *metric = (METRIC *)smh; struct rrdengine_instance *ctx = mrg_metric_ctx(metric); struct rrdeng_query_handle *handle; @@ -911,7 +918,6 @@ void rrdeng_load_metric_finalize(struct storage_engine_query_handle *seqh) unregister_query_handle(handle); rrdeng_query_handle_release(handle); seqh->handle = NULL; - netdata_thread_enable_cancelability(); } time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *seqh) { @@ -946,7 +952,7 @@ time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *smh) { return oldest_time_s; } -bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s) +bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, nd_uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s) { struct rrdengine_instance *ctx = (struct rrdengine_instance *)si; if (unlikely(!ctx)) { @@ -1143,12 +1149,12 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path, return UV_EMFILE; } - if(ctxp) - *ctxp = ctx = callocz(1, sizeof(*ctx)); - else { - ctx = multidb_ctx[tier]; - memset(ctx, 0, sizeof(*ctx)); + if(ctxp) { + *ctxp = ctx = mallocz(sizeof(*ctx)); + initialize_single_ctx(ctx); } + else + ctx = multidb_ctx[tier]; ctx->config.tier = (int)tier; ctx->config.page_type = tier_page_type[tier]; @@ -1162,7 +1168,6 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path, ctx->atomic.transaction_id = 1; ctx->quiesce.enabled = false; - rw_spinlock_init(&ctx->njfv2idx.spinlock); ctx->atomic.first_time_s = LONG_MAX; ctx->atomic.metrics = 0; ctx->atomic.samples = 0; diff --git a/src/database/engine/rrdengineapi.h b/src/database/engine/rrdengineapi.h index 4d49421b5a1317..4809858cab690d 100644 --- a/src/database/engine/rrdengineapi.h +++ b/src/database/engine/rrdengineapi.h @@ -25,7 +25,7 @@ extern uint8_t tier_page_type[]; #define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type] STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si); -STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *si, uuid_t *uuid); +STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid); void rrdeng_metric_release(STORAGE_METRIC_HANDLE *smh); STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *smh); @@ -62,9 +62,9 @@ void rrdeng_exit_mode(struct rrdengine_instance *ctx); int rrdeng_exit(struct rrdengine_instance *ctx); void rrdeng_prepare_exit(struct rrdengine_instance *ctx); -bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s); +bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, nd_uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s); -extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, uuid_t *uuid); +extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid); extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg); typedef struct rrdengine_size_statistics { diff --git a/src/database/ram/rrddim_mem.c b/src/database/ram/rrddim_mem.c index 718dd357cb853e..fab1479a192a6e 100644 --- a/src/database/ram/rrddim_mem.c +++ b/src/database/ram/rrddim_mem.c @@ -9,7 +9,7 @@ static netdata_rwlock_t rrddim_JudyHS_rwlock = NETDATA_RWLOCK_INITIALIZER; // ---------------------------------------------------------------------------- // metrics groups -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si __maybe_unused, uuid_t *uuid __maybe_unused) { +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si __maybe_unused, nd_uuid_t *uuid __maybe_unused) { return NULL; } @@ -52,7 +52,7 @@ rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si __maybe_unused) { struct mem_metric_handle *mh = (struct mem_metric_handle *)rrddim_metric_get(si, &rd->metric_uuid); while(!mh) { netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock); - Pvoid_t *PValue = JudyHSIns(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0); + Pvoid_t *PValue = JudyHSIns(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(nd_uuid_t), PJE0); mh = *PValue; if(!mh) { mh = callocz(1, sizeof(struct mem_metric_handle)); @@ -60,13 +60,13 @@ rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si __maybe_unused) { mh->refcount = 1; update_metric_handle_from_rrddim(mh, rd); *PValue = mh; - __atomic_add_fetch(&rrddim_db_memory_size, sizeof(struct mem_metric_handle) + JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED); + __atomic_add_fetch(&rrddim_db_memory_size, sizeof(struct mem_metric_handle) + JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(nd_uuid_t)), __ATOMIC_RELAXED); } else { if(__atomic_add_fetch(&mh->refcount, 1, __ATOMIC_RELAXED) <= 0) mh = NULL; } - netdata_rwlock_unlock(&rrddim_JudyHS_rwlock); + netdata_rwlock_wrunlock(&rrddim_JudyHS_rwlock); } internal_fatal(mh->rd != rd, "RRDDIM_MEM: incorrect pointer returned from index."); @@ -75,16 +75,16 @@ rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si __maybe_unused) { } STORAGE_METRIC_HANDLE * -rrddim_metric_get(STORAGE_INSTANCE *si __maybe_unused, uuid_t *uuid) { +rrddim_metric_get(STORAGE_INSTANCE *si __maybe_unused, nd_uuid_t *uuid) { struct mem_metric_handle *mh = NULL; netdata_rwlock_rdlock(&rrddim_JudyHS_rwlock); - Pvoid_t *PValue = JudyHSGet(rrddim_JudyHS_array, uuid, sizeof(uuid_t)); + Pvoid_t *PValue = JudyHSGet(rrddim_JudyHS_array, uuid, sizeof(nd_uuid_t)); if (likely(NULL != PValue)) { mh = *PValue; if(__atomic_add_fetch(&mh->refcount, 1, __ATOMIC_RELAXED) <= 0) mh = NULL; } - netdata_rwlock_unlock(&rrddim_JudyHS_rwlock); + netdata_rwlock_rdunlock(&rrddim_JudyHS_rwlock); return (STORAGE_METRIC_HANDLE *)mh; } @@ -107,16 +107,16 @@ void rrddim_metric_release(STORAGE_METRIC_HANDLE *smh __maybe_unused) { RRDDIM *rd = mh->rd; netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock); - JudyHSDel(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0); - netdata_rwlock_unlock(&rrddim_JudyHS_rwlock); + JudyHSDel(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(nd_uuid_t), PJE0); + netdata_rwlock_wrunlock(&rrddim_JudyHS_rwlock); freez(mh); - __atomic_sub_fetch(&rrddim_db_memory_size, sizeof(struct mem_metric_handle) + JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED); + __atomic_sub_fetch(&rrddim_db_memory_size, sizeof(struct mem_metric_handle) + JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(nd_uuid_t)), __ATOMIC_RELAXED); } } } -bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *si __maybe_unused, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s) { +bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *si __maybe_unused, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s) { STORAGE_METRIC_HANDLE *smh = rrddim_metric_get(si, uuid); if(!smh) return false; diff --git a/src/database/ram/rrddim_mem.h b/src/database/ram/rrddim_mem.h index 166b8777ab9fb8..74ffd566324447 100644 --- a/src/database/ram/rrddim_mem.h +++ b/src/database/ram/rrddim_mem.h @@ -23,13 +23,13 @@ struct mem_query_handle { }; STORAGE_METRIC_HANDLE *rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si); -STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *si, uuid_t *uuid); +STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid); STORAGE_METRIC_HANDLE *rrddim_metric_dup(STORAGE_METRIC_HANDLE *smh); void rrddim_metric_release(STORAGE_METRIC_HANDLE *smh); -bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s); +bool rrddim_metric_retention_by_uuid(STORAGE_INSTANCE *si, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s); -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, uuid_t *uuid); +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid); void rrddim_metrics_group_release(STORAGE_INSTANCE *si, STORAGE_METRICS_GROUP *smg); STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t update_every, STORAGE_METRICS_GROUP *smg); diff --git a/src/database/rrd.h b/src/database/rrd.h index 50ad44254e3a4c..a006fe7a057abf 100644 --- a/src/database/rrd.h +++ b/src/database/rrd.h @@ -270,7 +270,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s // RRD DIMENSION - this is a metric struct rrddim { - uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) + nd_uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) // ------------------------------------------------------------------------ // dimension definition @@ -361,9 +361,9 @@ size_t rrddim_size(void); // ------------------------------------------------------------------------ // DATA COLLECTION STORAGE OPS -STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, uuid_t *uuid); -STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, uuid_t *uuid); -static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, uuid_t *uuid) { +STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid); +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *si, nd_uuid_t *uuid); +static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND seb __maybe_unused, STORAGE_INSTANCE *si, nd_uuid_t *uuid) { internal_fatal(!is_valid_backend(seb), "STORAGE: invalid backend"); #ifdef ENABLE_DBENGINE @@ -635,11 +635,11 @@ static inline time_t storage_engine_align_to_optimal_before(struct storage_engin // function pointers for all APIs provided by a storage engine typedef struct storage_engine_api { // metric management - STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *si, uuid_t *uuid); + STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *si, nd_uuid_t *uuid); STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *si); void (*metric_release)(STORAGE_METRIC_HANDLE *); STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *); - bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *si, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s); + bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *si, nd_uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s); } STORAGE_ENGINE_API; typedef struct storage_engine { @@ -719,7 +719,7 @@ struct pluginsd_rrddim { }; struct rrdset { - uuid_t chart_uuid; // the global UUID for this chart + nd_uuid_t chart_uuid; // the global UUID for this chart // ------------------------------------------------------------------------ // chart configuration @@ -1025,8 +1025,8 @@ struct alarm_entry { uint32_t alarm_id; uint32_t alarm_event_id; usec_t global_id; - uuid_t config_hash_id; - uuid_t transition_id; + nd_uuid_t config_hash_id; + nd_uuid_t transition_id; time_t when; time_t duration; @@ -1231,7 +1231,7 @@ struct rrdhost { // the following are state information for the threading // streaming metrics from this netdata to an upstream netdata struct sender_state *sender; - netdata_thread_t rrdpush_sender_thread; // the sender thread + ND_THREAD *rrdpush_sender_thread; // the sender thread size_t rrdpush_sender_replicating_charts; // the number of charts currently being replicated to a parent struct aclk_sync_cfg_t *aclk_config; @@ -1307,8 +1307,8 @@ struct rrdhost { time_t last_time_s; } retention; - uuid_t host_uuid; // Global GUID for this host - uuid_t *node_id; // Cloud node_id + nd_uuid_t host_uuid; // Global GUID for this host + nd_uuid_t *node_id; // Cloud node_id netdata_mutex_t aclk_state_lock; aclk_rrdhost_state aclk_state; @@ -1365,7 +1365,8 @@ extern netdata_rwlock_t rrd_rwlock; #define rrd_rdlock() netdata_rwlock_rdlock(&rrd_rwlock) #define rrd_wrlock() netdata_rwlock_wrlock(&rrd_rwlock) -#define rrd_unlock() netdata_rwlock_unlock(&rrd_rwlock) +#define rrd_rdunlock() netdata_rwlock_rdunlock(&rrd_rwlock) +#define rrd_wrunlock() netdata_rwlock_wrunlock(&rrd_rwlock) // ---------------------------------------------------------------------------- diff --git a/src/database/rrdcollector.c b/src/database/rrdcollector.c index 1a116c0c2e6318..59c5c459d971d5 100644 --- a/src/database/rrdcollector.c +++ b/src/database/rrdcollector.c @@ -68,7 +68,7 @@ void rrd_collector_started(void) { if(!thread_rrd_collector) thread_rrd_collector = callocz(1, sizeof(struct rrd_collector)); - thread_rrd_collector->tid = gettid(); + thread_rrd_collector->tid = gettid_cached(); __atomic_store_n(&thread_rrd_collector->running, true, __ATOMIC_RELAXED); } diff --git a/src/database/rrdfunctions-inflight.c b/src/database/rrdfunctions-inflight.c index 585a6d2699c409..adb27b3e7d7d92 100644 --- a/src/database/rrdfunctions-inflight.c +++ b/src/database/rrdfunctions-inflight.c @@ -10,7 +10,7 @@ struct rrd_function_inflight { bool used; RRDHOST *host; - uuid_t transaction_uuid; + nd_uuid_t transaction_uuid; const char *transaction; const char *cmd; const char *sanitized_cmd; @@ -491,7 +491,7 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, // validate and parse the transaction, or generate a new transaction id char uuid_str[UUID_COMPACT_STR_LEN]; - uuid_t uuid; + nd_uuid_t uuid; if(!transaction || !*transaction || uuid_parse_flexi(transaction, uuid) != 0) uuid_generate_random(uuid); @@ -597,7 +597,7 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, return rrd_call_function_async(r, wait); } -bool rrd_function_has_this_original_result_callback(uuid_t *transaction, rrd_function_result_callback_t cb) { +bool rrd_function_has_this_original_result_callback(nd_uuid_t *transaction, rrd_function_result_callback_t cb) { bool ret = false; char str[UUID_COMPACT_STR_LEN]; uuid_unparse_lower_compact(*transaction, str); @@ -684,7 +684,7 @@ void rrd_function_progress(const char *transaction) { dictionary_acquired_item_release(rrd_functions_inflight_requests, item); } -void rrd_function_call_progresser(uuid_t *transaction) { +void rrd_function_call_progresser(nd_uuid_t *transaction) { char str[UUID_COMPACT_STR_LEN]; uuid_unparse_lower_compact(*transaction, str); rrd_function_progress(str); diff --git a/src/database/rrdfunctions-inflight.h b/src/database/rrdfunctions-inflight.h index 3cad3178476e5e..8fd43f0afea4ad 100644 --- a/src/database/rrdfunctions-inflight.h +++ b/src/database/rrdfunctions-inflight.h @@ -11,6 +11,6 @@ void rrd_functions_inflight_init(void); void rrd_function_cancel(const char *transaction); void rrd_function_progress(const char *transaction); -void rrd_function_call_progresser(uuid_t *transaction); +void rrd_function_call_progresser(nd_uuid_t *transaction); #endif //NETDATA_RRDFUNCTIONS_INFLIGHT_H diff --git a/src/database/rrdfunctions.c b/src/database/rrdfunctions.c index cb7cdab367628a..9411c4c3f74bd4 100644 --- a/src/database/rrdfunctions.c +++ b/src/database/rrdfunctions.c @@ -333,10 +333,10 @@ int rrd_functions_find_by_name(RRDHOST *host, BUFFER *wb, const char *name, size s = &buffer[key_length - 1]; // skip a word from the end - while (s >= buffer && !isspace(*s)) *s-- = '\0'; + while (s >= buffer && !isspace((uint8_t)*s)) *s-- = '\0'; // skip all spaces - while (s >= buffer && isspace(*s)) *s-- = '\0'; + while (s >= buffer && isspace((uint8_t)*s)) *s-- = '\0'; } } diff --git a/src/database/rrdfunctions.h b/src/database/rrdfunctions.h index c9df26f4d6cf37..d3c7f0e13df33f 100644 --- a/src/database/rrdfunctions.h +++ b/src/database/rrdfunctions.h @@ -19,7 +19,7 @@ typedef void (*rrd_function_progresser_cb_t)(void *data); typedef void (*rrd_function_register_progresser_cb_t)(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data); struct rrd_function_execute { - uuid_t *transaction; + nd_uuid_t *transaction; const char *function; BUFFER *payload; const char *source; @@ -85,7 +85,7 @@ int rrd_call_function_error(BUFFER *wb, const char *msg, int code); bool rrd_function_available(RRDHOST *host, const char *function); -bool rrd_function_has_this_original_result_callback(uuid_t *transaction, rrd_function_result_callback_t cb); +bool rrd_function_has_this_original_result_callback(nd_uuid_t *transaction, rrd_function_result_callback_t cb); #include "rrdfunctions-inline.h" #include "rrdfunctions-inflight.h" diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c index a2f6d4d1ae9474..02e2d7da455543 100644 --- a/src/database/rrdhost.c +++ b/src/database/rrdhost.c @@ -35,13 +35,13 @@ time_t rrdhost_free_ephemeral_time_s = 86400; RRDHOST *find_host_by_node_id(char *node_id) { - uuid_t node_uuid; + nd_uuid_t node_uuid; if (unlikely(!node_id || uuid_parse(node_id, node_uuid))) return NULL; RRDHOST *host, *ret = NULL; dfe_start_read(rrdhost_root_index, host) { - if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) { + if (host->node_id && uuid_eq(*host->node_id, node_uuid)) { ret = host; break; } @@ -319,7 +319,7 @@ static RRDHOST *prepare_host_for_unittest(RRDHOST *host) rrd_wrlock(); rrdhost_free___while_having_rrd_wrlock(host, true); - rrd_unlock(); + rrd_wrunlock(); return NULL; } return host; @@ -492,7 +492,7 @@ static RRDHOST *rrdhost_create( if (!is_localhost) rrdhost_free___while_having_rrd_wrlock(host, true); - rrd_unlock(); + rrd_wrunlock(); return NULL; } @@ -503,7 +503,7 @@ static RRDHOST *rrdhost_create( else DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(localhost, host, prev, next); - rrd_unlock(); + rrd_wrunlock(); // ------------------------------------------------------------------------ @@ -732,7 +732,7 @@ RRDHOST *rrdhost_find_or_create( rrd_wrlock(); rrdhost_free___while_having_rrd_wrlock(host, true); host = NULL; - rrd_unlock(); + rrd_wrunlock(); } if(!host) { @@ -811,7 +811,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim #ifdef ENABLE_DBENGINE struct dbengine_initialization { - netdata_thread_t thread; + ND_THREAD *thread; char path[FILENAME_MAX + 1]; int disk_space_mb; size_t tier; @@ -937,8 +937,7 @@ void dbengine_init(char *hostname) { if(parallel_initialization) { char tag[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier); - netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE, - dbengine_tier_init, &tiers_init[tier]); + tiers_init[tier].thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, dbengine_tier_init, &tiers_init[tier]); } else dbengine_tier_init(&tiers_init[tier]); @@ -947,10 +946,8 @@ void dbengine_init(char *hostname) { config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); for(size_t tier = 0; tier < storage_tiers ;tier++) { - void *ptr; - if(parallel_initialization) - netdata_thread_join(tiers_init[tier].thread, &ptr); + nd_thread_join(tiers_init[tier].thread); if(tiers_init[tier].ret != 0) { nd_log(NDLS_DAEMON, NDLP_ERR, @@ -1289,7 +1286,7 @@ void rrdhost_free_all(void) { if(localhost) rrdhost_free___while_having_rrd_wrlock(localhost, true); - rrd_unlock(); + rrd_wrunlock(); } void rrd_finalize_collection_for_all_hosts(void) { diff --git a/src/database/rrdset.c b/src/database/rrdset.c index cb4407b529617f..e5273532f2dea0 100644 --- a/src/database/rrdset.c +++ b/src/database/rrdset.c @@ -1081,7 +1081,7 @@ void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_las last_time_s = now.tv_sec; if(min_delta > permanent_min_delta) { - netdata_log_info("MINIMUM MICROSECONDS DELTA of thread %d increased from %"PRIi64" to %"PRIi64" (+%"PRIi64")", gettid(), permanent_min_delta, min_delta, min_delta - permanent_min_delta); + netdata_log_info("MINIMUM MICROSECONDS DELTA of thread %d increased from %"PRIi64" to %"PRIi64" (+%"PRIi64")", gettid_cached(), permanent_min_delta, min_delta, min_delta - permanent_min_delta); permanent_min_delta = min_delta; } @@ -1313,6 +1313,7 @@ void store_metric_collection_completed() { struct rda_item { const DICTIONARY_ITEM *item; RRDDIM *rd; + bool reset_or_overflow; }; static __thread struct rda_item *thread_rda = NULL; @@ -1353,7 +1354,6 @@ static inline size_t rrdset_done_interpolate( , usec_t last_collect_ut , usec_t now_collect_ut , char store_this_entry - , uint32_t has_reset_value ) { RRDDIM *rd; @@ -1368,11 +1368,6 @@ static inline size_t rrdset_done_interpolate( size_t counter = st->counter; long current_entry = st->db.current_entry; - SN_FLAGS storage_flags = SN_DEFAULT_FLAGS; - - if (has_reset_value) - storage_flags |= SN_FLAG_RESET; - for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) { internal_error(iterations < 0, @@ -1398,6 +1393,11 @@ static inline size_t rrdset_done_interpolate( rd = rda->rd; if(unlikely(!rd)) continue; + SN_FLAGS storage_flags = SN_DEFAULT_FLAGS; + + if (rda->reset_or_overflow) + storage_flags |= SN_FLAG_RESET; + NETDATA_DOUBLE new_value; switch(rd->algorithm) { @@ -1514,9 +1514,6 @@ static inline size_t rrdset_done_interpolate( ml_chart_update_end(st); - // reset the storage flags for the next point, if any; - storage_flags = SN_DEFAULT_FLAGS; - st->counter = ++counter; st->db.current_entry = current_entry = ((current_entry + 1) >= st->db.entries) ? 0 : current_entry + 1; @@ -1678,8 +1675,6 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) if(stream_buffer.wb && !stream_buffer.v2) rrdset_push_metrics_v1(&stream_buffer, st); - uint32_t has_reset_value = 0; - size_t rda_slots = dictionary_entries(st->rrddim_root_index); struct rda_item *rda_base = rrdset_thread_rda_get(&rda_slots); @@ -1697,12 +1692,14 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { rda->item = NULL; rda->rd = NULL; + rda->reset_or_overflow = false; continue; } // store the dimension in the array rda->item = dictionary_acquired_item_dup(st->rrddim_root_index, rd_dfe.item); rda->rd = dictionary_acquired_item_value(rda->item); + rda->reset_or_overflow = false; // calculate totals if(likely(rrddim_check_updated(rd))) { @@ -1717,7 +1714,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) ); if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS))) - has_reset_value = 1; + rda->reset_or_overflow = true; rd->collector.last_collected_value = rd->collector.collected_value; } @@ -1820,7 +1817,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) , rd->collector.collected_value); if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS))) - has_reset_value = 1; + rda->reset_or_overflow = true; uint64_t last = (uint64_t)rd->collector.last_collected_value; uint64_t new = (uint64_t)rd->collector.collected_value; @@ -1944,7 +1941,6 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) , last_collect_ut , now_collect_ut , store_this_entry - , has_reset_value ); for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) { diff --git a/src/database/sqlite/sqlite_aclk.c b/src/database/sqlite/sqlite_aclk.c index bbb33076bd8a3b..c5a41e5d81998f 100644 --- a/src/database/sqlite/sqlite_aclk.c +++ b/src/database/sqlite/sqlite_aclk.c @@ -97,7 +97,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column is_registered = str2i(argv[IDX_IS_REGISTERED]); char guid[UUID_STR_LEN]; - uuid_unparse_lower(*(uuid_t *)argv[IDX_HOST_ID], guid); + uuid_unparse_lower(*(nd_uuid_t *)argv[IDX_HOST_ID], guid); if (is_ephemeral && age > rrdhost_free_ephemeral_time_s) { netdata_log_info( @@ -116,7 +116,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column system_info->hops = str2i((const char *) argv[IDX_HOPS]); - sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info); + sql_build_host_system_info((nd_uuid_t *)argv[IDX_HOST_ID], system_info); RRDHOST *host = rrdhost_find_or_create( (const char *)argv[IDX_HOSTNAME], @@ -157,7 +157,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column if (is_ephemeral) host->child_disconnected_time = now_realtime_sec(); - host->rrdlabels = sql_load_host_labels((uuid_t *)argv[IDX_HOST_ID]); + host->rrdlabels = sql_load_host_labels((nd_uuid_t *)argv[IDX_HOST_ID]); host->last_connected = last_connected; } @@ -175,7 +175,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column #ifdef ENABLE_ACLK #define SQL_SELECT_HOST_BY_UUID "SELECT host_id FROM host WHERE host_id = @host_id" -static int is_host_available(uuid_t *host_id) +static int is_host_available(nd_uuid_t *host_id) { sqlite3_stmt *res = NULL; int rc = 0; @@ -205,7 +205,7 @@ static void sql_delete_aclk_table_list(char *host_guid) char host_str[UUID_STR_LEN]; int rc; - uuid_t host_uuid; + nd_uuid_t host_uuid; if (unlikely(!host_guid)) return; @@ -249,7 +249,7 @@ static void sql_delete_aclk_table_list(char *host_guid) static void sql_unregister_node(char *machine_guid) { int rc; - uuid_t host_uuid; + nd_uuid_t host_uuid; if (unlikely(!machine_guid)) return; @@ -333,11 +333,11 @@ static void sql_maint_aclk_sync_database_all(void) static int aclk_config_parameters(void *data __maybe_unused, int argc __maybe_unused, char **argv, char **column __maybe_unused) { char uuid_str[UUID_STR_LEN]; - uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str); + uuid_unparse_lower(*((nd_uuid_t *) argv[0]), uuid_str); RRDHOST *host = rrdhost_find_by_guid(uuid_str); if (host != localhost) - sql_create_aclk_table(host, (uuid_t *) argv[0], (uuid_t *) argv[1]); + sql_create_aclk_table(host, (nd_uuid_t *) argv[0], (nd_uuid_t *) argv[1]); return 0; } @@ -374,7 +374,7 @@ static void timer_cb(uv_timer_t *handle) static void aclk_synchronization(void *arg __maybe_unused) { struct aclk_sync_config_s *config = arg; - uv_thread_set_name_np(config->thread, "ACLKSYNC"); + uv_thread_set_name_np("ACLKSYNC"); worker_register("ACLKSYNC"); service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true); @@ -487,7 +487,7 @@ static void aclk_synchronization_init(void) // ------------------------------------------------------------- -void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __maybe_unused, uuid_t *node_id __maybe_unused) +void sql_create_aclk_table(RRDHOST *host __maybe_unused, nd_uuid_t *host_uuid __maybe_unused, nd_uuid_t *node_id __maybe_unused) { #ifdef ENABLE_ACLK char uuid_str[UUID_STR_LEN]; diff --git a/src/database/sqlite/sqlite_aclk.h b/src/database/sqlite/sqlite_aclk.h index 7da04270d95f4b..ce9fed84063d02 100644 --- a/src/database/sqlite/sqlite_aclk.h +++ b/src/database/sqlite/sqlite_aclk.h @@ -9,7 +9,7 @@ #define ACLK_DELETE_ACK_ALERTS_INTERNAL (86400) #define ACLK_SYNC_QUERY_SIZE 512 -static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out) +static inline void uuid_unparse_lower_fix(nd_uuid_t *uuid, char *out) { uuid_unparse_lower(*uuid, out); out[8] = '_'; @@ -18,7 +18,7 @@ static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out) out[23] = '_'; } -static inline int uuid_parse_fix(char *in, uuid_t uuid) +static inline int uuid_parse_fix(char *in, nd_uuid_t uuid) { in[8] = '-'; in[13] = '-'; @@ -80,7 +80,7 @@ typedef struct aclk_sync_cfg_t { uint64_t alerts_log_last_sequence_id; } aclk_sync_cfg_t; -void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id); +void sql_create_aclk_table(RRDHOST *host, nd_uuid_t *host_uuid, nd_uuid_t *node_id); void sql_aclk_sync_init(void); void aclk_push_alert_config(const char *node_id, const char *config_hash); void aclk_push_node_alert_snapshot(const char *node_id); diff --git a/src/database/sqlite/sqlite_aclk_alert.c b/src/database/sqlite/sqlite_aclk_alert.c index 71717242b3ceb9..0982d32bd78747 100644 --- a/src/database/sqlite/sqlite_aclk_alert.c +++ b/src/database/sqlite/sqlite_aclk_alert.c @@ -45,7 +45,7 @@ static void update_filtered(ALARM_ENTRY *ae, int64_t unique_id, char *uuid_str) "WHERE hld.unique_id = @unique_id AND hl.config_hash_id = ah.hash_id AND hld.health_log_id = hl.health_log_id " \ "AND hl.host_id = @host_id AND ah.warn IS NULL AND ah.crit IS NULL" -static inline bool is_event_from_alert_variable_config(int64_t unique_id, uuid_t *host_id) +static inline bool is_event_from_alert_variable_config(int64_t unique_id, nd_uuid_t *host_id) { sqlite3_stmt *res = NULL; @@ -106,15 +106,15 @@ static bool should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae) param = 0; int rc = sqlite3_step_monitored(res); if (likely(rc == SQLITE_ROW)) { - uuid_t config_hash_id; + nd_uuid_t config_hash_id; RRDCALC_STATUS status = (RRDCALC_STATUS)sqlite3_column_int(res, 0); if (sqlite3_column_type(res, 1) != SQLITE_NULL) - uuid_copy(config_hash_id, *((uuid_t *)sqlite3_column_blob(res, 1))); + uuid_copy(config_hash_id, *((nd_uuid_t *)sqlite3_column_blob(res, 1))); int64_t unique_id = sqlite3_column_int64(res, 2); - if (ae->new_status != (RRDCALC_STATUS)status || uuid_memcmp(&ae->config_hash_id, &config_hash_id)) + if (ae->new_status != (RRDCALC_STATUS)status || !uuid_eq(ae->config_hash_id, config_hash_id)) send = true; else update_filtered(ae, unique_id, host->aclk_config->uuid_str); @@ -203,7 +203,7 @@ static inline char *sqlite3_uuid_unparse_strdupz(sqlite3_stmt *res, int iCol) { if(sqlite3_column_type(res, iCol) == SQLITE_NULL) uuid_str[0] = '\0'; else - uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, iCol)), uuid_str); + uuid_unparse_lower(*((nd_uuid_t *) sqlite3_column_blob(res, iCol)), uuid_str); return strdupz(uuid_str); } @@ -499,7 +499,7 @@ void aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_has if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_ALERT_CONFIG, &res)) return; - uuid_t hash_uuid; + nd_uuid_t hash_uuid; if (uuid_parse(config_hash, hash_uuid)) return; @@ -593,7 +593,7 @@ void aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_has // Start streaming alerts void aclk_start_alert_streaming(char *node_id, bool resets) { - uuid_t node_uuid; + nd_uuid_t node_uuid; if (unlikely(!node_id || uuid_parse(node_id, node_uuid))) return; @@ -675,7 +675,7 @@ void sql_queue_removed_alerts_to_aclk(RRDHOST *host) void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unused, char *snapshot_uuid) { - uuid_t node_uuid; + nd_uuid_t node_uuid; if (unlikely(!node_id || uuid_parse(node_id, node_uuid))) return; diff --git a/src/database/sqlite/sqlite_aclk_node.c b/src/database/sqlite/sqlite_aclk_node.c index dcc8c375cb2fc3..ce4dcc521d3cd1 100644 --- a/src/database/sqlite/sqlite_aclk_node.c +++ b/src/database/sqlite/sqlite_aclk_node.c @@ -113,7 +113,7 @@ static void build_node_info(RRDHOST *host) host->machine_guid, host == localhost ? "parent" : "child"); - rrd_unlock(); + rrd_rdunlock(); freez(node_info.claim_id); freez(node_info.node_instance_capabilities); freez(host_version); diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c index 022e60030bee87..1e49dd2bf877bb 100644 --- a/src/database/sqlite/sqlite_context.c +++ b/src/database/sqlite/sqlite_context.c @@ -78,7 +78,7 @@ int sql_init_context_database(int memory) #define CTX_GET_CHART_LIST "SELECT c.chart_id, c.type||'.'||c.id, c.name, c.context, c.title, c.unit, c.priority, " \ "c.update_every, c.chart_type, c.family FROM chart c WHERE c.host_id = @host_id AND c.chart_id IS NOT NULL" -void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data) +void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data) { static __thread sqlite3_stmt *res = NULL; @@ -96,7 +96,7 @@ void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, voi param = 0; SQL_CHART_DATA chart_data = { 0 }; while (sqlite3_step_monitored(res) == SQLITE_ROW) { - uuid_copy(chart_data.chart_id, *((uuid_t *)sqlite3_column_blob(res, 0))); + uuid_copy(chart_data.chart_id, *((nd_uuid_t *)sqlite3_column_blob(res, 0))); chart_data.id = (char *) sqlite3_column_text(res, 1); chart_data.name = (char *) sqlite3_column_text(res, 2); chart_data.context = (char *) sqlite3_column_text(res, 3); @@ -117,7 +117,7 @@ void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, voi // Dimension list #define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \ "FROM dimension d WHERE d.chart_id = @id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC" -void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data) +void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data) { static __thread sqlite3_stmt *res = NULL; @@ -131,7 +131,7 @@ void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DA param = 0; while (sqlite3_step_monitored(res) == SQLITE_ROW) { - uuid_copy(dimension_data.dim_id, *((uuid_t *)sqlite3_column_blob(res, 0))); + uuid_copy(dimension_data.dim_id, *((nd_uuid_t *)sqlite3_column_blob(res, 0))); dimension_data.id = (char *) sqlite3_column_text(res, 1); dimension_data.name = (char *) sqlite3_column_text(res, 2); dimension_data.hidden = sqlite3_column_int(res, 3); @@ -146,7 +146,7 @@ void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DA // LABEL LIST #define CTX_GET_LABEL_LIST "SELECT l.label_key, l.label_value, l.source_type FROM meta.chart_label l WHERE l.chart_id = @id" -void ctx_get_label_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data) +void ctx_get_label_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data) { static __thread sqlite3_stmt *res = NULL; @@ -175,7 +175,7 @@ void ctx_get_label_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, v #define CTX_GET_CONTEXT_LIST "SELECT id, version, title, chart_type, unit, priority, first_time_t, " \ "last_time_t, deleted, family FROM context c WHERE c.host_id = @host_id" -void ctx_get_context_list(uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_DATA *, void *), void *data) +void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_DATA *, void *), void *data) { if (unlikely(!host_uuid)) @@ -220,7 +220,7 @@ void ctx_get_context_list(uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_D "(host_id, id, version, title, chart_type, unit, priority, first_time_t, last_time_t, deleted, family) " \ "VALUES (@host_id, @context, @version, @title, @chart_type, @unit, @priority, @first_t, @last_t, @delete, @family)" -int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) +int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) { int rc_stored = 1; sqlite3_stmt *res = NULL; @@ -259,7 +259,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) // Delete a context #define CTX_DELETE_CONTEXT "DELETE FROM context WHERE host_id = @host_id AND id = @context" -int ctx_delete_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) +int ctx_delete_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) { int rc_stored = 1; sqlite3_stmt *res = NULL; @@ -293,9 +293,7 @@ int sql_context_cache_stats(int op) if (unlikely(!db_context_meta)) return 0; - netdata_thread_disable_cancelability(); sqlite3_db_status(db_context_meta, op, &count, &dummy, 0); - netdata_thread_enable_cancelability(); return count; } @@ -336,7 +334,7 @@ static void dict_ctx_get_context_list_cb(VERSIONED_CONTEXT_DATA *context_data, v int ctx_unittest(void) { - uuid_t host_uuid; + nd_uuid_t host_uuid; uuid_generate(host_uuid); if (sqlite_library_init()) diff --git a/src/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h index ab8271e95a6d27..ca73767305896d 100644 --- a/src/database/sqlite/sqlite_context.h +++ b/src/database/sqlite/sqlite_context.h @@ -8,7 +8,7 @@ int sql_context_cache_stats(int op); typedef struct ctx_chart { - uuid_t chart_id; + nd_uuid_t chart_id; const char *id; const char *name; const char *context; @@ -21,7 +21,7 @@ typedef struct ctx_chart { } SQL_CHART_DATA; typedef struct ctx_dimension { - uuid_t dim_id; + nd_uuid_t dim_id; char *id; char *name; bool hidden; @@ -52,17 +52,17 @@ typedef struct versioned_context_data { } VERSIONED_CONTEXT_DATA; -void ctx_get_context_list(uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_DATA *, void *), void *data); +void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_DATA *, void *), void *data); -void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data); -void ctx_get_label_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data); -void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data); +void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data); +void ctx_get_label_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data); +void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data); -int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data); +int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data); #define ctx_update_context(host_uuid, context_data) ctx_store_context(host_uuid, context_data) -int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data); +int ctx_delete_context(nd_uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data); int sql_init_context_database(int memory); uint64_t sqlite_get_context_space(void); diff --git a/src/database/sqlite/sqlite_health.c b/src/database/sqlite/sqlite_health.c index 47b641b6279a9c..51e38d05aa8945 100644 --- a/src/database/sqlite/sqlite_health.c +++ b/src/database/sqlite/sqlite_health.c @@ -249,7 +249,7 @@ void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed) { #define SQL_INJECT_REMOVED_UPDATE_LOG \ "UPDATE health_log SET last_transition_id = ?1 WHERE alarm_id = ?2 AND last_transition_id = ?3 AND host_id = ?4" -bool sql_update_removed_in_health_log(RRDHOST *host, uint32_t alarm_id, uuid_t *transition_id, uuid_t *last_transition) +bool sql_update_removed_in_health_log(RRDHOST *host, uint32_t alarm_id, nd_uuid_t *transition_id, nd_uuid_t *last_transition) { int rc = 0; sqlite3_stmt *res; @@ -275,7 +275,7 @@ bool sql_update_removed_in_health_log(RRDHOST *host, uint32_t alarm_id, uuid_t * return (param == 0 && rc == SQLITE_DONE); } -bool sql_update_removed_in_health_log_detail(uint32_t unique_id, uint32_t max_unique_id, uuid_t *prev_transition_id) +bool sql_update_removed_in_health_log_detail(uint32_t unique_id, uint32_t max_unique_id, nd_uuid_t *prev_transition_id) { int rc = 0; sqlite3_stmt *res; @@ -307,7 +307,7 @@ void sql_inject_removed_status( uint32_t alarm_event_id, uint32_t unique_id, uint32_t max_unique_id, - uuid_t *last_transition) + nd_uuid_t *last_transition) { if (!alarm_id || !alarm_event_id || !unique_id || !max_unique_id) return; @@ -317,7 +317,7 @@ void sql_inject_removed_status( if (!PREPARE_STATEMENT(db_meta, SQL_INJECT_REMOVED, &res)) return; - uuid_t transition_id; + nd_uuid_t transition_id; uuid_generate_random(transition_id); int param = 0; @@ -379,7 +379,7 @@ void sql_check_removed_alerts_state(RRDHOST *host) { uint32_t max_unique_id = 0; sqlite3_stmt *res = NULL; - uuid_t transition_id; + nd_uuid_t transition_id; if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_LAST_STATUSES, &res)) return; @@ -396,7 +396,7 @@ void sql_check_removed_alerts_state(RRDHOST *host) unique_id = (uint32_t)sqlite3_column_int64(res, 1); alarm_id = (uint32_t)sqlite3_column_int64(res, 2); alarm_event_id = (uint32_t)sqlite3_column_int64(res, 3); - uuid_copy(transition_id, *((uuid_t *)sqlite3_column_blob(res, 4))); + uuid_copy(transition_id, *((nd_uuid_t *)sqlite3_column_blob(res, 4))); if (unlikely(status != RRDCALC_STATUS_REMOVED)) { if (unlikely(!max_unique_id)) @@ -414,12 +414,12 @@ void sql_check_removed_alerts_state(RRDHOST *host) "DELETE FROM health_log WHERE host_id = @host_id AND chart NOT IN " \ "(SELECT type||'.'||id FROM chart WHERE host_id = @host_id)" -static void sql_remove_alerts_from_deleted_charts(RRDHOST *host, uuid_t *host_uuid) +static void sql_remove_alerts_from_deleted_charts(RRDHOST *host, nd_uuid_t *host_uuid) { sqlite3_stmt *res = NULL; int ret; - uuid_t *actual_uuid = host ? &host->host_uuid : host_uuid; + nd_uuid_t *actual_uuid = host ? &host->host_uuid : host_uuid; if (!actual_uuid) return; @@ -446,10 +446,10 @@ static int clean_host_alerts(void *data, int argc, char **argv, char **column) UNUSED(column); char guid[UUID_STR_LEN]; - uuid_unparse_lower(*(uuid_t *)argv[0], guid); + uuid_unparse_lower(*(nd_uuid_t *)argv[0], guid); netdata_log_info("Checking host %s (%s)", guid, (const char *) argv[1]); - sql_remove_alerts_from_deleted_charts(NULL, (uuid_t *)argv[0]); + sql_remove_alerts_from_deleted_charts(NULL, (nd_uuid_t *)argv[0]); return 0; } @@ -568,7 +568,7 @@ void sql_health_alarm_log_load(RRDHOST *host) ae->alarm_id = alarm_id; if (sqlite3_column_type(res, 3) != SQLITE_NULL) - uuid_copy(ae->config_hash_id, *((uuid_t *) sqlite3_column_blob(res, 3))); + uuid_copy(ae->config_hash_id, *((nd_uuid_t *) sqlite3_column_blob(res, 3))); ae->alarm_event_id = (uint32_t) sqlite3_column_int64(res, 2); ae->updated_by_id = (uint32_t) sqlite3_column_int64(res, 4); @@ -609,7 +609,7 @@ void sql_health_alarm_log_load(RRDHOST *host) ae->chart_context = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 29); if (sqlite3_column_type(res, 30) != SQLITE_NULL) - uuid_copy(ae->transition_id, *((uuid_t *)sqlite3_column_blob(res, 30))); + uuid_copy(ae->transition_id, *((nd_uuid_t *)sqlite3_column_blob(res, 30))); if (sqlite3_column_type(res, 31) != SQLITE_NULL) ae->global_id = sqlite3_column_int64(res, 31); @@ -894,11 +894,11 @@ void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, time_t after, const ch char new_value_string[100 + 1]; char config_hash_id[UUID_STR_LEN]; - uuid_unparse_lower(*((uuid_t *)sqlite3_column_blob(stmt_query, 3)), config_hash_id); + uuid_unparse_lower(*((nd_uuid_t *)sqlite3_column_blob(stmt_query, 3)), config_hash_id); char transition_id[UUID_STR_LEN] = {0}; if (sqlite3_column_type(stmt_query, 30) != SQLITE_NULL) - uuid_unparse_lower(*((uuid_t *)sqlite3_column_blob(stmt_query, 30)), transition_id); + uuid_unparse_lower(*((nd_uuid_t *)sqlite3_column_blob(stmt_query, 30)), transition_id); char *edit_command = sqlite3_column_bytes(stmt_query, 16) > 0 ? health_edit_command_from_source((char *)sqlite3_column_text(stmt_query, 16)) : @@ -988,7 +988,7 @@ int health_migrate_old_health_log_table(char *table) { } char *uuid_from_table = strdupz(table + 11); - uuid_t uuid; + nd_uuid_t uuid; if (uuid_parse_fix(uuid_from_table, uuid)) { freez(uuid_from_table); return 0; @@ -1175,7 +1175,7 @@ bool sql_find_alert_transition( char machine_guid[UUID_STR_LEN]; - uuid_t transition_uuid; + nd_uuid_t transition_uuid; if (uuid_parse(transition, transition_uuid)) return false; @@ -1190,7 +1190,7 @@ bool sql_find_alert_transition( param = 0; while (sqlite3_step_monitored(res) == SQLITE_ROW) { ok = true; - uuid_unparse_lower(*(uuid_t *) sqlite3_column_blob(res, 1), machine_guid); + uuid_unparse_lower(*(nd_uuid_t *) sqlite3_column_blob(res, 1), machine_guid); cb(machine_guid, (const char *) sqlite3_column_text(res, 2), sqlite3_column_int(res, 0), data); } @@ -1234,7 +1234,7 @@ void sql_alert_transitions( void *data, bool debug __maybe_unused) { - uuid_t transition_uuid; + nd_uuid_t transition_uuid; char sql[512]; int rc; sqlite3_stmt *res = NULL; @@ -1273,7 +1273,7 @@ void sql_alert_transitions( void *t; dfe_start_read(nodes, t) { - uuid_t host_uuid; + nd_uuid_t host_uuid; uuid_parse( t_dfe.name, host_uuid); rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC); @@ -1323,9 +1323,9 @@ run_query:; param = 0; while (sqlite3_step(res) == SQLITE_ROW) { - atd.host_id = (uuid_t *) sqlite3_column_blob(res, 0); + atd.host_id = (nd_uuid_t *) sqlite3_column_blob(res, 0); atd.alarm_id = sqlite3_column_int64(res, 1); - atd.config_hash_id = (uuid_t *)sqlite3_column_blob(res, 2); + atd.config_hash_id = (nd_uuid_t *)sqlite3_column_blob(res, 2); atd.alert_name = (const char *) sqlite3_column_text(res, 3); atd.chart = (const char *) sqlite3_column_text(res, 4); atd.chart_name = (const char *) sqlite3_column_text(res, 5); @@ -1347,7 +1347,7 @@ run_query:; atd.new_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 21); atd.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 22); atd.last_repeat = sqlite3_column_int64(res, 23); - atd.transition_id = (uuid_t *) sqlite3_column_blob(res, 24); + atd.transition_id = (nd_uuid_t *) sqlite3_column_blob(res, 24); atd.global_id = sqlite3_column_int64(res, 25); atd.classification = (const char *) sqlite3_column_text(res, 26); atd.type = (const char *) sqlite3_column_text(res, 27); @@ -1413,7 +1413,7 @@ int sql_get_alert_configuration( void *t; dfe_start_read(configs, t) { - uuid_t hash_id; + nd_uuid_t hash_id; uuid_parse( t_dfe.name, hash_id); rc = sqlite3_bind_blob(res, 1, &hash_id, sizeof(hash_id), SQLITE_STATIC); @@ -1446,7 +1446,7 @@ int sql_get_alert_configuration( int param; while (sqlite3_step(res) == SQLITE_ROW) { param = 0; - acd.config_hash_id = (uuid_t *) sqlite3_column_blob(res, param++); + acd.config_hash_id = (nd_uuid_t *) sqlite3_column_blob(res, param++); acd.name = (const char *) sqlite3_column_text(res, param++); acd.selectors.on_template = (const char *) sqlite3_column_text(res, param++); acd.selectors.on_key = (const char *) sqlite3_column_text(res, param++); diff --git a/src/database/sqlite/sqlite_metadata.c b/src/database/sqlite/sqlite_metadata.c index 9213106b6370aa..e0c4c43c1ddcd9 100644 --- a/src/database/sqlite/sqlite_metadata.c +++ b/src/database/sqlite/sqlite_metadata.c @@ -224,13 +224,11 @@ int sql_metadata_cache_stats(int op) if (!REQUIRE_DB(db_meta)) return 0; - netdata_thread_disable_cancelability(); sqlite3_db_status(db_meta, op, &count, &dummy, 0); - netdata_thread_enable_cancelability(); return count; } -static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) +static inline void set_host_node_id(RRDHOST *host, nd_uuid_t *node_id) { if (unlikely(!host)) return; @@ -244,7 +242,7 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) struct aclk_sync_cfg_t *wc = host->aclk_config; if (unlikely(!host->node_id)) { - uuid_t *t = mallocz(sizeof(*host->node_id)); + nd_uuid_t *t = mallocz(sizeof(*host->node_id)); uuid_copy(*t, *node_id); __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED); } @@ -260,7 +258,7 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) #define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id" -int update_node_id(uuid_t *host_id, uuid_t *node_id) +int update_node_id(nd_uuid_t *host_id, nd_uuid_t *node_id) { sqlite3_stmt *res = NULL; RRDHOST *host = NULL; @@ -272,7 +270,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id) host = rrdhost_find_by_guid(host_guid); if (likely(host)) set_host_node_id(host, node_id); - rrd_unlock(); + rrd_wrunlock(); if (!REQUIRE_DB(db_meta)) return 1; @@ -298,7 +296,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id) #define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL" -int get_node_id(uuid_t *host_id, uuid_t *node_id) +int get_node_id(nd_uuid_t *host_id, nd_uuid_t *node_id) { sqlite3_stmt *res = NULL; @@ -314,7 +312,7 @@ int get_node_id(uuid_t *host_id, uuid_t *node_id) param = 0; rc = sqlite3_step_monitored(res); if (likely(rc == SQLITE_ROW && node_id)) - uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0))); + uuid_copy(*node_id, *((nd_uuid_t *) sqlite3_column_blob(res, 0))); done: REPORT_BIND_FAIL(res, param); @@ -326,7 +324,7 @@ int get_node_id(uuid_t *host_id, uuid_t *node_id) "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \ "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))" -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id) +void invalidate_node_instances(nd_uuid_t *host_id, nd_uuid_t *claim_id) { sqlite3_stmt *res = NULL; @@ -384,10 +382,10 @@ struct node_instance_list *get_node_list(void) // TODO: Check to remove lock rrd_rdlock(); while (sqlite3_step_monitored(res) == SQLITE_ROW) { - if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t)) - uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0))); - if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) { - uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1); + if (sqlite3_column_bytes(res, 0) == sizeof(nd_uuid_t)) + uuid_copy(node_list[row].node_id, *((nd_uuid_t *)sqlite3_column_blob(res, 0))); + if (sqlite3_column_bytes(res, 1) == sizeof(nd_uuid_t)) { + nd_uuid_t *host_id = (nd_uuid_t *)sqlite3_column_blob(res, 1); uuid_unparse_lower(*host_id, host_guid); RRDHOST *host = rrdhost_find_by_guid(host_guid); if (!host) @@ -404,7 +402,7 @@ struct node_instance_list *get_node_list(void) node_list[row].live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0; node_list[row].hops = host->system_info ? host->system_info->hops : - uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0; + uuid_eq(*host_id, localhost->host_uuid) ? 0 : 1; node_list[row].hostname = sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL; } @@ -412,7 +410,7 @@ struct node_instance_list *get_node_list(void) if (row == max_rows) break; } - rrd_unlock(); + rrd_rdunlock(); failed: SQLITE_FINALIZE(res); @@ -438,8 +436,8 @@ void sql_load_node_id(RRDHOST *host) param = 0; int rc = sqlite3_step_monitored(res); if (likely(rc == SQLITE_ROW)) { - if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t))) - set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0)); + if (likely(sqlite3_column_bytes(res, 0) == sizeof(nd_uuid_t))) + set_host_node_id(host, (nd_uuid_t *)sqlite3_column_blob(res, 0)); else set_host_node_id(host, NULL); } @@ -451,7 +449,7 @@ void sql_load_node_id(RRDHOST *host) #define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id" -void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info) +void sql_build_host_system_info(nd_uuid_t *host_id, struct rrdhost_system_info *system_info) { sqlite3_stmt *res = NULL; @@ -475,7 +473,7 @@ void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *sys #define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \ "AND label_key IS NOT NULL AND label_value IS NOT NULL" -RRDLABELS *sql_load_host_labels(uuid_t *host_id) +RRDLABELS *sql_load_host_labels(nd_uuid_t *host_id) { RRDLABELS *labels = NULL; sqlite3_stmt *res = NULL; @@ -503,7 +501,7 @@ RRDLABELS *sql_load_host_labels(uuid_t *host_id) return labels; } -static int exec_statement_with_uuid(const char *sql, uuid_t *uuid) +static int exec_statement_with_uuid(const char *sql, nd_uuid_t *uuid) { int result = 1; sqlite3_stmt *res = NULL; @@ -573,7 +571,7 @@ static void recover_database(const char *sqlite_database, const char *new_sqlite static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv) { - uuid_t uuid; + nd_uuid_t uuid; if ( argc != 1 ){ sqlite3_result_null(context); @@ -585,7 +583,7 @@ static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value return ; } - sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); + sqlite3_result_blob(context, &uuid, sizeof(nd_uuid_t), SQLITE_TRANSIENT); } void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv) @@ -608,9 +606,9 @@ void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv (void)argc; (void)argv; - uuid_t uuid; + nd_uuid_t uuid; uuid_generate_random(uuid); - sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); + sqlite3_result_blob(context, &uuid, sizeof(nd_uuid_t), SQLITE_TRANSIENT); } // Init @@ -726,7 +724,7 @@ struct query_build { #define SQL_DELETE_CHART_LABELS_BY_HOST \ "DELETE FROM chart_label WHERE chart_id in (SELECT chart_id FROM chart WHERE host_id = @host_id)" -static void delete_host_chart_labels(uuid_t *host_uuid) +static void delete_host_chart_labels(nd_uuid_t *host_uuid) { sqlite3_stmt *res = NULL; @@ -809,7 +807,7 @@ static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t } // If the machine guid has changed, then existing one with hops 0 will be marked as hops 1 (child) -void detect_machine_guid_change(uuid_t *host_uuid) +void detect_machine_guid_change(nd_uuid_t *host_uuid) { int rc; @@ -820,7 +818,7 @@ void detect_machine_guid_change(uuid_t *host_uuid) } } -static int store_claim_id(uuid_t *host_id, uuid_t *claim_id) +static int store_claim_id(nd_uuid_t *host_id, nd_uuid_t *claim_id) { sqlite3_stmt *res = NULL; int rc = 0; @@ -850,7 +848,7 @@ static int store_claim_id(uuid_t *host_id, uuid_t *claim_id) return rc != SQLITE_DONE; } -static void delete_dimension_uuid(uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused) +static void delete_dimension_uuid(nd_uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused) { static __thread sqlite3_stmt *res = NULL; int rc; @@ -913,7 +911,7 @@ static int store_host_metadata(RRDHOST *host) return 1; } -static int add_host_sysinfo_key_value(const char *name, const char *value, uuid_t *uuid) +static int add_host_sysinfo_key_value(const char *name, const char *value, nd_uuid_t *uuid) { static __thread sqlite3_stmt *res = NULL; @@ -1065,7 +1063,7 @@ static int store_dimension_metadata(RRDDIM *rd) return 1; } -static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused) +static bool dimension_can_be_deleted(nd_uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused) { #ifdef ENABLE_DBENGINE if(dbengine_enabled) { @@ -1093,8 +1091,8 @@ static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_st static bool run_cleanup_loop( sqlite3_stmt *res, struct metadata_wc *wc, - bool (*check_cb)(uuid_t *, sqlite3_stmt **, bool), - void (*action_cb)(uuid_t *, sqlite3_stmt **, bool), + bool (*check_cb)(nd_uuid_t *, sqlite3_stmt **, bool), + void (*action_cb)(nd_uuid_t *, sqlite3_stmt **, bool), uint32_t *total_checked, uint32_t *total_deleted, uint64_t *row_id, @@ -1118,10 +1116,10 @@ static bool run_cleanup_loop( break; *row_id = sqlite3_column_int64(res, 1); - rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag); + rc = check_cb((nd_uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag); if (rc == true) { - action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag); + action_cb((nd_uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag); (*total_deleted)++; } @@ -1135,7 +1133,7 @@ static bool run_cleanup_loop( #define SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION "SELECT count(1) FROM dimension WHERE chart_id = @chart_id" #define SQL_CHECK_CHART_EXISTENCE_IN_CHART "SELECT count(1) FROM chart WHERE chart_id = @chart_id" -static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension) +static bool chart_can_be_deleted(nd_uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension) { int rc, result = 1; sqlite3_stmt *res = check_res ? *check_res : NULL; @@ -1173,7 +1171,7 @@ static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, b #define SQL_DELETE_CHART_BY_UUID "DELETE FROM chart WHERE chart_id = @chart_id" #define SQL_DELETE_CHART_LABEL_BY_UUID "DELETE FROM chart_label WHERE chart_id = @chart_id" -static void delete_chart_uuid(uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only) +static void delete_chart_uuid(nd_uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only) { int rc; sqlite3_stmt *res = action_res ? *action_res : NULL; @@ -1812,7 +1810,7 @@ static void do_chart_label_cleanup(struct host_chart_label_cleanup *cl_cleanup_d RRDHOST *host = rrdhost_find_by_guid(machine_guid); if (likely(!host)) { - uuid_t host_uuid; + nd_uuid_t host_uuid; if (!uuid_parse(machine_guid, host_uuid)) delete_host_chart_labels(&host_uuid); } @@ -1883,7 +1881,7 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused) if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_CLAIMID))) { rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_CLAIMID); - uuid_t uuid; + nd_uuid_t uuid; int rc; if (likely(host->aclk_state.claimed_id && !uuid_parse(host->aclk_state.claimed_id, uuid))) rc = store_claim_id(&host->host_uuid, &uuid); @@ -1952,7 +1950,7 @@ static void metadata_event_loop(void *arg) struct metadata_wc *wc = arg; enum metadata_opcode opcode; - uv_thread_set_name_np(wc->thread, "METASYNC"); + uv_thread_set_name_np("METASYNC"); loop = wc->loop = mallocz(sizeof(uv_loop_t)); ret = uv_loop_init(loop); if (ret) { @@ -1991,7 +1989,7 @@ static void metadata_event_loop(void *arg) struct host_chart_label_cleanup *cl_cleanup_data = NULL; while (shutdown == 0 || (wc->flags & METADATA_FLAG_PROCESSING)) { - uuid_t *uuid; + nd_uuid_t *uuid; RRDHOST *host = NULL; worker_is_idle(); @@ -2021,13 +2019,13 @@ static void metadata_event_loop(void *arg) case METADATA_DATABASE_TIMER: break; case METADATA_DEL_DIMENSION: - uuid = (uuid_t *) cmd.param[0]; + uuid = (nd_uuid_t *) cmd.param[0]; if (likely(dimension_can_be_deleted(uuid, NULL, false))) delete_dimension_uuid(uuid, NULL, false); freez(uuid); break; case METADATA_STORE_CLAIM_ID: - store_claim_id((uuid_t *) cmd.param[0], (uuid_t *) cmd.param[1]); + store_claim_id((nd_uuid_t *) cmd.param[0], (nd_uuid_t *) cmd.param[1]); freez((void *) cmd.param[0]); freez((void *) cmd.param[1]); break; @@ -2217,22 +2215,22 @@ static inline void queue_metadata_cmd(enum metadata_opcode opcode, const void *p } // Public -void metaqueue_delete_dimension_uuid(uuid_t *uuid) +void metaqueue_delete_dimension_uuid(nd_uuid_t *uuid) { if (unlikely(!metasync_worker.loop)) return; - uuid_t *use_uuid = mallocz(sizeof(*uuid)); + nd_uuid_t *use_uuid = mallocz(sizeof(*uuid)); uuid_copy(*use_uuid, *uuid); queue_metadata_cmd(METADATA_DEL_DIMENSION, use_uuid, NULL); } -void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid) +void metaqueue_store_claim_id(nd_uuid_t *host_uuid, nd_uuid_t *claim_uuid) { if (unlikely(!host_uuid)) return; - uuid_t *local_host_uuid = mallocz(sizeof(*host_uuid)); - uuid_t *local_claim_uuid = NULL; + nd_uuid_t *local_host_uuid = mallocz(sizeof(*host_uuid)); + nd_uuid_t *local_claim_uuid = NULL; uuid_copy(*local_host_uuid, *host_uuid); if (likely(claim_uuid)) { @@ -2318,13 +2316,12 @@ static void *metadata_unittest_threads(void) threads_to_create, (long long)seconds_to_run); - netdata_thread_t threads[threads_to_create]; + ND_THREAD *threads[threads_to_create]; tu.join = 0; for (int i = 0; i < threads_to_create; i++) { char buf[100 + 1]; snprintf(buf, sizeof(buf) - 1, "META[%d]", i); - netdata_thread_create( - &threads[i], + threads[i] = nd_thread_create( buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, unittest_queue_metadata, @@ -2335,8 +2332,7 @@ static void *metadata_unittest_threads(void) __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED); for (int i = 0; i < threads_to_create; i++) { - void *retval; - netdata_thread_join(threads[i], &retval); + nd_thread_join(threads[i]); } sleep_usec(5 * USEC_PER_SEC); diff --git a/src/database/sqlite/sqlite_metadata.h b/src/database/sqlite/sqlite_metadata.h index 559121b017da09..3b1fd96610fbd8 100644 --- a/src/database/sqlite/sqlite_metadata.h +++ b/src/database/sqlite/sqlite_metadata.h @@ -8,8 +8,8 @@ // return a node list struct node_instance_list { - uuid_t node_id; - uuid_t host_id; + nd_uuid_t node_id; + nd_uuid_t host_id; char *hostname; int live; int queryable; @@ -29,26 +29,26 @@ void metadata_sync_init(void); void metadata_sync_shutdown(void); void metadata_sync_shutdown_prepare(void); -void metaqueue_delete_dimension_uuid(uuid_t *uuid); -void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid); +void metaqueue_delete_dimension_uuid(nd_uuid_t *uuid); +void metaqueue_store_claim_id(nd_uuid_t *host_uuid, nd_uuid_t *claim_uuid); void metaqueue_host_update_info(RRDHOST *host); void metaqueue_ml_load_models(RRDDIM *rd); -void detect_machine_guid_change(uuid_t *host_uuid); +void detect_machine_guid_change(nd_uuid_t *host_uuid); void metadata_queue_load_host_context(RRDHOST *host); void metadata_delete_host_chart_labels(char *machine_guid); void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc); int sql_metadata_cache_stats(int op); -int get_node_id(uuid_t *host_id, uuid_t *node_id); -int update_node_id(uuid_t *host_id, uuid_t *node_id); +int get_node_id(nd_uuid_t *host_id, nd_uuid_t *node_id); +int update_node_id(nd_uuid_t *host_id, nd_uuid_t *node_id); struct node_instance_list *get_node_list(void); void sql_load_node_id(RRDHOST *host); // Help build archived hosts in memory when agent starts -void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info); -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id); -RRDLABELS *sql_load_host_labels(uuid_t *host_id); +void sql_build_host_system_info(nd_uuid_t *host_id, struct rrdhost_system_info *system_info); +void invalidate_node_instances(nd_uuid_t *host_id, nd_uuid_t *claim_id); +RRDLABELS *sql_load_host_labels(nd_uuid_t *host_id); uint64_t sqlite_get_meta_space(void); int sql_init_meta_database(db_check_action_type_t rebuild, int memory); diff --git a/src/exporting/aws_kinesis/aws_kinesis.c b/src/exporting/aws_kinesis/aws_kinesis.c index 498d9ee237ba41..27f2fb0ba83a09 100644 --- a/src/exporting/aws_kinesis/aws_kinesis.c +++ b/src/exporting/aws_kinesis/aws_kinesis.c @@ -100,6 +100,10 @@ void aws_kinesis_connector_worker(void *instance_p) struct aws_kinesis_specific_config *connector_specific_config = instance->config.connector_specific_config; struct aws_kinesis_specific_data *connector_specific_data = instance->connector_specific_data; + char threadname[ND_THREAD_TAG_MAX + 1]; + snprintfz(threadname, ND_THREAD_TAG_MAX, "EXPKNSS[%zu]", instance->index); + uv_thread_set_name_np(threadname); + while (!instance->engine->exit) { unsigned long long partition_key_seq = 0; struct stats *stats = &instance->stats; diff --git a/src/exporting/exporting_engine.c b/src/exporting/exporting_engine.c index 739c14baf6258e..eb5f8a0a8bdf49 100644 --- a/src/exporting/exporting_engine.c +++ b/src/exporting/exporting_engine.c @@ -119,9 +119,11 @@ static void exporting_clean_engine() * * @param ptr thread data. */ -static void exporting_main_cleanup(void *ptr) +static void exporting_main_cleanup(void *pptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_info("cleaning up..."); @@ -174,7 +176,7 @@ static void exporting_main_cleanup(void *ptr) */ void *exporting_main(void *ptr) { - netdata_thread_cleanup_push(exporting_main_cleanup, ptr); + CLEANUP_FUNCTION_REGISTER(exporting_main_cleanup) cleanup_ptr = ptr; engine = read_exporting_config(); if (!engine) { @@ -214,6 +216,5 @@ void *exporting_main(void *ptr) } cleanup: - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/exporting/graphite/graphite.c b/src/exporting/graphite/graphite.c index 9edde4503a47ce..1fc1f2b0410d68 100644 --- a/src/exporting/graphite/graphite.c +++ b/src/exporting/graphite/graphite.c @@ -74,7 +74,7 @@ int init_graphite_instance(struct instance *instance) void sanitize_graphite_label_value(char *dst, const char *src, size_t len) { while (*src != '\0' && len) { - if (isspace(*src) || *src == ';' || *src == '~') + if (isspace((uint8_t)*src) || *src == ';' || *src == '~') *dst++ = '_'; else *dst++ = *src; diff --git a/src/exporting/init_connectors.c b/src/exporting/init_connectors.c index 165046f673cbe6..3481ded4fa58bd 100644 --- a/src/exporting/init_connectors.c +++ b/src/exporting/init_connectors.c @@ -95,9 +95,6 @@ int init_connectors(struct engine *engine) netdata_log_error("EXPORTING: cannot create thread worker. uv_thread_create(): %s", uv_strerror(error)); return 1; } - char threadname[NETDATA_THREAD_NAME_MAX + 1]; - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "EXPORTING-%zu", instance->index); - uv_thread_set_name_np(instance->thread, threadname); analytics_statistic_t statistic = { "EXPORTING_START", "OK", instance->config.type_name }; analytics_statistic_send(&statistic); diff --git a/src/exporting/mongodb/mongodb.c b/src/exporting/mongodb/mongodb.c index c65f8d4cc58ee8..1a278dcbd8e940 100644 --- a/src/exporting/mongodb/mongodb.c +++ b/src/exporting/mongodb/mongodb.c @@ -285,6 +285,10 @@ void mongodb_connector_worker(void *instance_p) struct mongodb_specific_data *connector_specific_data = (struct mongodb_specific_data *)instance->connector_specific_data; + char threadname[ND_THREAD_TAG_MAX + 1]; + snprintfz(threadname, ND_THREAD_TAG_MAX, "EXPMNG[%zu]", instance->index); + uv_thread_set_name_np(threadname); + while (!instance->engine->exit) { struct stats *stats = &instance->stats; diff --git a/src/exporting/opentsdb/opentsdb.c b/src/exporting/opentsdb/opentsdb.c index 41f8e200a0604c..ab4495cb2428db 100644 --- a/src/exporting/opentsdb/opentsdb.c +++ b/src/exporting/opentsdb/opentsdb.c @@ -127,7 +127,7 @@ int init_opentsdb_http_instance(struct instance *instance) void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len) { while (*src != '\0' && len) { - if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src)) + if (isalpha((uint8_t)*src) || isdigit((uint8_t)*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src)) *dst++ = *src; else *dst++ = '_'; diff --git a/src/exporting/process_data.c b/src/exporting/process_data.c index 4adf4b5872a34c..1c7eaa192a95e2 100644 --- a/src/exporting/process_data.c +++ b/src/exporting/process_data.c @@ -336,7 +336,6 @@ void end_batch_formatting(struct engine *engine) */ void prepare_buffers(struct engine *engine) { - netdata_thread_disable_cancelability(); start_batch_formatting(engine); rrd_rdlock(); @@ -358,8 +357,7 @@ void prepare_buffers(struct engine *engine) variables_formatting(engine, host); end_host_formatting(engine, host); } - rrd_unlock(); - netdata_thread_enable_cancelability(); + rrd_rdunlock(); end_batch_formatting(engine); } diff --git a/src/exporting/prometheus/prometheus.c b/src/exporting/prometheus/prometheus.c index bd8cab7a283678..5fadb84ab70d5e 100644 --- a/src/exporting/prometheus/prometheus.c +++ b/src/exporting/prometheus/prometheus.c @@ -552,8 +552,8 @@ static void prometheus_print_os_info( if (!in_val_part) { /* Only accepts alphabetic characters and '_' * in key part */ - if (isalpha(*in) || *in == '_') { - *(sanitized++) = tolower(*in); + if (isalpha((uint8_t)*in) || *in == '_') { + *(sanitized++) = tolower((uint8_t)*in); } else if (*in == '=') { in_val_part = 1; *(sanitized++) = '='; @@ -568,7 +568,7 @@ static void prometheus_print_os_info( case '\t': break; default: - if (isprint(*in)) { + if (isprint((uint8_t)*in)) { *(sanitized++) = *in; } } diff --git a/src/exporting/pubsub/pubsub.c b/src/exporting/pubsub/pubsub.c index 4989160a41d1cf..7fc4162584955a 100644 --- a/src/exporting/pubsub/pubsub.c +++ b/src/exporting/pubsub/pubsub.c @@ -99,6 +99,10 @@ void pubsub_connector_worker(void *instance_p) struct pubsub_specific_config *connector_specific_config = instance->config.connector_specific_config; struct pubsub_specific_data *connector_specific_data = instance->connector_specific_data; + char threadname[ND_THREAD_TAG_MAX + 1]; + snprintfz(threadname, ND_THREAD_TAG_MAX, "EXPPBSB[%zu]", instance->index); + uv_thread_set_name_np(threadname); + while (!instance->engine->exit) { struct stats *stats = &instance->stats; char error_message[ERROR_LINE_MAX + 1] = ""; diff --git a/src/exporting/send_data.c b/src/exporting/send_data.c index 187a6828ab1e5d..b79f0a3e309c98 100644 --- a/src/exporting/send_data.c +++ b/src/exporting/send_data.c @@ -217,6 +217,10 @@ void simple_connector_worker(void *instance_p) struct instance *instance = (struct instance*)instance_p; struct simple_connector_data *connector_specific_data = instance->connector_specific_data; + char threadname[ND_THREAD_TAG_MAX + 1]; + snprintfz(threadname, ND_THREAD_TAG_MAX, "EXPSMPL[%zu]", instance->index); + uv_thread_set_name_np(threadname); + #ifdef ENABLE_HTTPS uint32_t options = (uint32_t)instance->config.options; diff --git a/src/health/health.h b/src/health/health.h index 8aca6dcb2b7452..b1ac5a9e15b039 100644 --- a/src/health/health.h +++ b/src/health/health.h @@ -89,7 +89,7 @@ void health_string2json(BUFFER *wb, const char *prefix, const char *label, const void health_log_alert_transition_with_trace(RRDHOST *host, ALARM_ENTRY *ae, int line, const char *file, const char *function); #define health_log_alert(host, ae) health_log_alert_transition_with_trace(host, ae, __LINE__, __FILE__, __FUNCTION__) -bool health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(RRDCALC *rc, usec_t *global_id, uuid_t *transitions_id); +bool health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(RRDCALC *rc, usec_t *global_id, nd_uuid_t *transitions_id); int alert_variable_lookup_trace(RRDHOST *host, RRDSET *st, const char *variable, BUFFER *wb); diff --git a/src/health/health_config.c b/src/health/health_config.c index d8c735c3f6c107..05ced8bddc8457 100644 --- a/src/health/health_config.c +++ b/src/health/health_config.c @@ -19,14 +19,14 @@ static inline int health_parse_delay( while(*s) { char *key = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!*key) break; char *value = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!strcasecmp(key, "up")) { if (!config_parse_duration(value, delay_up_duration)) { @@ -91,12 +91,12 @@ static inline ALERT_ACTION_OPTIONS health_parse_options(const char *s) { buf[0] = '\0'; // skip spaces - while(*s && isspace(*s)) + while(*s && isspace((uint8_t)*s)) s++; // find the next space size_t count = 0; - while(*s && count < 100 && !isspace(*s)) + while(*s && count < 100 && !isspace((uint8_t)*s)) buf[count++] = *s++; if(buf[0]) { @@ -124,14 +124,14 @@ static inline int health_parse_repeat( while(*s) { char *key = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!*key) break; char *value = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!strcasecmp(key, "off")) { *warn_repeat_every = 0; @@ -176,8 +176,8 @@ static inline int health_parse_db_lookup(size_t line, const char *filename, char // first is the group method key = s; - while(*s && !isspace(*s) && *s != '(') s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s) && *s != '(') s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!*s) { netdata_log_error("Health configuration invalid chart calculation at line %zu of file '%s': expected group method followed by the 'after' time, but got '%s'", line, filename, key); @@ -224,12 +224,12 @@ static inline int health_parse_db_lookup(size_t line, const char *filename, char ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_LESS; } - while(*s && isspace(*s)) s++; + while(*s && isspace((uint8_t)*s)) s++; if(*s) { - if(isdigit(*s) || *s == '.') { + if(isdigit((uint8_t)*s) || *s == '.') { ac->time_group_value = str2ndd(s, &s); - while(s && *s && isspace(*s)) s++; + while(s && *s && isspace((uint8_t)*s)) s++; if(!s || *s != ')') { netdata_log_error("Health configuration at line %zu of file '%s': missing closing parenthesis after number in aggregation method on '%s'", @@ -270,8 +270,8 @@ static inline int health_parse_db_lookup(size_t line, const char *filename, char // then is the 'after' time key = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!config_parse_duration(key, &ac->after)) { netdata_log_error("Health configuration at line %zu of file '%s': invalid duration '%s' after group method", @@ -285,14 +285,14 @@ static inline int health_parse_db_lookup(size_t line, const char *filename, char // now we may have optional parameters while(*s) { key = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if(!*key) break; if(!strcasecmp(key, "at")) { char *value = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if (!config_parse_duration(value, &ac->before)) { netdata_log_error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword", @@ -301,8 +301,8 @@ static inline int health_parse_db_lookup(size_t line, const char *filename, char } else if(!strcasecmp(key, HEALTH_EVERY_KEY)) { char *value = s; - while(*s && !isspace(*s)) s++; - while(*s && isspace(*s)) *s++ = '\0'; + while(*s && !isspace((uint8_t)*s)) s++; + while(*s && isspace((uint8_t)*s)) *s++ = '\0'; if (!config_parse_duration(value, &ac->update_every)) { netdata_log_error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword", diff --git a/src/health/health_event_loop.c b/src/health/health_event_loop.c index 2a1dd721b4ef9d..756ffa1653c60e 100644 --- a/src/health/health_event_loop.c +++ b/src/health/health_event_loop.c @@ -740,16 +740,16 @@ static void health_event_loop(void) { } -static void health_main_cleanup(void *ptr) { - worker_unregister(); +static void health_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; netdata_log_info("cleaning up..."); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; - nd_log(NDLS_DAEMON, NDLP_DEBUG, - "Health thread ended."); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Health thread ended."); } void *health_main(void *ptr) { @@ -765,10 +765,7 @@ void *health_main(void *ptr) { worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init"); worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init"); - netdata_thread_cleanup_push(health_main_cleanup, ptr); - { - health_event_loop(); - } - netdata_thread_cleanup_pop(1); + CLEANUP_FUNCTION_REGISTER(health_main_cleanup) cleanup_ptr = ptr; + health_event_loop(); return NULL; } diff --git a/src/health/health_notifications.c b/src/health/health_notifications.c index c986ba29200793..79426f48c14e37 100644 --- a/src/health/health_notifications.c +++ b/src/health/health_notifications.c @@ -108,7 +108,7 @@ static bool prepare_command(BUFFER *wb, const char *classification, const char *edit_command, const char *machine_guid, - uuid_t *transition_id, + nd_uuid_t *transition_id, const char *summary, const char *context, const char *component, @@ -479,7 +479,7 @@ void health_send_notification(RRDHOST *host, ALARM_ENTRY *ae, struct health_rais health_alarm_log_save(host, ae); } -bool health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(RRDCALC *rc, usec_t *global_id, uuid_t *transitions_id) { +bool health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(RRDCALC *rc, usec_t *global_id, nd_uuid_t *transitions_id) { if(!rc->rrdset) return false; diff --git a/src/health/health_prototypes.c b/src/health/health_prototypes.c index 00bea507c4ef71..58cad410ae581f 100644 --- a/src/health/health_prototypes.c +++ b/src/health/health_prototypes.c @@ -374,7 +374,7 @@ static void health_prototype_activate_match_patterns(struct rrd_alert_match *am) void health_prototype_hash_id(RRD_ALERT_PROTOTYPE *ap) { CLEAN_BUFFER *wb = buffer_create(100, NULL); health_prototype_to_json(wb, ap, true); - UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb)); + ND_UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb)); uuid_copy(ap->config.hash_id, uuid.uuid); sql_alert_store_config(ap); diff --git a/src/health/health_prototypes.h b/src/health/health_prototypes.h index cbb5dba066a9cf..e226c1929a9af5 100644 --- a/src/health/health_prototypes.h +++ b/src/health/health_prototypes.h @@ -57,7 +57,7 @@ struct rrd_alert_match { void rrd_alert_match_cleanup(struct rrd_alert_match *am); struct rrd_alert_config { - uuid_t hash_id; + nd_uuid_t hash_id; STRING *name; // the name of this alarm diff --git a/src/health/rrdcalc.c b/src/health/rrdcalc.c index 7ee536ae20a4cf..bce709bf41892f 100644 --- a/src/health/rrdcalc.c +++ b/src/health/rrdcalc.c @@ -60,13 +60,13 @@ inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { } } -uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id) { +uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, nd_uuid_t *config_hash_id) { rw_spinlock_read_lock(&host->health_log.spinlock); // re-use old IDs, by looking them up in the alarm log ALARM_ENTRY *ae = NULL; for(ae = host->health_log.alarms; ae ;ae = ae->next) { - if(unlikely(name == ae->name && chart == ae->chart && !uuid_memcmp(&ae->config_hash_id, config_hash_id))) { + if(unlikely(name == ae->name && chart == ae->chart && uuid_eq(ae->config_hash_id, *config_hash_id))) { if(next_event_id) *next_event_id = ae->alarm_event_id + 1; break; } diff --git a/src/health/rrdcalc.h b/src/health/rrdcalc.h index f8a58aad5fdc53..3a7951a73095e3 100644 --- a/src/health/rrdcalc.h +++ b/src/health/rrdcalc.h @@ -121,7 +121,7 @@ RRDCALC *rrdcalc_acquired_to_rrdcalc(const RRDCALC_ACQUIRED *rca); const char *rrdcalc_status2string(RRDCALC_STATUS status); -uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id); +uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, nd_uuid_t *config_hash_id); static inline int rrdcalc_isrepeating(RRDCALC *rc) { if (unlikely(rc->config.warn_repeat_every > 0 || rc->config.crit_repeat_every > 0)) { diff --git a/src/health/rrdvar.c b/src/health/rrdvar.c index aff318c125de5a..4e28e62a385331 100644 --- a/src/health/rrdvar.c +++ b/src/health/rrdvar.c @@ -12,7 +12,7 @@ typedef struct rrdvar { inline int rrdvar_fix_name(char *variable) { int fixed = 0; while(*variable) { - if (!isalnum(*variable) && *variable != '.' && *variable != '_') { + if (!isalnum((uint8_t)*variable) && *variable != '.' && *variable != '_') { *variable++ = '_'; fixed++; } diff --git a/src/libnetdata/aral/aral.c b/src/libnetdata/aral/aral.c index b8ed47f04a75c4..2611b283b9188b 100644 --- a/src/libnetdata/aral/aral.c +++ b/src/libnetdata/aral/aral.c @@ -1018,14 +1018,16 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) { }; usec_t started_ut = now_monotonic_usec(); - netdata_thread_t thread_ptrs[threads]; + ND_THREAD *thread_ptrs[threads]; for(size_t i = 0; i < threads ; i++) { - char tag[NETDATA_THREAD_NAME_MAX + 1]; - snprintfz(tag, NETDATA_THREAD_NAME_MAX, "TH[%zu]", i); - netdata_thread_create(&thread_ptrs[i], tag, - NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, - aral_test_thread, &auc); + char tag[ND_THREAD_TAG_MAX + 1]; + snprintfz(tag, ND_THREAD_TAG_MAX, "TH[%zu]", i); + thread_ptrs[i] = nd_thread_create( + tag, + NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG, + aral_test_thread, + &auc); } size_t malloc_done = 0; @@ -1047,12 +1049,12 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) { // fprintf(stderr, "Cancelling the threads...\n"); // for(size_t i = 0; i < threads ; i++) { -// netdata_thread_cancel(thread_ptrs[i]); +// nd_thread_signal_cancel(thread_ptrs[i]); // } fprintf(stderr, "Waiting the threads to finish...\n"); for(size_t i = 0; i < threads ; i++) { - netdata_thread_join(thread_ptrs[i], NULL); + nd_thread_join(thread_ptrs[i]); } usec_t ended_ut = now_monotonic_usec(); diff --git a/src/libnetdata/avl/avl.c b/src/libnetdata/avl/avl.c index eef4c3116af800..e1d4064dc01473 100644 --- a/src/libnetdata/avl/avl.c +++ b/src/libnetdata/avl/avl.c @@ -334,7 +334,7 @@ static inline void avl_write_lock(avl_tree_lock *t) { static inline void avl_read_unlock(avl_tree_lock *t) { #if defined(AVL_LOCK_WITH_RWLOCK) - netdata_rwlock_unlock(&t->rwlock); + netdata_rwlock_rdunlock(&t->rwlock); #else rw_spinlock_read_unlock(&t->rwlock); #endif @@ -342,7 +342,7 @@ static inline void avl_read_unlock(avl_tree_lock *t) { static inline void avl_write_unlock(avl_tree_lock *t) { #if defined(AVL_LOCK_WITH_RWLOCK) - netdata_rwlock_unlock(&t->rwlock); + netdata_rwlock_wrunlock(&t->rwlock); #else rw_spinlock_write_unlock(&t->rwlock); #endif diff --git a/src/libnetdata/buffer/buffer.h b/src/libnetdata/buffer/buffer.h index 900907d491b644..92e14afb1885d0 100644 --- a/src/libnetdata/buffer/buffer.h +++ b/src/libnetdata/buffer/buffer.h @@ -141,6 +141,9 @@ static inline void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE typ assert(wb->json.depth <= BUFFER_JSON_MAX_DEPTH && "BUFFER JSON: max nesting reached"); #endif wb->json.depth++; +#ifdef NETDATA_INTERNAL_CHECKS + assert(wb->json.depth >= 0 && "Depth wrapped around and is negative"); +#endif wb->json.stack[wb->json.depth].count = 0; wb->json.stack[wb->json.depth].type = type; } @@ -772,7 +775,7 @@ static inline void buffer_json_member_add_quoted_string(BUFFER *wb, const char * wb->json.stack[wb->json.depth].count++; } -static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, uuid_t *value) { +static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, nd_uuid_t *value) { buffer_print_json_comma_newline_spacing(wb); buffer_print_json_key(wb, key); buffer_fast_strcat(wb, ":", 1); @@ -834,7 +837,7 @@ static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *val wb->json.stack[wb->json.depth].count++; } -static inline void buffer_json_add_array_item_uuid(BUFFER *wb, uuid_t *value) { +static inline void buffer_json_add_array_item_uuid(BUFFER *wb, nd_uuid_t *value) { if(value && !uuid_is_null(*value)) { char uuid[GUID_LEN + 1]; uuid_unparse_lower(*value, uuid); @@ -844,7 +847,7 @@ static inline void buffer_json_add_array_item_uuid(BUFFER *wb, uuid_t *value) { buffer_json_add_array_item_string(wb, NULL); } -static inline void buffer_json_add_array_item_uuid_compact(BUFFER *wb, uuid_t *value) { +static inline void buffer_json_add_array_item_uuid_compact(BUFFER *wb, nd_uuid_t *value) { if(value && !uuid_is_null(*value)) { char uuid[GUID_LEN + 1]; uuid_unparse_lower_compact(*value, uuid); diff --git a/src/libnetdata/buffered_reader/buffered_reader.h b/src/libnetdata/buffered_reader/buffered_reader.h index 4db57cd29e66c4..1ec1d762b88c86 100644 --- a/src/libnetdata/buffered_reader/buffered_reader.h +++ b/src/libnetdata/buffered_reader/buffered_reader.h @@ -26,7 +26,7 @@ typedef enum { BUFFERED_READER_READ_POLLNVAL = -5, BUFFERED_READER_READ_POLL_UNKNOWN = -6, BUFFERED_READER_READ_POLL_TIMEOUT = -7, - BUFFERED_READER_READ_POLL_FAILED = -8, + BUFFERED_READER_READ_POLL_CANCELLED = -8, } buffered_reader_ret_t; @@ -53,48 +53,47 @@ static inline buffered_reader_ret_t buffered_reader_read(struct buffered_reader } static inline buffered_reader_ret_t buffered_reader_read_timeout(struct buffered_reader *reader, int fd, int timeout_ms, bool log_error) { - errno = 0; - struct pollfd fds[1]; - - fds[0].fd = fd; - fds[0].events = POLLIN; - - int ret = poll(fds, 1, timeout_ms); + short int revents = 0; + switch(wait_on_socket_or_cancel_with_timeout( +#ifdef ENABLE_HTTPS + NULL, +#endif + fd, timeout_ms, POLLIN, &revents)) { - if (ret > 0) { - /* There is data to read */ - if (fds[0].revents & POLLIN) + case 0: // data are waiting return buffered_reader_read(reader, fd); - else if(fds[0].revents & POLLERR) { - if(log_error) - netdata_log_error("PARSER: read failed: POLLERR."); - return BUFFERED_READER_READ_POLLERR; - } - else if(fds[0].revents & POLLHUP) { + case 1: // timeout reached if(log_error) - netdata_log_error("PARSER: read failed: POLLHUP."); - return BUFFERED_READER_READ_POLLHUP; - } - else if(fds[0].revents & POLLNVAL) { - if(log_error) - netdata_log_error("PARSER: read failed: POLLNVAL."); - return BUFFERED_READER_READ_POLLNVAL; - } - - if(log_error) - netdata_log_error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set."); - return BUFFERED_READER_READ_POLL_UNKNOWN; - } - else if (ret == 0) { - if(log_error) - netdata_log_error("PARSER: timeout while waiting for data."); - return BUFFERED_READER_READ_POLL_TIMEOUT; + netdata_log_error("PARSER: timeout while waiting for data."); + return BUFFERED_READER_READ_POLL_TIMEOUT; + + case -1: // thread cancelled + netdata_log_error("PARSER: thread cancelled while waiting for data."); + return BUFFERED_READER_READ_POLL_CANCELLED; + + default: + case 2: // error on socket + if(revents & POLLERR) { + if(log_error) + netdata_log_error("PARSER: read failed: POLLERR."); + return BUFFERED_READER_READ_POLLERR; + } + if(revents & POLLHUP) { + if(log_error) + netdata_log_error("PARSER: read failed: POLLHUP."); + return BUFFERED_READER_READ_POLLHUP; + } + if(revents & POLLNVAL) { + if(log_error) + netdata_log_error("PARSER: read failed: POLLNVAL."); + return BUFFERED_READER_READ_POLLNVAL; + } } if(log_error) - netdata_log_error("PARSER: poll() failed with code %d.", ret); - return BUFFERED_READER_READ_POLL_FAILED; + netdata_log_error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set."); + return BUFFERED_READER_READ_POLL_UNKNOWN; } /* Produce a full line if one exists, statefully return where we start next time. diff --git a/src/libnetdata/clocks/clocks.c b/src/libnetdata/clocks/clocks.c index 12f8f72e65a88f..e1a3e64cb3610b 100644 --- a/src/libnetdata/clocks/clocks.c +++ b/src/libnetdata/clocks/clocks.c @@ -92,7 +92,7 @@ void clocks_init(void) { inline time_t now_sec(clockid_t clk_id) { struct timespec ts; if(unlikely(clock_gettime(clk_id, &ts) == -1)) { - netdata_log_error("clock_gettime(%d, ×pec) failed.", clk_id); + netdata_log_error("clock_gettime(%ld, ×pec) failed.", (long int)clk_id); return 0; } return ts.tv_sec; @@ -101,7 +101,7 @@ inline time_t now_sec(clockid_t clk_id) { inline usec_t now_usec(clockid_t clk_id) { struct timespec ts; if(unlikely(clock_gettime(clk_id, &ts) == -1)) { - netdata_log_error("clock_gettime(%d, ×pec) failed.", clk_id); + netdata_log_error("clock_gettime(%ld, ×pec) failed.", (long int)clk_id); return 0; } return (usec_t)ts.tv_sec * USEC_PER_SEC + (usec_t)(ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC; @@ -111,7 +111,7 @@ inline int now_timeval(clockid_t clk_id, struct timeval *tv) { struct timespec ts; if(unlikely(clock_gettime(clk_id, &ts) == -1)) { - netdata_log_error("clock_gettime(%d, ×pec) failed.", clk_id); + netdata_log_error("clock_gettime(%ld, ×pec) failed.", (long int)clk_id); tv->tv_sec = 0; tv->tv_usec = 0; return -1; diff --git a/src/libnetdata/config/appconfig.c b/src/libnetdata/config/appconfig.c index fe4c1222d487e7..81946b594a276f 100644 --- a/src/libnetdata/config/appconfig.c +++ b/src/libnetdata/config/appconfig.c @@ -904,7 +904,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) * @return It returns 1 on success and 0 otherwise */ int config_parse_duration(const char* string, int* result) { - while(*string && isspace(*string)) string++; + while(*string && isspace((uint8_t)*string)) string++; if(unlikely(!*string)) goto fallback; @@ -915,7 +915,7 @@ int config_parse_duration(const char* string, int* result) { } // make sure it is a number - if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback; + if(!(isdigit((uint8_t)*string) || *string == '+' || *string == '-')) goto fallback; char *e = NULL; NETDATA_DOUBLE n = str2ndd(string, &e); diff --git a/src/libnetdata/config/dyncfg.c b/src/libnetdata/config/dyncfg.c index 0d6d5058db64d2..28615df2c0ec9a 100644 --- a/src/libnetdata/config/dyncfg.c +++ b/src/libnetdata/config/dyncfg.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "../libnetdata.h" +#include "../../libnetdata/libnetdata.h" // ---------------------------------------------------------------------------- @@ -203,13 +203,32 @@ bool dyncfg_is_valid_id(const char *id) { const char *s = id; while(*s) { - if(isspace(*s) || *s == '\'') return false; + if(isspace((uint8_t)*s) || *s == '\'') return false; s++; } return true; } +static inline bool is_forbidden_char(char c) { + if(isspace((uint8_t)c) || !isprint((uint8_t)c)) + return true; + + switch(c) { + case '/': + return true; + +#ifdef COMPILED_FOR_WINDOWS + case ':': + case '|': + return true; +#endif + + default: + return false; + } +} + char *dyncfg_escape_id_for_filename(const char *id) { if (id == NULL) return NULL; @@ -221,7 +240,7 @@ char *dyncfg_escape_id_for_filename(const char *id) { char *dest = escaped; while (*src) { - if (*src == '/' || isspace(*src) || !isprint(*src)) { + if (is_forbidden_char(*src)) { sprintf(dest, "%%%02X", (unsigned char)*src); dest += 3; } else { diff --git a/src/libnetdata/datetime/rfc3339.c b/src/libnetdata/datetime/rfc3339.c index ac9f59597c4536..5c4e990ddb6a2c 100644 --- a/src/libnetdata/datetime/rfc3339.c +++ b/src/libnetdata/datetime/rfc3339.c @@ -102,8 +102,8 @@ usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) { if (*s == '+' || *s == '-') { // Parse the hours:mins part of the timezone - if (!isdigit(s[1]) || !isdigit(s[2]) || s[3] != ':' || - !isdigit(s[4]) || !isdigit(s[5])) + if (!isdigit((uint8_t)s[1]) || !isdigit((uint8_t)s[2]) || s[3] != ':' || + !isdigit((uint8_t)s[4]) || !isdigit((uint8_t)s[5])) return 0; // Parsing error char tz_sign = *s; diff --git a/src/libnetdata/dictionary/dictionary-item.h b/src/libnetdata/dictionary/dictionary-item.h index d9c67bcb5149ea..f7c6e47a7dd541 100644 --- a/src/libnetdata/dictionary/dictionary-item.h +++ b/src/libnetdata/dictionary/dictionary-item.h @@ -53,7 +53,7 @@ static inline DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, memset(item, 0, sizeof(DICTIONARY_ITEM)); #ifdef NETDATA_INTERNAL_CHECKS - item->creator_pid = gettid(); + item->creator_pid = gettid_cached(); #endif item->refcount = 1; @@ -256,7 +256,7 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dict->items.list, item, prev, next); #ifdef NETDATA_INTERNAL_CHECKS - item->ll_adder_pid = gettid(); + item->ll_adder_pid = gettid_cached(); #endif // clear the BEING created flag, @@ -273,7 +273,7 @@ static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *it DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next); #ifdef NETDATA_INTERNAL_CHECKS - item->ll_remover_pid = gettid(); + item->ll_remover_pid = gettid_cached(); #endif garbage_collect_pending_deletes(dict); diff --git a/src/libnetdata/dictionary/dictionary-locks.h b/src/libnetdata/dictionary/dictionary-locks.h index 01eca1546d0084..90e428104e3a5b 100644 --- a/src/libnetdata/dictionary/dictionary-locks.h +++ b/src/libnetdata/dictionary/dictionary-locks.h @@ -22,19 +22,19 @@ static inline size_t dictionary_locks_destroy(DICTIONARY *dict __maybe_unused) { } static inline void ll_recursive_lock_set_thread_as_writer(DICTIONARY *dict) { - pid_t expected = 0, desired = gettid(); + pid_t expected = 0, desired = gettid_cached(); if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) - fatal("DICTIONARY: Cannot set thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED)); + fatal("DICTIONARY: Cannot set thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid_cached(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED)); } static inline void ll_recursive_unlock_unset_thread_writer(DICTIONARY *dict) { - pid_t expected = gettid(), desired = 0; + pid_t expected = gettid_cached(), desired = 0; if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) - fatal("DICTIONARY: Cannot unset thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED)); + fatal("DICTIONARY: Cannot unset thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid_cached(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED)); } static inline bool ll_recursive_lock_is_thread_the_writer(DICTIONARY *dict) { - pid_t tid = gettid(); + pid_t tid = gettid_cached(); return tid > 0 && tid == __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED); } diff --git a/src/libnetdata/dictionary/dictionary-refcount.h b/src/libnetdata/dictionary/dictionary-refcount.h index 8c7627f52aab89..47d5c275fa65c1 100644 --- a/src/libnetdata/dictionary/dictionary-refcount.h +++ b/src/libnetdata/dictionary/dictionary-refcount.h @@ -219,7 +219,7 @@ static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY #ifdef NETDATA_INTERNAL_CHECKS if(ret == RC_ITEM_OK) - item->deleter_pid = gettid(); + item->deleter_pid = gettid_cached(); #endif if(unlikely(spins > 1)) diff --git a/src/libnetdata/dictionary/dictionary-unittest.c b/src/libnetdata/dictionary/dictionary-unittest.c index 6db427690e2978..716d194a3dd235 100644 --- a/src/libnetdata/dictionary/dictionary-unittest.c +++ b/src/libnetdata/dictionary/dictionary-unittest.c @@ -569,7 +569,7 @@ struct thread_unittest { DICTIONARY *dict; int dups; - netdata_thread_t thread; + ND_THREAD *thread; struct dictionary_stats stats; }; @@ -638,7 +638,7 @@ static void *unittest_dict_thread(void *arg) { // test concurrent deletions and flushes { - if(gettid() % 2) { + if(gettid_cached() % 2) { char buf [256 + 1]; for (int i = 0; i < 1000; i++) { @@ -690,8 +690,7 @@ static int dictionary_unittest_threads() { char buf[100 + 1]; snprintf(buf, 100, "dict%d", i); - netdata_thread_create( - &tu[i].thread, + tu[i].thread = nd_thread_create( buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, unittest_dict_thread, @@ -703,8 +702,7 @@ static int dictionary_unittest_threads() { for (int i = 0; i < threads_to_create; i++) { __atomic_store_n(&tu[i].join, 1, __ATOMIC_RELAXED); - void *retval; - netdata_thread_join(tu[i].thread, &retval); + nd_thread_join(tu[i].thread); if(i) { tu[0].stats.ops.inserts += tu[i].stats.ops.inserts; @@ -870,18 +868,16 @@ static int dictionary_unittest_view_threads() { "\nChecking dictionary concurrency with 1 master and 1 view threads for %lld seconds...\n", (long long)seconds_to_run); - netdata_thread_t master_thread, view_thread; + ND_THREAD *master_thread, *view_thread; tv.join = 0; - netdata_thread_create( - &master_thread, + master_thread = nd_thread_create( "master", NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, unittest_dict_master_thread, &tv); - netdata_thread_create( - &view_thread, + view_thread = nd_thread_create( "view", NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, unittest_dict_view_thread, @@ -890,9 +886,8 @@ static int dictionary_unittest_view_threads() { sleep_usec(seconds_to_run * USEC_PER_SEC); __atomic_store_n(&tv.join, 1, __ATOMIC_RELAXED); - void *retval; - netdata_thread_join(view_thread, &retval); - netdata_thread_join(master_thread, &retval); + nd_thread_join(view_thread); + nd_thread_join(master_thread); #ifdef DICT_WITH_STATS fprintf(stderr, diff --git a/src/libnetdata/dictionary/dictionary.c b/src/libnetdata/dictionary/dictionary.c index 18d6596d79ab9c..9d50ed62cff734 100644 --- a/src/libnetdata/dictionary/dictionary.c +++ b/src/libnetdata/dictionary/dictionary.c @@ -555,7 +555,7 @@ DICTIONARY *dictionary_create_view(DICTIONARY *master) { dict->creation_function = function; dict->creation_file = file; dict->creation_line = line; - dict->creation_tid = gettid(); + dict->creation_tid = gettid_cached(); #endif DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict); diff --git a/src/libnetdata/eval/eval.c b/src/libnetdata/eval/eval.c index 7e968632a64fbc..bacac9c17c4beb 100644 --- a/src/libnetdata/eval/eval.c +++ b/src/libnetdata/eval/eval.c @@ -383,7 +383,7 @@ static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error) // skip spaces static inline void skip_spaces(const char **string) { const char *s = *string; - while(isspace(*s)) s++; + while(isspace((uint8_t)*s)) s++; *string = s; } @@ -1219,7 +1219,7 @@ void expression_hardcode_variable(EVAL_EXPRESSION *expression, STRING *variable, } if (s) { - if (s == s1 && (isalnum(s[len]) || s[len] == '_')) { + if (s == s1 && (isalnum((uint8_t)s[len]) || s[len] == '_')) { // Move past the variable if it's part of a larger word. source_ptr = s + len; continue; diff --git a/src/libnetdata/facets/facets.c b/src/libnetdata/facets/facets.c index a5379e68b7ec2c..3c746cbc314e6a 100644 --- a/src/libnetdata/facets/facets.c +++ b/src/libnetdata/facets/facets.c @@ -2229,8 +2229,8 @@ static int facets_keys_reorder_compar(const void *a, const void *b) { if(!an) an = "0"; if(!bn) bn = "0"; - while(*an && ispunct(*an)) an++; - while(*bn && ispunct(*bn)) bn++; + while(*an && ispunct((uint8_t)*an)) an++; + while(*bn && ispunct((uint8_t)*bn)) bn++; return strcasecmp(an, bn); } @@ -2256,8 +2256,8 @@ static int facets_key_values_reorder_by_name_compar(const void *a, const void *b const char *an = (av->name && av->name_len) ? av->name : "0"; const char *bn = (bv->name && bv->name_len) ? bv->name : "0"; - while(*an && ispunct(*an)) an++; - while(*bn && ispunct(*bn)) bn++; + while(*an && ispunct((uint8_t)*an)) an++; + while(*bn && ispunct((uint8_t)*bn)) bn++; int ret = strcasecmp(an, bn); return ret; @@ -2309,7 +2309,7 @@ static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) { if(all_values_numeric && !v->empty && v->name && v->name_len) { const char *s = v->name; - while(isdigit(*s)) s++; + while(isdigit((uint8_t)*s)) s++; if(*s != '\0') all_values_numeric = false; } diff --git a/src/libnetdata/functions_evloop/functions_evloop.c b/src/libnetdata/functions_evloop/functions_evloop.c index b21abe6294720b..5000d038f9c213 100644 --- a/src/libnetdata/functions_evloop/functions_evloop.c +++ b/src/libnetdata/functions_evloop/functions_evloop.c @@ -49,9 +49,10 @@ struct functions_evloop_globals { netdata_mutex_t *stdout_mutex; bool *plugin_should_exit; + bool workers_exit; // all workers are waiting on the same condition - this makes them all exit, when any is cancelled - netdata_thread_t reader_thread; - netdata_thread_t *worker_threads; + ND_THREAD *reader_thread; + ND_THREAD **worker_threads; struct { DICTIONARY *nodes; @@ -60,13 +61,28 @@ struct functions_evloop_globals { struct rrd_functions_expectation *expectations; }; +static void rrd_functions_worker_canceller(void *data) { + struct functions_evloop_globals *wg = data; + pthread_mutex_lock(&wg->worker_mutex); + wg->workers_exit = true; + pthread_cond_signal(&wg->worker_cond_var); + pthread_mutex_unlock(&wg->worker_mutex); +} + static void *rrd_functions_worker_globals_worker_main(void *arg) { struct functions_evloop_globals *wg = arg; + nd_thread_register_canceller(rrd_functions_worker_canceller, wg); + bool last_acquired = true; while (true) { pthread_mutex_lock(&wg->worker_mutex); + if(wg->workers_exit || nd_thread_signaled_to_cancel()) { + pthread_mutex_unlock(&wg->worker_mutex); + break; + } + if(dictionary_entries(wg->worker_queue) == 0 || !last_acquired) pthread_cond_wait(&wg->worker_cond_var, &wg->worker_mutex); @@ -84,6 +100,13 @@ static void *rrd_functions_worker_globals_worker_main(void *arg) { pthread_mutex_unlock(&wg->worker_mutex); + if(wg->workers_exit || nd_thread_signaled_to_cancel()) { + if(acquired) + dictionary_acquired_item_release(wg->worker_queue, acquired); + + break; + } + if(acquired) { ND_LOG_STACK lgs[] = { ND_LOG_FIELD_TXT(NDF_REQUEST, j->cmd), @@ -101,6 +124,7 @@ static void *rrd_functions_worker_globals_worker_main(void *arg) { else last_acquired = false; } + return NULL; } @@ -146,7 +170,9 @@ static void worker_add_job(struct functions_evloop_globals *wg, const char *keyw else { found = true; j->used = true; + pthread_mutex_lock(&wg->worker_mutex); pthread_cond_signal(&wg->worker_cond_var); + pthread_mutex_unlock(&wg->worker_mutex); } } } @@ -310,18 +336,18 @@ struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, co wg->plugin_should_exit = plugin_should_exit; wg->stdout_mutex = stdout_mutex; wg->workers = worker_threads; - wg->worker_threads = callocz(wg->workers, sizeof(netdata_thread_t )); + wg->worker_threads = callocz(wg->workers, sizeof(ND_THREAD *)); wg->tag = tag; char tag_buffer[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(tag_buffer, NETDATA_THREAD_TAG_MAX, "%s_READER", wg->tag); - netdata_thread_create(&wg->reader_thread, tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG, - rrd_functions_worker_globals_reader_main, wg); + wg->reader_thread = nd_thread_create(tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG, + rrd_functions_worker_globals_reader_main, wg); for(size_t i = 0; i < wg->workers ; i++) { snprintfz(tag_buffer, NETDATA_THREAD_TAG_MAX, "%s_WORK[%zu]", wg->tag, i+1); - netdata_thread_create(&wg->worker_threads[i], tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG, - rrd_functions_worker_globals_worker_main, wg); + wg->worker_threads[i] = nd_thread_create(tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG, + rrd_functions_worker_globals_worker_main, wg); } functions_evloop_add_function(wg, "config", functions_evloop_config_cb, 120, wg); @@ -339,11 +365,11 @@ void functions_evloop_add_function(struct functions_evloop_globals *wg, const ch DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wg->expectations, we, prev, next); } -void functions_evloop_cancel_threads(struct functions_evloop_globals *wg){ - for(size_t i = 0; i < wg->workers ; i++) - netdata_thread_cancel(wg->worker_threads[i]); +void functions_evloop_cancel_threads(struct functions_evloop_globals *wg) { + nd_thread_signal_cancel(wg->reader_thread); - netdata_thread_cancel(wg->reader_thread); + for(size_t i = 0; i < wg->workers ; i++) + nd_thread_signal_cancel(wg->worker_threads[i]); } // ---------------------------------------------------------------------------- diff --git a/src/libnetdata/inlined.h b/src/libnetdata/inlined.h index 9c0d2dd0b6989b..6b71590c99af78 100644 --- a/src/libnetdata/inlined.h +++ b/src/libnetdata/inlined.h @@ -453,7 +453,7 @@ static inline bool sanitize_command_argument_string(char *dst, const char *src, if (dst_size < 1) return false; - if (iscntrl(*src) || *src == '$') { + if (iscntrl((uint8_t)*src) || *src == '$') { // remove control characters and characters that are expanded by bash *dst++ = '_'; dst_size--; @@ -597,7 +597,7 @@ static inline char *strsep_skip_consecutive_separators(char **ptr, char *s) { // remove leading and trailing spaces; may return NULL static inline char *trim(char *s) { // skip leading spaces - while (*s && isspace(*s)) s++; + while (*s && isspace((uint8_t)*s)) s++; if (!*s) return NULL; // skip tailing spaces @@ -605,7 +605,7 @@ static inline char *trim(char *s) { ssize_t l = (ssize_t)strlen(s); if (--l >= 0) { char *p = s + l; - while (p > s && isspace(*p)) p--; + while (p > s && isspace((uint8_t)*p)) p--; *++p = '\0'; } @@ -619,27 +619,27 @@ static inline char *trim_all(char *buffer) { char *d = buffer, *s = buffer; // skip spaces - while(isspace(*s)) s++; + while(isspace((uint8_t)*s)) s++; while(*s) { // copy the non-space part - while(*s && !isspace(*s)) *d++ = *s++; + while(*s && !isspace((uint8_t)*s)) *d++ = *s++; // add a space if we have to - if(*s && isspace(*s)) { + if(*s && isspace((uint8_t)*s)) { *d++ = ' '; s++; } // skip spaces - while(isspace(*s)) s++; + while(isspace((uint8_t)*s)) s++; } *d = '\0'; if(d > buffer) { d--; - if(isspace(*d)) *d = '\0'; + if(isspace((uint8_t)*d)) *d = '\0'; } if(!buffer[0]) return NULL; diff --git a/src/libnetdata/libnetdata.c b/src/libnetdata/libnetdata.c index 2dcd8865c56986..7536ab26d446bd 100644 --- a/src/libnetdata/libnetdata.c +++ b/src/libnetdata/libnetdata.c @@ -2,17 +2,17 @@ #include "libnetdata.h" -#ifdef __APPLE__ -#define INHERIT_NONE 0 -#endif /* __APPLE__ */ -#if defined(__FreeBSD__) || defined(__APPLE__) -# define O_NOATIME 0 -# define MADV_DONTFORK INHERIT_NONE -#endif /* __FreeBSD__ || __APPLE__*/ +#if !defined(MADV_DONTFORK) +#define MADV_DONTFORK 0 +#endif + +#if !defined(O_NOATIME) +#define O_NOATIME 0 +#endif struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 }; -#ifdef MADV_MERGEABLE +#if defined(MADV_MERGEABLE) int enable_ksm = 1; #else int enable_ksm = 0; diff --git a/src/libnetdata/libnetdata.h b/src/libnetdata/libnetdata.h index 21ae7ddc47b7ba..41a3bf7c95de06 100644 --- a/src/libnetdata/libnetdata.h +++ b/src/libnetdata/libnetdata.h @@ -76,6 +76,8 @@ extern "C" { #define NETDATA_OS_TYPE "freebsd" #elif defined(__APPLE__) #define NETDATA_OS_TYPE "macos" +#elif defined(COMPILED_FOR_WINDOWS) +#define NETDATA_OS_TYPE "windows" #else #define NETDATA_OS_TYPE "linux" #endif /* __FreeBSD__, __APPLE__*/ @@ -90,40 +92,112 @@ extern "C" { #include #include #include -#include -#include -#include #include #include #include #include -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ARPA_INET_H +#include +#endif + +#ifdef HAVE_NETINET_TCP_H +#include +#endif + +#ifdef HAVE_SYS_IOCTL_H +#include +#endif + +#ifdef HAVE_GRP_H +#include +#else +typedef uint32_t gid_t; +#endif + +#ifdef HAVE_PWD_H +#include +#else +typedef uint32_t uid_t; +#endif + +#ifdef HAVE_NET_IF_H #include +#endif + +#ifdef HAVE_POLL_H #include -#include +#endif + +#ifdef HAVE_SYSLOG_H #include +#else +/* priorities */ +#define LOG_EMERG 0 /* system is unusable */ +#define LOG_ALERT 1 /* action must be taken immediately */ +#define LOG_CRIT 2 /* critical conditions */ +#define LOG_ERR 3 /* error conditions */ +#define LOG_WARNING 4 /* warning conditions */ +#define LOG_NOTICE 5 /* normal but significant condition */ +#define LOG_INFO 6 /* informational */ +#define LOG_DEBUG 7 /* debug-level messages */ + +/* facility codes */ +#define LOG_KERN (0<<3) /* kernel messages */ +#define LOG_USER (1<<3) /* random user-level messages */ +#define LOG_MAIL (2<<3) /* mail system */ +#define LOG_DAEMON (3<<3) /* system daemons */ +#define LOG_AUTH (4<<3) /* security/authorization messages */ +#define LOG_SYSLOG (5<<3) /* messages generated internally by syslogd */ +#define LOG_LPR (6<<3) /* line printer subsystem */ +#define LOG_NEWS (7<<3) /* network news subsystem */ +#define LOG_UUCP (8<<3) /* UUCP subsystem */ +#define LOG_CRON (9<<3) /* clock daemon */ +#define LOG_AUTHPRIV (10<<3) /* security/authorization messages (private) */ +#define LOG_FTP (11<<3) /* ftp daemon */ + +/* other codes through 15 reserved for system use */ +#define LOG_LOCAL0 (16<<3) /* reserved for local use */ +#define LOG_LOCAL1 (17<<3) /* reserved for local use */ +#define LOG_LOCAL2 (18<<3) /* reserved for local use */ +#define LOG_LOCAL3 (19<<3) /* reserved for local use */ +#define LOG_LOCAL4 (20<<3) /* reserved for local use */ +#define LOG_LOCAL5 (21<<3) /* reserved for local use */ +#define LOG_LOCAL6 (22<<3) /* reserved for local use */ +#define LOG_LOCAL7 (23<<3) /* reserved for local use */ +#endif + +#ifdef HAVE_SYS_MMAN_H #include +#endif + +#ifdef HAVE_SYS_RESOURCE_H #include +#endif + +#ifdef HAVE_SYS_SOCKET_H #include -#include -#include -#include +#endif + +#ifdef HAVE_SYS_WAIT_H #include +#endif + +#ifdef HAVE_SYS_UN_H #include -#include -#include -#include -#include -#include -#include +#endif -// CentOS 7 has older version that doesn't define this -// same goes for MacOS -#ifndef UUID_STR_LEN -#define UUID_STR_LEN (37) +#ifdef HAVE_SPAWN_H +#include #endif #ifdef HAVE_NETINET_IN_H @@ -190,6 +264,10 @@ extern "C" { #endif +#ifndef O_CLOEXEC +#define O_CLOEXEC (0) +#endif + // ---------------------------------------------------------------------------- // netdata common definitions @@ -213,8 +291,10 @@ extern "C" { #define MALLOCLIKE #endif -#ifdef HAVE_FUNC_ATTRIBUTE_FORMAT -#define PRINTFLIKE(f, a) __attribute__ ((format(__printf__, f, a))) +#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT) && !defined(COMPILED_FOR_MACOS) +#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a))) +#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT) +#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a))) #else #define PRINTFLIKE(f, a) #endif @@ -376,6 +456,8 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds); void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) NORETURN; extern char *netdata_configured_host_prefix; +#include "os/os.h" + #define XXH_INLINE_ALL #include "xxhash.h" @@ -386,7 +468,6 @@ extern char *netdata_configured_host_prefix; #include "config/dyncfg.h" #include "libjudy/src/Judy.h" #include "july/july.h" -#include "os.h" #include "threads/threads.h" #include "buffer/buffer.h" #include "locks/locks.h" @@ -594,6 +675,33 @@ static inline void freez_const_charp(const char **p) { #define CLEAN_CONST_CHAR_P _cleanup_(freez_const_charp) const char #define CLEAN_CHAR_P _cleanup_(freez_charp) char +// -------------------------------------------------------------------------------------------------------------------- +// automatic cleanup function, instead of pthread pop/push + +// volatile: Tells the compiler that the variable defined might be accessed in unexpected ways +// (e.g., by the cleanup function). This prevents it from being optimized out. +#define CLEANUP_FUNCTION_REGISTER(func) volatile void * __attribute__((cleanup(func))) + +static inline void *CLEANUP_FUNCTION_GET_PTR(void *pptr) { + void *ret; + void **p = (void **)pptr; + if(p) { + ret = *p; + *p = NULL; // use it only once - this will prevent using it again + + if(!ret) + nd_log(NDLS_DAEMON, NDLP_ERR, "cleanup function called multiple times!"); + } + else { + nd_log(NDLS_DAEMON, NDLP_ERR, "cleanup function called with NULL pptr!"); + ret = NULL; + } + + return ret; +} + +// -------------------------------------------------------------------------------------------------------------------- + # ifdef __cplusplus } # endif diff --git a/src/libnetdata/locks/locks.c b/src/libnetdata/locks/locks.c index adf683af232ffc..d01ee29f143851 100644 --- a/src/libnetdata/locks/locks.c +++ b/src/libnetdata/locks/locks.c @@ -18,64 +18,6 @@ #endif // NETDATA_TRACE_RWLOCKS -// ---------------------------------------------------------------------------- -// automatic thread cancelability management, based on locks - -static __thread int netdata_thread_first_cancelability = 0; -static __thread int netdata_thread_nested_disables = 0; - -static __thread size_t netdata_locks_acquired_rwlocks = 0; -static __thread size_t netdata_locks_acquired_mutexes = 0; - -inline void netdata_thread_disable_cancelability(void) { - if(!netdata_thread_nested_disables) { - int old; - int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old); - - if(ret != 0) - netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", - netdata_thread_tag(), ret); - - netdata_thread_first_cancelability = old; - } - - netdata_thread_nested_disables++; -} - -inline void netdata_thread_enable_cancelability(void) { - if(unlikely(netdata_thread_nested_disables < 1)) { - internal_fatal(true, "THREAD_CANCELABILITY: trying to enable cancelability, but it was not not disabled"); - - netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d " - "on thread %s - results will be undefined - please report this!", - netdata_thread_nested_disables, netdata_thread_tag()); - - netdata_thread_nested_disables = 1; - } - - if(netdata_thread_nested_disables == 1) { - int old = 1; - int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old); - if(ret != 0) - netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", - netdata_thread_tag(), - ret); - else { - if(old != PTHREAD_CANCEL_DISABLE) { - internal_fatal(true, "THREAD_CANCELABILITY: invalid old state cancelability"); - - netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability " - "on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", - netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, - (old == PTHREAD_CANCEL_ENABLE) ? "ENABLED" : "UNKNOWN", - old); - } - } - } - - netdata_thread_nested_disables--; -} - // ---------------------------------------------------------------------------- // mutex @@ -94,27 +36,22 @@ int __netdata_mutex_destroy(netdata_mutex_t *mutex) { } int __netdata_mutex_lock(netdata_mutex_t *mutex) { - netdata_thread_disable_cancelability(); - int ret = pthread_mutex_lock(mutex); if(unlikely(ret != 0)) { - netdata_thread_enable_cancelability(); netdata_log_error("MUTEX_LOCK: failed to get lock (code %d)", ret); } else - netdata_locks_acquired_mutexes++; + nd_thread_mutex_locked(); return ret; } int __netdata_mutex_trylock(netdata_mutex_t *mutex) { - netdata_thread_disable_cancelability(); - int ret = pthread_mutex_trylock(mutex); if(ret != 0) - netdata_thread_enable_cancelability(); + ; else - netdata_locks_acquired_mutexes++; + nd_thread_mutex_locked(); return ret; } @@ -123,10 +60,8 @@ int __netdata_mutex_unlock(netdata_mutex_t *mutex) { int ret = pthread_mutex_unlock(mutex); if(unlikely(ret != 0)) netdata_log_error("MUTEX_LOCK: failed to unlock (code %d).", ret); - else { - netdata_locks_acquired_mutexes--; - netdata_thread_enable_cancelability(); - } + else + nd_thread_mutex_unlocked(); return ret; } @@ -226,65 +161,61 @@ int __netdata_rwlock_init(netdata_rwlock_t *rwlock) { } int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) { - netdata_thread_disable_cancelability(); - int ret = pthread_rwlock_rdlock(&rwlock->rwlock_t); - if(unlikely(ret != 0)) { - netdata_thread_enable_cancelability(); + if(unlikely(ret != 0)) netdata_log_error("RW_LOCK: failed to obtain read lock (code %d)", ret); - } else - netdata_locks_acquired_rwlocks++; + nd_thread_rwlock_read_locked(); return ret; } int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) { - netdata_thread_disable_cancelability(); - int ret = pthread_rwlock_wrlock(&rwlock->rwlock_t); - if(unlikely(ret != 0)) { + if(unlikely(ret != 0)) netdata_log_error("RW_LOCK: failed to obtain write lock (code %d)", ret); - netdata_thread_enable_cancelability(); - } else - netdata_locks_acquired_rwlocks++; + nd_thread_rwlock_write_locked(); return ret; } -int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) { +int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_unlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret); - else { - netdata_thread_enable_cancelability(); - netdata_locks_acquired_rwlocks--; - } + else + nd_thread_rwlock_read_unlocked(); return ret; } -int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) { - netdata_thread_disable_cancelability(); +int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock) { + int ret = pthread_rwlock_unlock(&rwlock->rwlock_t); + if(unlikely(ret != 0)) + netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret); + else + nd_thread_rwlock_write_unlocked(); + return ret; +} + +int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_tryrdlock(&rwlock->rwlock_t); if(ret != 0) - netdata_thread_enable_cancelability(); + ; else - netdata_locks_acquired_rwlocks++; + nd_thread_rwlock_read_locked(); return ret; } int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) { - netdata_thread_disable_cancelability(); - int ret = pthread_rwlock_trywrlock(&rwlock->rwlock_t); if(ret != 0) - netdata_thread_enable_cancelability(); + ; else - netdata_locks_acquired_rwlocks++; + nd_thread_rwlock_write_locked(); return ret; } @@ -297,16 +228,11 @@ void spinlock_init(SPINLOCK *spinlock) { memset(spinlock, 0, sizeof(SPINLOCK)); } -static inline void spinlock_lock_internal(SPINLOCK *spinlock, bool cancelable) { - static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; - +static inline void spinlock_lock_internal(SPINLOCK *spinlock) { #ifdef NETDATA_INTERNAL_CHECKS size_t spins = 0; #endif - if (!cancelable) - netdata_thread_disable_cancelability(); - for(int i = 1; __atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) || __atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE) @@ -318,7 +244,7 @@ static inline void spinlock_lock_internal(SPINLOCK *spinlock, bool cancelable) { #endif if(unlikely(i == 8)) { i = 0; - nanosleep(&ns, NULL); + tinysleep(); } } @@ -326,63 +252,60 @@ static inline void spinlock_lock_internal(SPINLOCK *spinlock, bool cancelable) { #ifdef NETDATA_INTERNAL_CHECKS spinlock->spins += spins; - spinlock->locker_pid = gettid(); + spinlock->locker_pid = gettid_cached(); #endif + + nd_thread_spinlock_locked(); } -static inline void spinlock_unlock_internal(SPINLOCK *spinlock, bool cancelable) { +static inline void spinlock_unlock_internal(SPINLOCK *spinlock) { #ifdef NETDATA_INTERNAL_CHECKS spinlock->locker_pid = 0; #endif __atomic_clear(&spinlock->locked, __ATOMIC_RELEASE); - if (!cancelable) - netdata_thread_enable_cancelability(); + nd_thread_spinlock_unlocked(); } -static inline bool spinlock_trylock_internal(SPINLOCK *spinlock, bool cancelable) { - if (!cancelable) - netdata_thread_disable_cancelability(); - +static inline bool spinlock_trylock_internal(SPINLOCK *spinlock) { if(!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) && - !__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE)) + !__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE)) { // we got the lock + nd_thread_spinlock_locked(); return true; + } - // we didn't get the lock - if (!cancelable) - netdata_thread_enable_cancelability(); return false; } void spinlock_lock(SPINLOCK *spinlock) { - spinlock_lock_internal(spinlock, false); + spinlock_lock_internal(spinlock); } void spinlock_unlock(SPINLOCK *spinlock) { - spinlock_unlock_internal(spinlock, false); + spinlock_unlock_internal(spinlock); } bool spinlock_trylock(SPINLOCK *spinlock) { - return spinlock_trylock_internal(spinlock, false); + return spinlock_trylock_internal(spinlock); } void spinlock_lock_cancelable(SPINLOCK *spinlock) { - spinlock_lock_internal(spinlock, true); + spinlock_lock_internal(spinlock); } void spinlock_unlock_cancelable(SPINLOCK *spinlock) { - spinlock_unlock_internal(spinlock, true); + spinlock_unlock_internal(spinlock); } bool spinlock_trylock_cancelable(SPINLOCK *spinlock) { - return spinlock_trylock_internal(spinlock, true); + return spinlock_trylock_internal(spinlock); } // ---------------------------------------------------------------------------- @@ -394,11 +317,11 @@ void rw_spinlock_init(RW_SPINLOCK *rw_spinlock) { } void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock) { - netdata_thread_disable_cancelability(); - spinlock_lock(&rw_spinlock->spinlock); __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); spinlock_unlock(&rw_spinlock->spinlock); + + nd_thread_rwspinlock_read_locked(); } void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) { @@ -410,12 +333,10 @@ void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) { fatal("RW_SPINLOCK: readers is negative %d", x); #endif - netdata_thread_enable_cancelability(); + nd_thread_rwspinlock_read_unlocked(); } void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) { - static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; - size_t spins = 0; while(1) { spins++; @@ -426,21 +347,24 @@ void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) { // Busy wait until all readers have released their locks. spinlock_unlock(&rw_spinlock->spinlock); - nanosleep(&ns, NULL); + tinysleep(); } (void)spins; + + nd_thread_rwspinlock_write_locked(); } void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock) { spinlock_unlock(&rw_spinlock->spinlock); + nd_thread_rwspinlock_write_unlocked(); } bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock) { if(spinlock_trylock(&rw_spinlock->spinlock)) { __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); spinlock_unlock(&rw_spinlock->spinlock); - netdata_thread_disable_cancelability(); + nd_thread_rwspinlock_read_locked(); return true; } @@ -451,6 +375,7 @@ bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock) { if(spinlock_trylock(&rw_spinlock->spinlock)) { if (__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) { // No readers, we've successfully acquired the write lock + nd_thread_rwspinlock_write_locked(); return true; } else { @@ -583,7 +508,22 @@ int netdata_rwlock_wrlock_debug(const char *file __maybe_unused, const char *fun return ret; } -int netdata_rwlock_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, +int netdata_rwlock_rdunlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + + netdata_rwlock_locker *locker = find_rwlock_locker(file, function, line, rwlock); + + if(unlikely(!locker)) + fatal("UNLOCK WITHOUT LOCK"); + + int ret = __netdata_rwlock_rdunlock(rwlock); + if(likely(!ret)) + remove_rwlock_locker(file, function, line, rwlock, locker); + + return ret; +} + +int netdata_rwlock_wrunlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { netdata_rwlock_locker *locker = find_rwlock_locker(file, function, line, rwlock); @@ -591,7 +531,7 @@ int netdata_rwlock_unlock_debug(const char *file __maybe_unused, const char *fun if(unlikely(!locker)) fatal("UNLOCK WITHOUT LOCK"); - int ret = __netdata_rwlock_unlock(rwlock); + int ret = __netdata_rwlock_wrunlock(rwlock); if(likely(!ret)) remove_rwlock_locker(file, function, line, rwlock, locker); diff --git a/src/libnetdata/locks/locks.h b/src/libnetdata/locks/locks.h index 09adfb41fc6556..d3873c295204af 100644 --- a/src/libnetdata/locks/locks.h +++ b/src/libnetdata/locks/locks.h @@ -106,13 +106,11 @@ int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock); int __netdata_rwlock_init(netdata_rwlock_t *rwlock); int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock); int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock); -int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock); +int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock); +int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock); int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock); int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock); -void netdata_thread_disable_cancelability(void); -void netdata_thread_enable_cancelability(void); - #ifdef NETDATA_TRACE_RWLOCKS int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex); @@ -125,7 +123,8 @@ int netdata_rwlock_destroy_debug( const char *file, const char *function, const int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); -int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +int netdata_rwlock_rdunlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +int netdata_rwlock_wrunlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); @@ -139,7 +138,8 @@ int netdata_rwlock_trywrlock_debug( const char *file, const char *function, cons #define netdata_rwlock_init(rwlock) netdata_rwlock_init_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) #define netdata_rwlock_rdlock(rwlock) netdata_rwlock_rdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) #define netdata_rwlock_wrlock(rwlock) netdata_rwlock_wrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) -#define netdata_rwlock_unlock(rwlock) netdata_rwlock_unlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_rdunlock(rwlock) netdata_rwlock_rdunlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_wrunlock(rwlock) netdata_rwlock_wrunlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) #define netdata_rwlock_tryrdlock(rwlock) netdata_rwlock_tryrdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) #define netdata_rwlock_trywrlock(rwlock) netdata_rwlock_trywrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) @@ -155,7 +155,8 @@ int netdata_rwlock_trywrlock_debug( const char *file, const char *function, cons #define netdata_rwlock_init(rwlock) __netdata_rwlock_init(rwlock) #define netdata_rwlock_rdlock(rwlock) __netdata_rwlock_rdlock(rwlock) #define netdata_rwlock_wrlock(rwlock) __netdata_rwlock_wrlock(rwlock) -#define netdata_rwlock_unlock(rwlock) __netdata_rwlock_unlock(rwlock) +#define netdata_rwlock_rdunlock(rwlock) __netdata_rwlock_rdunlock(rwlock) +#define netdata_rwlock_wrunlock(rwlock) __netdata_rwlock_wrunlock(rwlock) #define netdata_rwlock_tryrdlock(rwlock) __netdata_rwlock_tryrdlock(rwlock) #define netdata_rwlock_trywrlock(rwlock) __netdata_rwlock_trywrlock(rwlock) diff --git a/src/libnetdata/log/journal.c b/src/libnetdata/log/journal.c index 0f1248de7fd1ab..2182212f6dcfc7 100644 --- a/src/libnetdata/log/journal.c +++ b/src/libnetdata/log/journal.c @@ -67,7 +67,7 @@ int journal_direct_fd(const char *path) { return fd; } -static inline bool journal_send_with_memfd(int fd, const char *msg, size_t msg_len) { +static inline bool journal_send_with_memfd(int fd __maybe_unused, const char *msg __maybe_unused, size_t msg_len __maybe_unused) { #if defined(__NR_memfd_create) && defined(MFD_ALLOW_SEALING) && defined(F_ADD_SEALS) && defined(F_SEAL_SHRINK) && defined(F_SEAL_GROW) && defined(F_SEAL_WRITE) // Create a memory file descriptor int memfd = (int)syscall(__NR_memfd_create, "journald", MFD_ALLOW_SEALING); diff --git a/src/libnetdata/log/log.c b/src/libnetdata/log/log.c index 2be7a788139d20..501b6632452e2b 100644 --- a/src/libnetdata/log/log.c +++ b/src/libnetdata/log/log.c @@ -22,8 +22,6 @@ #include #endif -#include - const char *program_name = ""; uint64_t debug_flags = 0; @@ -368,7 +366,7 @@ struct nd_log_source { }; static struct { - uuid_t invocation_id; + nd_uuid_t invocation_id; ND_LOG_SOURCES overwrite_process_source; @@ -1599,7 +1597,7 @@ static bool needs_quotes_for_logfmt(const char *s) return true; while(*s) { - if(*s == '=' || isspace(*s) || !safe_for_logfmt[(uint8_t)*s]) + if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s]) return true; s++; @@ -1733,7 +1731,7 @@ bool nd_log_journal_socket_available(void) { return is_path_unix_socket("/run/systemd/journal/socket"); } -static bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields_max) { +static bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) { #ifdef HAVE_SYSTEMD // --- FIELD_PARSER_VERSIONS --- @@ -2153,17 +2151,10 @@ static void nd_logger(const char *file, const char *function, const unsigned lon } if(likely(!thread_log_fields[NDF_TID].entry.set)) - thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid()); + thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached()); - char os_threadname[NETDATA_THREAD_NAME_MAX + 1]; if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) { - const char *thread_tag = netdata_thread_tag(); - if (!netdata_thread_tag_exists()) { - os_thread_get_current_name_np(os_threadname); - if ('\0' != os_threadname[0]) - /* If it is not an empty string replace "MAIN" thread_tag */ - thread_tag = os_threadname; - } + const char *thread_tag = nd_thread_tag(); thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag); // TODO: fix the ND_MODULE in logging by setting proper module name in threads @@ -2303,17 +2294,7 @@ void netdata_logger_fatal( const char *file, const char *function, const unsigne char action_data[70+1]; snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno); - char os_threadname[NETDATA_THREAD_NAME_MAX + 1]; - const char *thread_tag = netdata_thread_tag(); - if (!netdata_thread_tag_exists()) { - os_thread_get_current_name_np(os_threadname); - if ('\0' != os_threadname[0]) - /* If it is not an empty string replace "MAIN" thread_tag */ - thread_tag = os_threadname; - } - if(!thread_tag) - thread_tag = "UNKNOWN"; - + const char *thread_tag = nd_thread_tag(); const char *tag_to_send = thread_tag; // anonymize thread names diff --git a/src/libnetdata/log/log.h b/src/libnetdata/log/log.h index 3bafbe89bf17f1..338a5d53b7ef91 100644 --- a/src/libnetdata/log/log.h +++ b/src/libnetdata/log/log.h @@ -170,7 +170,7 @@ struct log_stack_entry { uint64_t u64; int64_t i64; double dbl; - const uuid_t *uuid; + const nd_uuid_t *uuid; struct { log_formatter_callback_t formatter; void *formatter_data; diff --git a/src/libnetdata/maps/local-sockets.h b/src/libnetdata/maps/local-sockets.h index ce5201242393a7..d407e6be6b5293 100644 --- a/src/libnetdata/maps/local-sockets.h +++ b/src/libnetdata/maps/local-sockets.h @@ -203,7 +203,7 @@ typedef struct local_socket { // -------------------------------------------------------------------------------------------------------------------- -static inline void local_sockets_log(LS_STATE *ls, const char *format, ...) __attribute__ ((format(__printf__, 2, 3))); +static inline void local_sockets_log(LS_STATE *ls, const char *format, ...) PRINTFLIKE(2, 3); static inline void local_sockets_log(LS_STATE *ls, const char *format, ...) { if(++ls->stats.errors_encountered == ls->config.max_errors) { nd_log(NDLS_COLLECTORS, NDLP_ERR, "LOCAL-SOCKETS: max number of logs reached. Not logging anymore"); diff --git a/src/libnetdata/os.c b/src/libnetdata/os.c deleted file mode 100644 index 51001aa2505d14..00000000000000 --- a/src/libnetdata/os.c +++ /dev/null @@ -1,300 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "os.h" - -// ---------------------------------------------------------------------------- -// system functions -// to retrieve settings of the system - -#define CPUS_FOR_COLLECTORS 0 -#define CPUS_FOR_NETDATA 1 - -long get_system_cpus_with_cache(bool cache, bool for_netdata) { - static long processors[2] = { 0, 0 }; - - int index = for_netdata ? CPUS_FOR_NETDATA : CPUS_FOR_COLLECTORS; - - if(likely(cache && processors[index] > 0)) - return processors[index]; - -#if defined(__APPLE__) || defined(__FreeBSD__) -#if defined(__APPLE__) -#define HW_CPU_NAME "hw.logicalcpu" -#else -#define HW_CPU_NAME "hw.ncpu" -#endif - - int32_t tmp_processors; - bool error = false; - - if (unlikely(GETSYSCTL_BY_NAME(HW_CPU_NAME, tmp_processors))) - error = true; - else - processors[index] = tmp_processors; - - if(processors[index] < 1) { - processors[index] = 1; - - if(error) - netdata_log_error("Assuming system has %d processors.", processors[index]); - } - - return processors[index]; -#else - - char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "%s/proc/stat", - (!for_netdata && netdata_configured_host_prefix) ? netdata_configured_host_prefix : ""); - - procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); - if(!ff) { - processors[index] = 1; - netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); - return processors[index]; - } - - ff = procfile_readall(ff); - if(!ff) { - processors[index] = 1; - netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); - return processors[index]; - } - - long tmp_processors = 0; - unsigned int i; - for(i = 0; i < procfile_lines(ff); i++) { - if(!procfile_linewords(ff, i)) continue; - - if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) - tmp_processors++; - } - procfile_close(ff); - - processors[index] = --tmp_processors; - - if(processors[index] < 1) - processors[index] = 1; - - netdata_log_debug(D_SYSTEM, "System has %ld processors.", processors[index]); - return processors[index]; - -#endif /* __APPLE__, __FreeBSD__ */ -} - -pid_t pid_max = 32768; -pid_t get_system_pid_max(void) { -#ifdef __APPLE__ - // As we currently do not know a solution to query pid_max from the os - // we use the number defined in bsd/sys/proc_internal.h in XNU sources - pid_max = 99999; - return pid_max; -#elif __FreeBSD__ - int32_t tmp_pid_max; - - if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) { - pid_max = 99999; - netdata_log_error("Assuming system's maximum pid is %d.", pid_max); - } else { - pid_max = tmp_pid_max; - } - - return pid_max; -#else - - static char read = 0; - if(unlikely(read)) return pid_max; - read = 1; - - char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix?netdata_configured_host_prefix:""); - - unsigned long long max = 0; - if(read_single_number_file(filename, &max) != 0) { - netdata_log_error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max); - return pid_max; - } - - if(!max) { - netdata_log_error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max); - return pid_max; - } - - pid_max = (pid_t) max; - return pid_max; - -#endif /* __APPLE__, __FreeBSD__ */ -} - -unsigned int system_hz; -void get_system_HZ(void) { - long ticks; - - if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { - netdata_log_error("Cannot get system clock ticks"); - } - - system_hz = (unsigned int) ticks; -} - -static inline unsigned long cpuset_str2ul(char **s) { - unsigned long n = 0; - char c; - for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) { - n *= 10; - n += c - '0'; - } - return n; -} - -unsigned long read_cpuset_cpus(const char *filename, long system_cpus) { - static char *buf = NULL; - static size_t buf_size = 0; - - if(!buf) { - buf_size = 100U + 6 * system_cpus + 1; // taken from kernel/cgroup/cpuset.c - buf = mallocz(buf_size); - } - - int ret = read_txt_file(filename, buf, buf_size); - - if(!ret) { - char *s = buf; - unsigned long ncpus = 0; - - // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use - while (*s) { - if (isspace(*s)) { - s++; - continue; - } - unsigned long n = cpuset_str2ul(&s); - ncpus++; - if(*s == ',') { - s++; - continue; - } - if(*s == '-') { - s++; - unsigned long m = cpuset_str2ul(&s); - ncpus += m - n; // calculate the number of cpus in the region - } - s++; - } - - if(!ncpus) - return 0; - - return ncpus; - } - - return 0; -} - -// ===================================================================================================================== -// FreeBSD - -#if __FreeBSD__ - -const char *os_type = "freebsd"; - -int getsysctl_by_name(const char *name, void *ptr, size_t len) { - size_t nlen = len; - - if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { - netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); - return 1; - } - if (unlikely(nlen != len)) { - netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); - return 1; - } - return 0; -} - -int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len) { - size_t nlen = len; - - if (unlikely(!mib[0])) - if (unlikely(getsysctl_mib(name, mib, miblen))) - return 1; - - if (unlikely(sysctl(mib, miblen, ptr, &nlen, NULL, 0) == -1)) { - netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); - return 1; - } - if (unlikely(nlen != len)) { - netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); - return 1; - } - - return 0; -} - -int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len) { - size_t nlen = *len; - - if (unlikely(!mib[0])) - if (unlikely(getsysctl_mib(name, mib, miblen))) - return 1; - - if (unlikely(sysctl(mib, miblen, ptr, len, NULL, 0) == -1)) { - netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); - return 1; - } - if (unlikely(ptr != NULL && nlen != *len)) { - netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen); - return 1; - } - - return 0; -} - -int getsysctl_mib(const char *name, int *mib, size_t len) { - size_t nlen = len; - - if (unlikely(sysctlnametomib(name, mib, &nlen) == -1)) { - netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); - return 1; - } - if (unlikely(nlen != len)) { - netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); - return 1; - } - return 0; -} - - -#endif - - -// ===================================================================================================================== -// MacOS - -#if __APPLE__ - -const char *os_type = "macos"; - -int getsysctl_by_name(const char *name, void *ptr, size_t len) { - size_t nlen = len; - - if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { - netdata_log_error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno)); - return 1; - } - if (unlikely(nlen != len)) { - netdata_log_error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); - return 1; - } - return 0; -} - -#endif - -// ===================================================================================================================== -// Linux - -#if __linux__ - -const char *os_type = "linux"; - -#endif diff --git a/src/libnetdata/os.h b/src/libnetdata/os.h deleted file mode 100644 index 0e4a369635de2d..00000000000000 --- a/src/libnetdata/os.h +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_OS_H -#define NETDATA_OS_H - -#include "libnetdata.h" - -// ===================================================================================================================== -// FreeBSD - -#if __FreeBSD__ - -#include - -#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var)) -int getsysctl_by_name(const char *name, void *ptr, size_t len); - -#define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int)) - -int getsysctl_mib(const char *name, int *mib, size_t len); - -#define GETSYSCTL_SIMPLE(name, mib, var) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), &(var), sizeof(var)) -#define GETSYSCTL_WSIZE(name, mib, var, size) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), var, size) - -int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len); - -#define GETSYSCTL_SIZE(name, mib, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), NULL, &(size)) -#define GETSYSCTL(name, mib, var, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), &(var), &(size)) - -int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len); - -#endif - -// ===================================================================================================================== -// MacOS - -#if __APPLE__ - -#include -#include "byteorder.h" - -#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var)) -int getsysctl_by_name(const char *name, void *ptr, size_t len); - -#endif - -// ===================================================================================================================== -// common defs for Apple/FreeBSD/Linux - -extern const char *os_type; - -#define get_system_cpus() get_system_cpus_with_cache(true, false) -#define get_system_cpus_uncached() get_system_cpus_with_cache(false, false) -long get_system_cpus_with_cache(bool cache, bool for_netdata); -unsigned long read_cpuset_cpus(const char *filename, long system_cpus); - -extern pid_t pid_max; -pid_t get_system_pid_max(void); - -extern unsigned int system_hz; -void get_system_HZ(void); - -#include -#if defined(__FreeBSD__) || defined(__APPLE__) -#define ADJUST_TIMEX(x) ntp_adjtime(x) -#else -#define ADJUST_TIMEX(x) adjtimex(x) -#endif - -#endif //NETDATA_OS_H diff --git a/src/libnetdata/os/adjtimex.c b/src/libnetdata/os/adjtimex.c new file mode 100644 index 00000000000000..d2f50eaf01e46d --- /dev/null +++ b/src/libnetdata/os/adjtimex.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +int os_adjtimex(struct timex *buf __maybe_unused) { +#if defined(COMPILED_FOR_MACOS) || defined(COMPILED_FOR_FREEBSD) + return ntp_adjtime(buf); +#endif + +#if defined(COMPILED_FOR_LINUX) + return adjtimex(buf); +#endif + + errno = ENOSYS; + return -1; +} diff --git a/src/libnetdata/os/adjtimex.h b/src/libnetdata/os/adjtimex.h new file mode 100644 index 00000000000000..0a48f8c7669062 --- /dev/null +++ b/src/libnetdata/os/adjtimex.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_ADJTIMEX_H +#define NETDATA_ADJTIMEX_H + +#if defined(COMPILED_FOR_LINUX) || defined(COMPILED_FOR_FREEBSD) || defined(COMPILED_FOR_MACOS) +#include +#endif + +struct timex; +int os_adjtimex(struct timex *buf); + +#endif //NETDATA_ADJTIMEX_H diff --git a/src/libnetdata/byteorder.h b/src/libnetdata/os/byteorder.h similarity index 100% rename from src/libnetdata/byteorder.h rename to src/libnetdata/os/byteorder.h diff --git a/src/libnetdata/os/get_pid_max.c b/src/libnetdata/os/get_pid_max.c new file mode 100644 index 00000000000000..c125b194c7f84b --- /dev/null +++ b/src/libnetdata/os/get_pid_max.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +pid_t pid_max = 32768; +pid_t os_get_system_pid_max(void) { +#if defined(COMPILED_FOR_MACOS) + + // As we currently do not know a solution to query pid_max from the os + // we use the number defined in bsd/sys/proc_internal.h in XNU sources + pid_max = 99999; + return pid_max; + +#elif defined(COMPILED_FOR_FREEBSD) + + int32_t tmp_pid_max; + + if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) { + pid_max = 99999; + netdata_log_error("Assuming system's maximum pid is %d.", pid_max); + } else { + pid_max = tmp_pid_max; + } + + return pid_max; + +#elif defined(COMPILED_FOR_LINUX) + + static char read = 0; + if(unlikely(read)) return pid_max; + read = 1; + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix?netdata_configured_host_prefix:""); + + unsigned long long max = 0; + if(read_single_number_file(filename, &max) != 0) { + netdata_log_error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max); + return pid_max; + } + + if(!max) { + netdata_log_error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max); + return pid_max; + } + + pid_max = (pid_t) max; + return pid_max; + +#else + + // just a big default + + pid_max = 4194304; + return pid_max; + +#endif +} diff --git a/src/libnetdata/os/get_pid_max.h b/src/libnetdata/os/get_pid_max.h new file mode 100644 index 00000000000000..a6da0a4dfe4234 --- /dev/null +++ b/src/libnetdata/os/get_pid_max.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_GET_PID_MAX_H +#define NETDATA_GET_PID_MAX_H + +#include + +extern pid_t pid_max; +pid_t os_get_system_pid_max(void); + +#endif //NETDATA_GET_PID_MAX_H diff --git a/src/libnetdata/os/get_system_cpus.c b/src/libnetdata/os/get_system_cpus.c new file mode 100644 index 00000000000000..f97d7ef177c78a --- /dev/null +++ b/src/libnetdata/os/get_system_cpus.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#if defined(COMPILED_FOR_WINDOWS) +#include +#endif + +#define CPUS_FOR_COLLECTORS 0 +#define CPUS_FOR_NETDATA 1 + +long os_get_system_cpus_cached(bool cache, bool for_netdata) { + static long processors[2] = { 0, 0 }; + + int index = for_netdata ? CPUS_FOR_NETDATA : CPUS_FOR_COLLECTORS; + + if(likely(cache && processors[index] > 0)) + return processors[index]; + +#if defined(COMPILED_FOR_FREEBSD) || defined(COMPILED_FOR_MACOS) +#if defined(COMPILED_FOR_MACOS) +#define HW_CPU_NAME "hw.logicalcpu" +#else +#define HW_CPU_NAME "hw.ncpu" +#endif + + int32_t tmp_processors; + bool error = false; + + if (unlikely(GETSYSCTL_BY_NAME(HW_CPU_NAME, tmp_processors))) + error = true; + else + processors[index] = tmp_processors; + + if(processors[index] < 1) { + processors[index] = 1; + + if(error) + netdata_log_error("Assuming system has %ld processors.", processors[index]); + } + + return processors[index]; +#elif defined(COMPILED_FOR_LINUX) + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/proc/stat", + (!for_netdata && netdata_configured_host_prefix) ? netdata_configured_host_prefix : ""); + + procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); + if(!ff) { + processors[index] = 1; + netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); + return processors[index]; + } + + ff = procfile_readall(ff); + if(!ff) { + processors[index] = 1; + netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); + return processors[index]; + } + + long tmp_processors = 0; + unsigned int i; + for(i = 0; i < procfile_lines(ff); i++) { + if(!procfile_linewords(ff, i)) continue; + + if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) + tmp_processors++; + } + procfile_close(ff); + + processors[index] = --tmp_processors; + + if(processors[index] < 1) + processors[index] = 1; + + netdata_log_debug(D_SYSTEM, "System has %ld processors.", processors[index]); + return processors[index]; + +#elif defined(COMPILED_FOR_WINDOWS) + + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + return (long) sysInfo.dwNumberOfProcessors; + +#else + + processors[index] = 1; + return processors[index]; + +#endif +} diff --git a/src/libnetdata/os/get_system_cpus.h b/src/libnetdata/os/get_system_cpus.h new file mode 100644 index 00000000000000..3c608df816242d --- /dev/null +++ b/src/libnetdata/os/get_system_cpus.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_GET_SYSTEM_CPUS_H +#define NETDATA_GET_SYSTEM_CPUS_H + +#include "../libnetdata.h" + +long os_get_system_cpus_cached(bool cache, bool for_netdata); + +#endif //NETDATA_GET_SYSTEM_CPUS_H diff --git a/src/libnetdata/os/getgrouplist.c b/src/libnetdata/os/getgrouplist.c new file mode 100644 index 00000000000000..0772860507599d --- /dev/null +++ b/src/libnetdata/os/getgrouplist.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +int os_getgrouplist(const char *username __maybe_unused, gid_t gid __maybe_unused, gid_t *supplementary_groups __maybe_unused, int *ngroups __maybe_unused) { +#if defined(COMPILED_FOR_LINUX) || defined(COMPILED_FOR_FREEBSD) + return getgrouplist(username, gid, supplementary_groups, ngroups); +#endif + +#if defined(COMPILED_FOR_MACOS) + return getgrouplist(username, gid, (int *)supplementary_groups, ngroups); +#endif + + errno = ENOSYS; + return -1; +} diff --git a/src/libnetdata/os/getgrouplist.h b/src/libnetdata/os/getgrouplist.h new file mode 100644 index 00000000000000..33646255320918 --- /dev/null +++ b/src/libnetdata/os/getgrouplist.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_GETGROUPLIST_H +#define NETDATA_GETGROUPLIST_H + +#include +int os_getgrouplist(const char *username, gid_t gid, gid_t *supplementary_groups, int *ngroups); + +#endif //NETDATA_GETGROUPLIST_H diff --git a/src/libnetdata/os/gettid.c b/src/libnetdata/os/gettid.c new file mode 100644 index 00000000000000..c179ba683ed470 --- /dev/null +++ b/src/libnetdata/os/gettid.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#if defined(COMPILED_FOR_WINDOWS) +#include +#endif + +pid_t os_gettid(void) { +#if defined(HAVE_GETTID) + return gettid(); +#elif defined(HAVE_PTHREAD_GETTHREADID_NP) + return (pid_t)pthread_getthreadid_np(); +#elif defined(HAVE_PTHREAD_THREADID_NP) + uint64_t curthreadid; + pthread_threadid_np(NULL, &curthreadid); + return curthreadid; +#elif defined(COMPILED_FOR_WINDOWS) + return (pid_t)GetCurrentThreadId(); +#elif defined(COMPILED_FOR_LINUX) + return (pid_t)syscall(SYS_gettid); +#else + return (pid_t)pthread_self(); +#endif +} + +static __thread pid_t gettid_cached_tid = 0; +pid_t gettid_cached(void) { + if(unlikely(gettid_cached_tid == 0)) + gettid_cached_tid = os_gettid(); + + return gettid_cached_tid; +} \ No newline at end of file diff --git a/src/libnetdata/os/gettid.h b/src/libnetdata/os/gettid.h new file mode 100644 index 00000000000000..f04d9c36574843 --- /dev/null +++ b/src/libnetdata/os/gettid.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_GETTID_H +#define NETDATA_GETTID_H + +#include + +pid_t os_gettid(void); +pid_t gettid_cached(void); + +#endif //NETDATA_GETTID_H diff --git a/src/libnetdata/os/os-freebsd-wrappers.c b/src/libnetdata/os/os-freebsd-wrappers.c new file mode 100644 index 00000000000000..b191a4ebbca726 --- /dev/null +++ b/src/libnetdata/os/os-freebsd-wrappers.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#if defined(COMPILED_FOR_FREEBSD) + +int getsysctl_by_name(const char *name, void *ptr, size_t len) { + size_t nlen = len; + + if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} + +int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len) { + size_t nlen = len; + + if (unlikely(!mib[0])) + if (unlikely(getsysctl_mib(name, mib, miblen))) + return 1; + + if (unlikely(sysctl(mib, miblen, ptr, &nlen, NULL, 0) == -1)) { + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + + return 0; +} + +int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len) { + size_t nlen = *len; + + if (unlikely(!mib[0])) + if (unlikely(getsysctl_mib(name, mib, miblen))) + return 1; + + if (unlikely(sysctl(mib, miblen, ptr, len, NULL, 0) == -1)) { + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(ptr != NULL && nlen != *len)) { + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen); + return 1; + } + + return 0; +} + +int getsysctl_mib(const char *name, int *mib, size_t len) { + size_t nlen = len; + + if (unlikely(sysctlnametomib(name, mib, &nlen) == -1)) { + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} + +#endif diff --git a/src/libnetdata/os/os-freebsd-wrappers.h b/src/libnetdata/os/os-freebsd-wrappers.h new file mode 100644 index 00000000000000..3a32f55f93f065 --- /dev/null +++ b/src/libnetdata/os/os-freebsd-wrappers.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_OS_FREEBSD_WRAPPERS_H +#define NETDATA_OS_FREEBSD_WRAPPERS_H + +#include "../libnetdata.h" + +#if defined(COMPILED_FOR_FREEBSD) +#include + +#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var)) +int getsysctl_by_name(const char *name, void *ptr, size_t len); + +#define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int)) + +int getsysctl_mib(const char *name, int *mib, size_t len); + +#define GETSYSCTL_SIMPLE(name, mib, var) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), &(var), sizeof(var)) +#define GETSYSCTL_WSIZE(name, mib, var, size) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), var, size) + +int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len); + +#define GETSYSCTL_SIZE(name, mib, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), NULL, &(size)) +#define GETSYSCTL(name, mib, var, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), &(var), &(size)) + +int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len); +#endif + +#endif //NETDATA_OS_FREEBSD_WRAPPERS_H diff --git a/src/libnetdata/os/os-macos-wrappers.c b/src/libnetdata/os/os-macos-wrappers.c new file mode 100644 index 00000000000000..c0d35556a83e09 --- /dev/null +++ b/src/libnetdata/os/os-macos-wrappers.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#if defined(COMPILED_FOR_MACOS) + +int getsysctl_by_name(const char *name, void *ptr, size_t len) { + size_t nlen = len; + + if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { + netdata_log_error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + netdata_log_error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} + +#endif diff --git a/src/libnetdata/os/os-macos-wrappers.h b/src/libnetdata/os/os-macos-wrappers.h new file mode 100644 index 00000000000000..2ca618a801c2bb --- /dev/null +++ b/src/libnetdata/os/os-macos-wrappers.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_OS_MACOS_WRAPPERS_H +#define NETDATA_OS_MACOS_WRAPPERS_H + +#include "../libnetdata.h" + +#if defined(COMPILED_FOR_MACOS) +#include +#include "byteorder.h" + +#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var)) +int getsysctl_by_name(const char *name, void *ptr, size_t len); + +#endif + +#endif //NETDATA_OS_MACOS_WRAPPERS_H diff --git a/src/libnetdata/os/os.c b/src/libnetdata/os/os.c new file mode 100644 index 00000000000000..c8904864e04eac --- /dev/null +++ b/src/libnetdata/os/os.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// system functions +// to retrieve settings of the system + +unsigned int system_hz; +void os_get_system_HZ(void) { + long ticks; + + if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { + netdata_log_error("Cannot get system clock ticks"); + } + + system_hz = (unsigned int) ticks; +} + +static inline unsigned long cpuset_str2ul(char **s) { + unsigned long n = 0; + char c; + for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +unsigned long os_read_cpuset_cpus(const char *filename, long system_cpus) { + static char *buf = NULL; + static size_t buf_size = 0; + + if(!buf) { + buf_size = 100U + 6 * system_cpus + 1; // taken from kernel/cgroup/cpuset.c + buf = mallocz(buf_size); + } + + int ret = read_txt_file(filename, buf, buf_size); + + if(!ret) { + char *s = buf; + unsigned long ncpus = 0; + + // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use + while (*s) { + if (isspace((uint8_t)*s)) { + s++; + continue; + } + unsigned long n = cpuset_str2ul(&s); + ncpus++; + if(*s == ',') { + s++; + continue; + } + if(*s == '-') { + s++; + unsigned long m = cpuset_str2ul(&s); + ncpus += m - n; // calculate the number of cpus in the region + } + s++; + } + + if(!ncpus) + return 0; + + return ncpus; + } + + return 0; +} + +// ===================================================================================================================== +// os_type + +#if defined(COMPILED_FOR_LINUX) +const char *os_type = "linux"; +#endif + +#if defined(COMPILED_FOR_FREEBSD) +const char *os_type = "freebsd"; +#endif + +#if defined(COMPILED_FOR_MACOS) +const char *os_type = "macos"; +#endif + +#if defined(COMPILED_FOR_WINDOWS) +const char *os_type = "windows"; +#endif + diff --git a/src/libnetdata/os/os.h b/src/libnetdata/os/os.h new file mode 100644 index 00000000000000..89e49e70172512 --- /dev/null +++ b/src/libnetdata/os/os.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_OS_H +#define NETDATA_OS_H + +#if defined(COMPILED_FOR_LINUX) || defined(COMPILED_FOR_FREEBSD) || defined(COMPILED_FOR_MACOS) +#include +#endif + +#include "setresuid.h" +#include "setresgid.h" +#include "getgrouplist.h" +#include "adjtimex.h" +#include "gettid.h" +#include "waitid.h" +#include "get_pid_max.h" +#include "get_system_cpus.h" +#include "tinysleep.h" +#include "uuid_generate.h" +#include "setenv.h" +#include "os-freebsd-wrappers.h" +#include "os-macos-wrappers.h" + +// ===================================================================================================================== +// common defs for Apple/FreeBSD/Linux + +extern const char *os_type; + +#define os_get_system_cpus() os_get_system_cpus_cached(true, false) +#define os_get_system_cpus_uncached() os_get_system_cpus_cached(false, false) +long os_get_system_cpus_cached(bool cache, bool for_netdata); +unsigned long os_read_cpuset_cpus(const char *filename, long system_cpus); + +extern unsigned int system_hz; +void os_get_system_HZ(void); + +#endif //NETDATA_OS_H diff --git a/src/libnetdata/os/setenv.c b/src/libnetdata/os/setenv.c new file mode 100644 index 00000000000000..5aa4302b8b6d81 --- /dev/null +++ b/src/libnetdata/os/setenv.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "config.h" + +#ifndef HAVE_SETENV + +#include +#include +#include + +int os_setenv(const char *name, const char *value, int overwrite) { + char *env_var; + int result; + + if (!overwrite) { + env_var = getenv(name); + if (env_var) return 0; // Already set + } + + size_t len = strlen(name) + strlen(value) + 2; // +2 for '=' and '\0' + env_var = malloc(len); + if (!env_var) return -1; // Allocation failure + snprintf(env_var, len, "%s=%s", name, value); + + result = putenv(env_var); + // free(env_var); // _putenv in Windows makes a copy of the string + return result; +} + +#endif diff --git a/src/libnetdata/os/setenv.h b/src/libnetdata/os/setenv.h new file mode 100644 index 00000000000000..3ed63714c424ae --- /dev/null +++ b/src/libnetdata/os/setenv.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SETENV_H +#define NETDATA_SETENV_H + +#include "config.h" + +#ifndef HAVE_SETENV +int os_setenv(const char *name, const char *value, int overwrite); +#define setenv(name, value, overwrite) os_setenv(name, value, overwrite) +#endif + +#endif //NETDATA_SETENV_H diff --git a/src/libnetdata/os/setresgid.c b/src/libnetdata/os/setresgid.c new file mode 100644 index 00000000000000..fcdad47a23f5ce --- /dev/null +++ b/src/libnetdata/os/setresgid.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +int os_setresgid(gid_t gid __maybe_unused, gid_t egid __maybe_unused, gid_t sgid __maybe_unused) { +#if defined(COMPILED_FOR_LINUX) || defined(COMPILED_FOR_FREEBSD) + return setresgid(gid, egid, sgid); +#endif + +#if defined(COMPILED_FOR_MACOS) + return setregid(gid, egid); +#endif + + errno = ENOSYS; + return -1; +} diff --git a/src/libnetdata/os/setresgid.h b/src/libnetdata/os/setresgid.h new file mode 100644 index 00000000000000..fc6d41f9510b1c --- /dev/null +++ b/src/libnetdata/os/setresgid.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SETRESGID_H +#define NETDATA_SETRESGID_H + +#include +int os_setresgid(gid_t gid, gid_t egid, gid_t sgid); + +#endif //NETDATA_SETRESGID_H diff --git a/src/libnetdata/os/setresuid.c b/src/libnetdata/os/setresuid.c new file mode 100644 index 00000000000000..8ab6e36b458db3 --- /dev/null +++ b/src/libnetdata/os/setresuid.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +int os_setresuid(uid_t uid __maybe_unused, uid_t euid __maybe_unused, uid_t suid __maybe_unused) { +#if defined(COMPILED_FOR_LINUX) || defined(COMPILED_FOR_FREEBSD) + return setresuid(uid, euid, suid); +#endif + +#if defined(COMPILED_FOR_MACOS) + return setreuid(uid, euid); +#endif + + errno = ENOSYS; + return -1; +} diff --git a/src/libnetdata/os/setresuid.h b/src/libnetdata/os/setresuid.h new file mode 100644 index 00000000000000..9f95d5d699b3ae --- /dev/null +++ b/src/libnetdata/os/setresuid.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SETRESUID_H +#define NETDATA_SETRESUID_H + +#include + +int os_setresuid(uid_t uid, uid_t euid, uid_t suid); + +#endif //NETDATA_SETRESUID_H diff --git a/src/libnetdata/os/strndup.c b/src/libnetdata/os/strndup.c new file mode 100644 index 00000000000000..17210f1245a491 --- /dev/null +++ b/src/libnetdata/os/strndup.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef HAVE_STRNDUP +#include "../libnetdata.h" + +static inline char *os_strndup( const char *s1, size_t n) +{ + char *copy= (char*)malloc( n+1 ); + memcpy( copy, s1, n ); + copy[n] = 0; + return copy; +}; +#endif diff --git a/src/libnetdata/os/strndup.h b/src/libnetdata/os/strndup.h new file mode 100644 index 00000000000000..9e51c8fd393ce5 --- /dev/null +++ b/src/libnetdata/os/strndup.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef STRNDUP_H +#define STRNDUP_H + +#include "config.h" + +#ifndef HAVE_STRNDUP +#define strndup(s, n) os_strndup(s, n) +#endif + +#endif //STRNDUP_H diff --git a/src/libnetdata/os/tinysleep.c b/src/libnetdata/os/tinysleep.c new file mode 100644 index 00000000000000..1c13753b760779 --- /dev/null +++ b/src/libnetdata/os/tinysleep.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#ifdef COMPILED_FOR_WINDOWS +#include + +void tinysleep(void) { + // Improve the system timer resolution to 1 ms + timeBeginPeriod(1); + + // Sleep for the desired duration + Sleep(1); + + // Reset the system timer resolution + timeEndPeriod(1); +} +#else +void tinysleep(void) { + static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; + nanosleep(&ns, NULL); +} +#endif diff --git a/src/libnetdata/os/tinysleep.h b/src/libnetdata/os/tinysleep.h new file mode 100644 index 00000000000000..480575a3a78ced --- /dev/null +++ b/src/libnetdata/os/tinysleep.h @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_TINYSLEEP_H +#define NETDATA_TINYSLEEP_H + +void tinysleep(void); + +#endif //NETDATA_TINYSLEEP_H diff --git a/src/libnetdata/os/uuid_generate.c b/src/libnetdata/os/uuid_generate.c new file mode 100644 index 00000000000000..04de8a608df963 --- /dev/null +++ b/src/libnetdata/os/uuid_generate.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" +#undef uuid_generate +#undef uuid_generate_random +#undef uuid_generate_time + +#ifdef COMPILED_FOR_WINDOWS +#include + +void os_uuid_generate(void *out) { + RPC_STATUS status = UuidCreate(out); + while (status != RPC_S_OK && status != RPC_S_UUID_LOCAL_ONLY) { + tinysleep(); + status = UuidCreate(out); + } +} + +void os_uuid_generate_random(void *out) { + os_uuid_generate(out); +} + +void os_uuid_generate_time(void *out) { + os_uuid_generate(out); +} + +#else + +#if !defined(COMPILED_FOR_MACOS) +#include +#endif + +void os_uuid_generate(void *out) { + uuid_generate(out); +} + +void os_uuid_generate_random(void *out) { + uuid_generate_random(out); +} + +void os_uuid_generate_time(void *out) { + uuid_generate_time(out); +} + +#endif diff --git a/src/libnetdata/os/uuid_generate.h b/src/libnetdata/os/uuid_generate.h new file mode 100644 index 00000000000000..95f07c79688e17 --- /dev/null +++ b/src/libnetdata/os/uuid_generate.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_UUID_GENERATE_H +#define NETDATA_UUID_GENERATE_H + +void os_uuid_generate(void *out); +void os_uuid_generate_random(void *out); +void os_uuid_generate_time(void *out); + +#endif //NETDATA_UUID_GENERATE_H diff --git a/src/libnetdata/os/waitid.c b/src/libnetdata/os/waitid.c new file mode 100644 index 00000000000000..b78d704ed0ec0d --- /dev/null +++ b/src/libnetdata/os/waitid.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +int os_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) { +#if defined(HAVE_WAITID) + return waitid(idtype, id, infop, options); +#else + // emulate waitid() using waitpid() + + // a cache for WNOWAIT + static const struct pid_status empty = { 0, 0 }; + static __thread struct pid_status last = { 0, 0 }; // the cache + struct pid_status current = { 0, 0 }; + + // zero the infop structure + memset(infop, 0, sizeof(*infop)); + + // from the infop structure we use only 3 fields: + // - si_pid + // - si_code + // - si_status + // so, we update only these 3 + + switch(idtype) { + case P_ALL: + current.pid = waitpid((pid_t)-1, ¤t.status, options); + if(options & WNOWAIT) + last = current; + else + last = empty; + break; + + case P_PID: + if(last.pid == (pid_t)id) { + current = last; + last = empty; + } + else + current.pid = waitpid((pid_t)id, ¤t.status, options); + + break; + + default: + errno = ENOSYS; + return -1; + } + + if (current.pid > 0) { + if (WIFEXITED(current.status)) { + infop->si_code = CLD_EXITED; + infop->si_status = WEXITSTATUS(current.status); + } else if (WIFSIGNALED(current.status)) { + infop->si_code = WTERMSIG(current.status) == SIGABRT ? CLD_DUMPED : CLD_KILLED; + infop->si_status = WTERMSIG(current.status); + } else if (WIFSTOPPED(current.status)) { + infop->si_code = CLD_STOPPED; + infop->si_status = WSTOPSIG(current.status); + } else if (WIFCONTINUED(current.status)) { + infop->si_code = CLD_CONTINUED; + infop->si_status = SIGCONT; + } + infop->si_pid = current.pid; + return 0; + } else if (current.pid == 0) { + // No change in state, depends on WNOHANG + return 0; + } + + return -1; +#endif +} diff --git a/src/libnetdata/os/waitid.h b/src/libnetdata/os/waitid.h new file mode 100644 index 00000000000000..350575e1501965 --- /dev/null +++ b/src/libnetdata/os/waitid.h @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_WAITID_H +#define NETDATA_WAITID_H + +#include "config.h" +#include +#include + +#ifdef HAVE_SYS_WAIT_H +#include +#endif + +#ifndef WNOWAIT +#define WNOWAIT 0x01000000 +#endif + +#ifndef WEXITED +#define WEXITED 4 +#endif + +#if !defined(HAVE_WAITID) +typedef enum +{ + P_ALL, /* Wait for any child. */ + P_PID, /* Wait for specified process. */ + P_PGID, /* Wait for members of process group. */ + P_PIDFD, /* Wait for the child referred by the PID file descriptor. */ +} idtype_t; + +struct pid_status { + pid_t pid; + int status; +}; + +#if defined(COMPILED_FOR_WINDOWS) && !defined(__CYGWIN__) +typedef uint32_t id_t; +typedef struct { + int si_code; /* Signal code. */ + int si_status; /* Exit value or signal. */ + pid_t si_pid; /* Sending process ID. */ +} siginfo_t; +#endif +#endif + +int os_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options); + +#endif //NETDATA_WAITID_H diff --git a/src/libnetdata/popen/popen.c b/src/libnetdata/popen/popen.c index 5f8bd2b4a66bd1..c1721e9b431de5 100644 --- a/src/libnetdata/popen/popen.c +++ b/src/libnetdata/popen/popen.c @@ -98,7 +98,7 @@ int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) { } else { // we haven't reaped this child yet - ret = waitid(idtype, id, infop, options); + ret = os_waitid(idtype, id, infop, options); if(mp && !mp->reaped) { mp->reaped = true; diff --git a/src/libnetdata/popen/popen.h b/src/libnetdata/popen/popen.h index 4f86158bcbe111..8f46abbc8cf3f8 100644 --- a/src/libnetdata/popen/popen.h +++ b/src/libnetdata/popen/popen.h @@ -3,6 +3,9 @@ #ifndef NETDATA_POPEN_H #define NETDATA_POPEN_H 1 +#include "../os/waitid.h" +int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options); + #include "../libnetdata.h" #define PIPE_READ 0 @@ -28,6 +31,5 @@ int netdata_popene_variadic_internal_dont_use_directly(volatile pid_t *pidptr, c int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid); int netdata_spawn(const char *command, volatile pid_t *pidptr); -int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options); #endif /* NETDATA_POPEN_H */ diff --git a/src/libnetdata/query_progress/progress.c b/src/libnetdata/query_progress/progress.c index 4ddb45135077ec..10e083e0cc09ec 100644 --- a/src/libnetdata/query_progress/progress.c +++ b/src/libnetdata/query_progress/progress.c @@ -12,7 +12,7 @@ struct query; #define SIMPLE_HASHTABLE_VALUE_TYPE struct query -#define SIMPLE_HASHTABLE_KEY_TYPE uuid_t +#define SIMPLE_HASHTABLE_KEY_TYPE nd_uuid_t #define SIMPLE_HASHTABLE_NAME _QUERY #define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION query_transaction #define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION query_compare_keys @@ -21,7 +21,7 @@ struct query; // ---------------------------------------------------------------------------- typedef struct query { - uuid_t transaction; + nd_uuid_t transaction; BUFFER *query; BUFFER *payload; @@ -48,12 +48,12 @@ typedef struct query { struct query *prev, *next; } QUERY_PROGRESS; -static inline uuid_t *query_transaction(QUERY_PROGRESS *qp) { +static inline nd_uuid_t *query_transaction(QUERY_PROGRESS *qp) { return qp ? &qp->transaction : NULL; } -static inline bool query_compare_keys(uuid_t *t1, uuid_t *t2) { - if(t1 == t2 || (t1 && t2 && memcmp(t1, t2, sizeof(uuid_t)) == 0)) +static inline bool query_compare_keys(nd_uuid_t *t1, nd_uuid_t *t2) { + if(t1 == t2 || (t1 && t2 && memcmp(t1, t2, sizeof(nd_uuid_t)) == 0)) return true; return false; @@ -75,7 +75,7 @@ static struct progress { .spinlock = NETDATA_SPINLOCK_INITIALIZER, }; -SIMPLE_HASHTABLE_HASH query_hash(uuid_t *transaction) { +SIMPLE_HASHTABLE_HASH query_hash(nd_uuid_t *transaction) { struct uuid_hi_lo_t { uint64_t hi; uint64_t lo; @@ -93,7 +93,7 @@ static void query_progress_init_unsafe(void) { // ---------------------------------------------------------------------------- -static inline QUERY_PROGRESS *query_progress_find_in_hashtable_unsafe(uuid_t *transaction) { +static inline QUERY_PROGRESS *query_progress_find_in_hashtable_unsafe(nd_uuid_t *transaction) { SIMPLE_HASHTABLE_HASH hash = query_hash(transaction); SIMPLE_HASHTABLE_SLOT_QUERY *slot = simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, transaction, true); QUERY_PROGRESS *qp = SIMPLE_HASHTABLE_SLOT_DATA(slot); @@ -136,7 +136,7 @@ static inline void query_progress_remove_from_hashtable_unsafe(QUERY_PROGRESS *q // ---------------------------------------------------------------------------- -static QUERY_PROGRESS *query_progress_alloc(uuid_t *transaction) { +static QUERY_PROGRESS *query_progress_alloc(nd_uuid_t *transaction) { QUERY_PROGRESS *qp; qp = callocz(1, sizeof(*qp)); uuid_copy(qp->transaction, *transaction); @@ -155,7 +155,7 @@ static void query_progress_free(QUERY_PROGRESS *qp) { freez(qp); } -static void query_progress_cleanup_to_reuse(QUERY_PROGRESS *qp, uuid_t *transaction) { +static void query_progress_cleanup_to_reuse(QUERY_PROGRESS *qp, nd_uuid_t *transaction) { assert(qp && qp->prev == NULL && qp->next == NULL); assert(!transaction || !qp->indexed); @@ -210,7 +210,7 @@ static inline void query_progress_unlink_from_cache_unsafe(QUERY_PROGRESS *qp) { // ---------------------------------------------------------------------------- // Progress API -void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client) { +void query_progress_start_or_update(nd_uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client) { if(!transaction) return; @@ -246,7 +246,7 @@ void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP spinlock_unlock(&progress.spinlock); } -void query_progress_set_finish_line(uuid_t *transaction, size_t all) { +void query_progress_set_finish_line(nd_uuid_t *transaction, size_t all) { if(!transaction) return; @@ -264,7 +264,7 @@ void query_progress_set_finish_line(uuid_t *transaction, size_t all) { spinlock_unlock(&progress.spinlock); } -void query_progress_done_step(uuid_t *transaction, size_t done) { +void query_progress_done_step(nd_uuid_t *transaction, size_t done) { if(!transaction) return; @@ -280,7 +280,7 @@ void query_progress_done_step(uuid_t *transaction, size_t done) { spinlock_unlock(&progress.spinlock); } -void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size) { +void query_progress_finished(nd_uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size) { if(!transaction) return; @@ -319,7 +319,7 @@ void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int } } -void query_progress_functions_update(uuid_t *transaction, size_t done, size_t all) { +void query_progress_functions_update(nd_uuid_t *transaction, size_t done, size_t all) { // functions send to the total 'done', not the increment if(!transaction) @@ -346,7 +346,7 @@ void query_progress_functions_update(uuid_t *transaction, size_t done, size_t al // ---------------------------------------------------------------------------- // /api/v2/progress - to get the progress of a transaction -int web_api_v2_report_progress(uuid_t *transaction, BUFFER *wb) { +int web_api_v2_report_progress(nd_uuid_t *transaction, BUFFER *wb) { buffer_flush(wb); buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); @@ -622,7 +622,7 @@ int progress_function_result(BUFFER *wb, const char *hostname) { int progress_unittest(void) { size_t permanent = 100; - uuid_t valid[permanent]; + nd_uuid_t valid[permanent]; usec_t started = now_monotonic_usec(); @@ -632,7 +632,7 @@ int progress_unittest(void) { } for(size_t n = 0; n < 5000000 ;n++) { - uuid_t t; + nd_uuid_t t; uuid_generate_random(t); query_progress_start_or_update(&t, 0, HTTP_REQUEST_MODE_OPTIONS, HTTP_ACL_WEBRTC, "ephemeral", NULL, "test"); query_progress_finished(&t, 0, 200, 1234, 123, 12); diff --git a/src/libnetdata/query_progress/progress.h b/src/libnetdata/query_progress/progress.h index 1adb8d2ba2ea1f..d45735dd28c9ad 100644 --- a/src/libnetdata/query_progress/progress.h +++ b/src/libnetdata/query_progress/progress.h @@ -5,13 +5,13 @@ #include "../libnetdata.h" -void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client); -void query_progress_done_step(uuid_t *transaction, size_t done); -void query_progress_set_finish_line(uuid_t *transaction, size_t all); -void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size); -void query_progress_functions_update(uuid_t *transaction, size_t done, size_t all); +void query_progress_start_or_update(nd_uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client); +void query_progress_done_step(nd_uuid_t *transaction, size_t done); +void query_progress_set_finish_line(nd_uuid_t *transaction, size_t all); +void query_progress_finished(nd_uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size); +void query_progress_functions_update(nd_uuid_t *transaction, size_t done, size_t all); -int web_api_v2_report_progress(uuid_t *transaction, BUFFER *wb); +int web_api_v2_report_progress(nd_uuid_t *transaction, BUFFER *wb); #define RRDFUNCTIONS_PROGRESS_HELP "View the progress on the running and latest Netdata API Requests" int progress_function_result(BUFFER *wb, const char *hostname); diff --git a/src/libnetdata/socket/security.c b/src/libnetdata/socket/security.c index 4deb766239a2d5..502998b79ffa56 100644 --- a/src/libnetdata/socket/security.c +++ b/src/libnetdata/socket/security.c @@ -233,6 +233,19 @@ static inline bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) { * (These are often the same value, but can be different on some systems.) */ +ssize_t netdata_ssl_pending(NETDATA_SSL *ssl) { + return SSL_pending(ssl->conn); +} + +bool netdata_ssl_has_pending(NETDATA_SSL *ssl) { + // this call was added on OpenSSL 1.1.0 + // however, it is more accurate than SSL_pending() + // unfortunately it does not exists in libressl. + // return SSL_has_pending(ssl->conn); + + return SSL_pending(ssl->conn) > 0; +} + ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num) { errno = 0; ssl->ssl_errno = 0; diff --git a/src/libnetdata/socket/security.h b/src/libnetdata/socket/security.h index fd17b6f3f28dad..283d81db856b80 100644 --- a/src/libnetdata/socket/security.h +++ b/src/libnetdata/socket/security.h @@ -39,7 +39,7 @@ typedef struct netdata_ssl { unsigned long ssl_errno; // The SSL errno of the last SSL call } NETDATA_SSL; -#define NETDATA_SSL_UNSET_CONNECTION (NETDATA_SSL){ .conn = NULL, .state = NETDATA_SSL_STATE_NOT_SSL } +#define NETDATA_SSL_UNSET_CONNECTION (NETDATA_SSL){ .conn = NULL, .state = NETDATA_SSL_STATE_NOT_SSL, .ssl_errno = 0 } #define SSL_connection(ssl) ((ssl)->conn && (ssl)->state != NETDATA_SSL_STATE_NOT_SSL) @@ -70,5 +70,8 @@ void netdata_ssl_close(NETDATA_SSL *ssl); ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num); ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num); +ssize_t netdata_ssl_pending(NETDATA_SSL *ssl); +bool netdata_ssl_has_pending(NETDATA_SSL *ssl); + # endif //ENABLE_HTTPS #endif //NETDATA_SECURITY_H diff --git a/src/libnetdata/socket/socket.c b/src/libnetdata/socket/socket.c index b157b157bd8a0d..0ba24b7474019b 100644 --- a/src/libnetdata/socket/socket.c +++ b/src/libnetdata/socket/socket.c @@ -201,7 +201,7 @@ void sock_setcloexec(int fd) #endif } -int sock_setreuse_port(int fd, int reuse) { +int sock_setreuse_port(int fd __maybe_unused, int reuse __maybe_unused) { int ret; #ifdef SO_REUSEPORT @@ -757,10 +757,10 @@ int listen_sockets_setup(LISTEN_SOCKETS *sockets) { char *e = s; // skip separators, moving both s(tart) and e(nd) - while(isspace(*e) || *e == ',') s = ++e; + while(isspace((uint8_t)*e) || *e == ',') s = ++e; // move e(nd) to the first separator - while(*e && !isspace(*e) && *e != ',') e++; + while(*e && !isspace((uint8_t)*e) && *e != ',') e++; // is there anything? if(!*s || s == e) break; @@ -872,6 +872,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t int fd = -1; for (ai = ai_head; ai != NULL && fd == -1; ai = ai->ai_next) { + if(nd_thread_signaled_to_cancel()) break; if (ai->ai_family == PF_INET6) { struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; @@ -925,53 +926,46 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t hostBfr, servBfr); // Convert 'struct timeval' to milliseconds for poll(): - int timeout_milliseconds = timeout->tv_sec * 1000 + timeout->tv_usec / 1000; - - struct pollfd fds[1]; - fds[0].fd = fd; - fds[0].events = POLLOUT; // We are looking for the ability to write to the socket - - int ret = poll(fds, 1, timeout_milliseconds); - if (ret > 0) { - // poll() completed normally. We can check the revents to see what happened - if (fds[0].revents & POLLOUT) { - // connect() completed successfully, socket is writable. + int timeout_ms = timeout->tv_sec * 1000 + timeout->tv_usec / 1000; + switch(wait_on_socket_or_cancel_with_timeout( +#ifdef ENABLE_HTTPS + NULL, +#endif + fd, timeout_ms, POLLOUT, NULL)) { + case 0: // proceed nd_log(NDLS_DAEMON, NDLP_DEBUG, "connect() to ip %s port %s completed successfully", hostBfr, servBfr); + break; - } - else { - // This means that the socket is in error. We will close it and set fd to -1 - + case -1: // thread cancelled nd_log(NDLS_DAEMON, NDLP_ERR, - "Failed to connect to '%s', port '%s'.", + "Thread is cancelled while connecting to '%s', port '%s'.", hostBfr, servBfr); close(fd); fd = -1; - } - } - else if (ret == 0) { - // poll() timed out, the connection is not established within the specified timeout. - errno = 0; + break; - nd_log(NDLS_DAEMON, NDLP_ERR, - "Timed out while connecting to '%s', port '%s'.", - hostBfr, servBfr); + case 1: // timeout + nd_log(NDLS_DAEMON, NDLP_ERR, + "Timed out while connecting to '%s', port '%s'.", + hostBfr, servBfr); - close(fd); - fd = -1; - } - else { // ret < 0 - // poll() returned an error. - nd_log(NDLS_DAEMON, NDLP_ERR, - "Failed to connect to '%s', port '%s'. poll() returned %d", - hostBfr, servBfr, ret); + close(fd); + fd = -1; + break; + + default: + case 2: // error + nd_log(NDLS_DAEMON, NDLP_ERR, + "Failed to connect to '%s', port '%s'.", + hostBfr, servBfr); - close(fd); - fd = -1; + close(fd); + fd = -1; + break; } } else { @@ -1091,10 +1085,10 @@ void foreach_entry_in_connection_string(const char *destination, bool (*callback const char *e = s; // skip separators, moving both s(tart) and e(nd) - while(isspace(*e) || *e == ',') s = ++e; + while(isspace((uint8_t)*e) || *e == ',') s = ++e; // move e(nd) to the first separator - while(*e && !isspace(*e) && *e != ',') e++; + while(*e && !isspace((uint8_t)*e) && *e != ',') e++; // is there anything? if(!*s || s == e) break; @@ -1177,38 +1171,91 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim // -------------------------------------------------------------------------------------------------------------------- // helpers to send/receive data in one call, in blocking mode, with a timeout +// returns: -1 = thread cancelled, 0 = proceed to read/write, 1 = time exceeded, 2 = error on fd +// timeout parameter can be zero to wait forever +inline int wait_on_socket_or_cancel_with_timeout( #ifdef ENABLE_HTTPS -ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { -#else -ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { + NETDATA_SSL *ssl, #endif + int fd, int timeout_ms, short int poll_events, short int *revents) { + struct pollfd pfd = { + .fd = fd, + .events = poll_events, + .revents = 0, + }; - for(;;) { - struct pollfd fd = { - .fd = sockfd, - .events = POLLIN, - .revents = 0 - }; + bool forever = (timeout_ms == 0); + + while (timeout_ms > 0 || forever) { + if(nd_thread_signaled_to_cancel()) { + errno = ECANCELED; + return -1; + } + +#ifdef ENABLE_HTTPS + if(poll_events == POLLIN && ssl && SSL_connection(ssl) && netdata_ssl_has_pending(ssl)) + return 0; +#endif + + const int wait_ms = (timeout_ms >= ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS || forever) ? + ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS : timeout_ms; errno = 0; - int retval = poll(&fd, 1, timeout * 1000); - if(retval == -1) { - // failed + // check every wait_ms + const int ret = poll(&pfd, 1, wait_ms); + + if(revents) + *revents = pfd.revents; + + if(ret == -1) { + // poll failed if(errno == EINTR || errno == EAGAIN) continue; - return -1; + return 2; } - if(!retval) { + if(ret == 0) { // timeout - return 0; + if(!forever) + timeout_ms -= wait_ms; + continue; } - if(fd.revents & POLLIN) + if(pfd.revents & poll_events) + return 0; + + // all other errors + return 2; + } + + errno = ETIMEDOUT; + return 1; +} + +ssize_t recv_timeout( +#ifdef ENABLE_HTTPS + NETDATA_SSL *ssl, +#endif + int sockfd, void *buf, size_t len, int flags, int timeout) { + + switch(wait_on_socket_or_cancel_with_timeout( +#ifdef ENABLE_HTTPS + ssl, +#endif + sockfd, timeout * 1000, POLLIN, NULL)) { + case 0: // data are waiting break; + + case 1: // timeout + return 0; + + default: + case -1: // thread cancelled + case 2: // error on socket + return -1; } #ifdef ENABLE_HTTPS @@ -1220,37 +1267,27 @@ ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) return recv(sockfd, buf, len, flags); } +ssize_t send_timeout( #ifdef ENABLE_HTTPS -ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { -#else -ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { + NETDATA_SSL *ssl, #endif + int sockfd, void *buf, size_t len, int flags, int timeout) { - for(;;) { - struct pollfd fd = { - .fd = sockfd, - .events = POLLOUT, - .revents = 0 - }; - - errno = 0; - int retval = poll(&fd, 1, timeout * 1000); - - if(retval == -1) { - // failed - - if(errno == EINTR || errno == EAGAIN) - continue; - - return -1; - } + switch(wait_on_socket_or_cancel_with_timeout( +#ifdef ENABLE_HTTPS + ssl, +#endif + sockfd, timeout * 1000, POLLOUT, NULL)) { + case 0: // data are waiting + break; - if(!retval) { - // timeout + case 1: // timeout return 0; - } - if(fd.revents & POLLOUT) break; + default: + case -1: // thread cancelled + case 2: // error on socket + return -1; } #ifdef ENABLE_HTTPS @@ -1560,7 +1597,6 @@ inline POLLINFO *poll_add_fd(POLLJOB *p pi->recv_count = 0; pi->send_count = 0; - netdata_thread_disable_cancelability(); p->used++; if(unlikely(pi->slot > p->max)) p->max = pi->slot; @@ -1572,7 +1608,6 @@ inline POLLINFO *poll_add_fd(POLLJOB *p if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) { p->min = pi->slot; } - netdata_thread_enable_cancelability(); return pi; } @@ -1584,8 +1619,6 @@ inline void poll_close_fd(POLLINFO *pi) { if(unlikely(pf->fd == -1)) return; - netdata_thread_disable_cancelability(); - if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { pi->del_callback(pi); @@ -1633,7 +1666,6 @@ inline void poll_close_fd(POLLINFO *pi) { } } } - netdata_thread_enable_cancelability(); } void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data) { @@ -1692,11 +1724,11 @@ void poll_default_tmr_callback(void *timer_data) { (void)timer_data; } -static void poll_events_cleanup(void *data) { - POLLJOB *p = (POLLJOB *)data; +static void poll_events_cleanup(void *pptr) { + POLLJOB *p = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!p) return; - size_t i; - for(i = 0 ; i <= p->max ; i++) { + for(size_t i = 0 ; i <= p->max ; i++) { POLLINFO *pi = &p->inf[i]; poll_close_fd(pi); } @@ -1933,7 +1965,6 @@ void poll_events(LISTEN_SOCKETS *sockets int listen_sockets_active = 1; - int timeout_ms = 1000; // in milliseconds time_t last_check = now_boottime_sec(); usec_t timer_usec = timer_milliseconds * USEC_PER_MS; @@ -1945,9 +1976,9 @@ void poll_events(LISTEN_SOCKETS *sockets next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec; } - netdata_thread_cleanup_push(poll_events_cleanup, &p); + CLEANUP_FUNCTION_REGISTER(poll_events_cleanup) cleanup_ptr = &p; - while(!check_to_stop_callback()) { + while(!check_to_stop_callback() && !nd_thread_signaled_to_cancel()) { if(unlikely(timer_usec)) { now_usec = now_boottime_usec(); @@ -1957,12 +1988,6 @@ void poll_events(LISTEN_SOCKETS *sockets now_usec = now_boottime_usec(); next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec; } - - usec_t dt_usec = next_timer_usec - now_usec; - if(dt_usec < 1000 * USEC_PER_MS) - timeout_ms = 1000; - else - timeout_ms = (int)(dt_usec / USEC_PER_MS); } // enable or disable the TCP listening sockets, based on the current number of sockets used and the limit set @@ -1980,7 +2005,7 @@ void poll_events(LISTEN_SOCKETS *sockets } } - retval = poll(p.fds, p.max + 1, timeout_ms); + retval = poll(p.fds, p.max + 1, ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS); time_t now = now_boottime_sec(); if(unlikely(retval == -1)) { @@ -2151,6 +2176,4 @@ void poll_events(LISTEN_SOCKETS *sockets } } } - - netdata_thread_cleanup_pop(1); } diff --git a/src/libnetdata/socket/socket.h b/src/libnetdata/socket/socket.h index d506f7aae19be1..8eab8bfdd5d4c7 100644 --- a/src/libnetdata/socket/socket.h +++ b/src/libnetdata/socket/socket.h @@ -9,6 +9,8 @@ #define MAX_LISTEN_FDS 50 #endif +#define ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS 100 + typedef struct listen_sockets { struct config *config; // the config file to use const char *config_section; // the netdata configuration section to read settings from @@ -40,9 +42,11 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim #ifdef ENABLE_HTTPS ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); +int wait_on_socket_or_cancel_with_timeout(NETDATA_SSL *ssl, int fd, int timeout_ms, short int poll_events, short int *revents); #else ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); +int wait_on_socket_or_cancel_with_timeout(int fd, int timeout_ms, short int poll_events, short int *revents); #endif bool fd_is_socket(int fd); diff --git a/src/libnetdata/string/string.c b/src/libnetdata/string/string.c index 0b4a6470d7c5c6..94c11f4b9a2eb8 100644 --- a/src/libnetdata/string/string.c +++ b/src/libnetdata/string/string.c @@ -661,21 +661,18 @@ int string_unittest(size_t entries) { threads_to_create, (long long)seconds_to_run); // check string concurrency - netdata_thread_t threads[threads_to_create]; + ND_THREAD *threads[threads_to_create]; tu.join = 0; for (int i = 0; i < threads_to_create; i++) { char buf[100 + 1]; snprintf(buf, 100, "string%d", i); - netdata_thread_create( - &threads[i], buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, &tu); + threads[i] = nd_thread_create(buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, &tu); } sleep_usec(seconds_to_run * USEC_PER_SEC); __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED); - for (int i = 0; i < threads_to_create; i++) { - void *retval; - netdata_thread_join(threads[i], &retval); - } + for (int i = 0; i < threads_to_create; i++) + nd_thread_join(threads[i]); size_t inserts, deletes, searches, sentries, references, memory, duplications, releases; string_statistics(&inserts, &deletes, &searches, &sentries, &references, &memory, &duplications, &releases); diff --git a/src/libnetdata/threads/threads.c b/src/libnetdata/threads/threads.c index 94761671add3c0..0e12d173ec67b5 100644 --- a/src/libnetdata/threads/threads.c +++ b/src/libnetdata/threads/threads.c @@ -2,135 +2,202 @@ #include "../libnetdata.h" -static pthread_attr_t *netdata_threads_attr = NULL; +#define nd_thread_status_get(nti) __atomic_load_n(&((nti)->options), __ATOMIC_ACQUIRE) +#define nd_thread_status_check(nti, flag) (__atomic_load_n(&((nti)->options), __ATOMIC_ACQUIRE) & (flag)) +#define nd_thread_status_set(nti, flag) __atomic_or_fetch(&((nti)->options), flag, __ATOMIC_RELEASE) +#define nd_thread_status_clear(nti, flag) __atomic_and_fetch(&((nti)->options), ~(flag), __ATOMIC_RELEASE) -// ---------------------------------------------------------------------------- -// per thread data +typedef void (*nd_thread_canceller)(void *data); -typedef struct { +struct nd_thread { void *arg; - char tag[NETDATA_THREAD_NAME_MAX + 1]; + pid_t tid; + char tag[ND_THREAD_TAG_MAX + 1]; + void *ret; // the return value of start routine void *(*start_routine) (void *); NETDATA_THREAD_OPTIONS options; -} NETDATA_THREAD; + pthread_t thread; + bool cancel_atomic; -static __thread NETDATA_THREAD *netdata_thread = NULL; +#ifdef NETDATA_INTERNAL_CHECKS + // keep track of the locks currently held + // used to detect locks that are left locked during exit + int rwlocks_read_locks; + int rwlocks_write_locks; + int mutex_locks; + int spinlock_locks; + int rwspinlock_read_locks; + int rwspinlock_write_locks; +#endif -inline int netdata_thread_tag_exists(void) { - return (netdata_thread && *netdata_thread->tag); + struct { + SPINLOCK spinlock; + nd_thread_canceller cb; + void *data; + } canceller; + + struct nd_thread *prev, *next; +}; + +static struct { + struct { + SPINLOCK spinlock; + ND_THREAD *list; + } exited; + + struct { + SPINLOCK spinlock; + ND_THREAD *list; + } running; + + pthread_attr_t *attr; +} threads_globals = { + .exited = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .list = NULL, + }, + .running = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .list = NULL, + }, + .attr = NULL, +}; + +static __thread ND_THREAD *_nd_thread_info = NULL; +static __thread char _nd_thread_os_name[ND_THREAD_TAG_MAX + 1] = ""; + +// -------------------------------------------------------------------------------------------------------------------- +// O/S abstraction + +// get the thread name from the operating system +static inline void os_get_thread_name(char *out, size_t size) { +#if defined(__FreeBSD__) + pthread_get_name_np(pthread_self(), out, size); + if(strcmp(_nd_thread_os_name, "netdata") == 0) + strncpyz(out, "MAIN", size - 1); +#elif defined(HAVE_PTHREAD_GETNAME_NP) + pthread_getname_np(pthread_self(), out, size - 1); + if(strcmp(out, "netdata") == 0) + strncpyz(out, "MAIN", size - 1); +#else + strncpyz(out, "MAIN", size - 1); +#endif } -static const char *thread_name_get(bool recheck) { - static __thread char threadname[NETDATA_THREAD_NAME_MAX + 1] = ""; - - if(netdata_thread_tag_exists()) - strncpyz(threadname, netdata_thread->tag, NETDATA_THREAD_NAME_MAX); - else { - if(!recheck && threadname[0]) - return threadname; - +// set the thread name to the operating system +static inline void os_set_thread_name(const char *name) { #if defined(__FreeBSD__) - pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1); - if(strcmp(threadname, "netdata") == 0) - strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX); + pthread_set_name_np(pthread_self(), name); #elif defined(__APPLE__) - strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX); -#elif defined(HAVE_PTHREAD_GETNAME_NP) - pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1); - if(strcmp(threadname, "netdata") == 0) - strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX); + pthread_setname_np(name); #else - strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX); + pthread_setname_np(pthread_self(), name); #endif - } - - return threadname; } -const char *netdata_thread_tag(void) { - return thread_name_get(false); +// -------------------------------------------------------------------------------------------------------------------- +// internal API for managing names + +inline int nd_thread_has_tag(void) { + return (_nd_thread_info && _nd_thread_info->tag[0]); } -static size_t webrtc_id = 0; -static __thread bool webrtc_name_set = false; -void webrtc_set_thread_name(void) { - if(!netdata_thread && !webrtc_name_set) { - webrtc_name_set = true; - char threadname[NETDATA_THREAD_NAME_MAX + 1]; +// For threads created by netdata, return the tag of the thread. +// For threads created by others (libuv, webrtc, etc), return the tag of the operating system. +// This caches the response, so that it won't query the operating system multiple times. +static inline const char *nd_thread_get_name(bool recheck) { + if(nd_thread_has_tag()) + return _nd_thread_info->tag; -#if defined(__FreeBSD__) - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED)); - pthread_set_name_np(pthread_self(), threadname); -#elif defined(__APPLE__) - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED)); - pthread_setname_np(threadname); -#elif defined(HAVE_PTHREAD_GETNAME_NP) - pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX+1); - if(strcmp(threadname, "netdata") == 0) { - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED)); - pthread_setname_np(pthread_self(), threadname); - } -#else - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED)); - pthread_setname_np(pthread_self(), threadname); -#endif + if(!recheck && _nd_thread_os_name[0]) + return _nd_thread_os_name; - thread_name_get(true); - } + os_get_thread_name(_nd_thread_os_name, sizeof(_nd_thread_os_name)); + + return _nd_thread_os_name; } -// ---------------------------------------------------------------------------- -// compatibility library functions +const char *nd_thread_tag(void) { + return nd_thread_get_name(false); +} -static __thread pid_t gettid_cached_tid = 0; -pid_t gettid(void) { - pid_t tid = 0; +void nd_thread_tag_set(const char *tag) { + if(!tag || !*tag) return; - if(likely(gettid_cached_tid > 0)) - return gettid_cached_tid; + if(_nd_thread_info) + strncpyz(_nd_thread_info->tag, tag, sizeof(_nd_thread_info->tag) - 1); -#ifdef __FreeBSD__ + strncpyz(_nd_thread_os_name, tag, sizeof(_nd_thread_os_name) - 1); - tid = (pid_t)pthread_getthreadid_np(); + os_set_thread_name(_nd_thread_os_name); +} -#elif defined(__APPLE__) +// -------------------------------------------------------------------------------------------------------------------- + +static __thread bool libuv_name_set = false; +void uv_thread_set_name_np(const char* name) { + if(libuv_name_set) return; + + strncpyz(_nd_thread_os_name, name, sizeof(_nd_thread_os_name) - 1); + os_set_thread_name(_nd_thread_os_name); + libuv_name_set = true; +} - #if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060) - uint64_t curthreadid; - pthread_threadid_np(NULL, &curthreadid); - tid = (pid_t)curthreadid; - #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */ - tid = (pid_t)pthread_self; - #endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */ +// -------------------------------------------------------------------------------------------------------------------- + +static size_t webrtc_id = 0; +static __thread bool webrtc_name_set = false; +void webrtc_set_thread_name(void) { + if(_nd_thread_info || webrtc_name_set) return; -#else /* __APPLE__*/ + webrtc_name_set = true; - tid = (pid_t)syscall(SYS_gettid); + char tmp[ND_THREAD_TAG_MAX + 1] = ""; + os_get_thread_name(tmp, sizeof(tmp)); -#endif /* __FreeBSD__, __APPLE__*/ + if(!tmp[0] || strcmp(tmp, "netdata") == 0) { + char name[ND_THREAD_TAG_MAX + 1]; + snprintfz(name, ND_THREAD_TAG_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED)); + os_set_thread_name(name); + } - gettid_cached_tid = tid; - return tid; + nd_thread_get_name(true); } -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- +// locks tracking + +#ifdef NETDATA_INTERNAL_CHECKS +void nd_thread_rwlock_read_locked(void) { if(_nd_thread_info) _nd_thread_info->rwlocks_read_locks++; } +void nd_thread_rwlock_read_unlocked(void) { if(_nd_thread_info) _nd_thread_info->rwlocks_read_locks--; } +void nd_thread_rwlock_write_locked(void) { if(_nd_thread_info) _nd_thread_info->rwlocks_write_locks++; } +void nd_thread_rwlock_write_unlocked(void) { if(_nd_thread_info) _nd_thread_info->rwlocks_write_locks--; } +void nd_thread_mutex_locked(void) { if(_nd_thread_info) _nd_thread_info->mutex_locks++; } +void nd_thread_mutex_unlocked(void) { if(_nd_thread_info) _nd_thread_info->mutex_locks--; } +void nd_thread_spinlock_locked(void) { if(_nd_thread_info) _nd_thread_info->spinlock_locks++; } +void nd_thread_spinlock_unlocked(void) { if(_nd_thread_info) _nd_thread_info->spinlock_locks--; } +void nd_thread_rwspinlock_read_locked(void) { if(_nd_thread_info) _nd_thread_info->rwspinlock_read_locks++; } +void nd_thread_rwspinlock_read_unlocked(void) { if(_nd_thread_info) _nd_thread_info->rwspinlock_read_locks--; } +void nd_thread_rwspinlock_write_locked(void) { if(_nd_thread_info) _nd_thread_info->rwspinlock_write_locks++; } +void nd_thread_rwspinlock_write_unlocked(void) { if(_nd_thread_info) _nd_thread_info->rwspinlock_write_locks--; } +#endif + +// -------------------------------------------------------------------------------------------------------------------- // early initialization size_t netdata_threads_init(void) { int i; - // -------------------------------------------------------------------- - // get the required stack size of the threads of netdata - - if(!netdata_threads_attr) { - netdata_threads_attr = callocz(1, sizeof(pthread_attr_t)); - i = pthread_attr_init(netdata_threads_attr); + if(!threads_globals.attr) { + threads_globals.attr = callocz(1, sizeof(pthread_attr_t)); + i = pthread_attr_init(threads_globals.attr); if (i != 0) fatal("pthread_attr_init() failed with code %d.", i); } + // get the required stack size of the threads of netdata size_t stacksize = 0; - i = pthread_attr_getstacksize(netdata_threads_attr, &stacksize); + i = pthread_attr_getstacksize(threads_globals.attr, &stacksize); if(i != 0) fatal("pthread_attr_getstacksize() failed with code %d.", i); @@ -143,11 +210,9 @@ size_t netdata_threads_init(void) { void netdata_threads_init_after_fork(size_t stacksize) { int i; - // ------------------------------------------------------------------------ // set pthread stack size - - if(netdata_threads_attr && stacksize > (size_t)PTHREAD_STACK_MIN) { - i = pthread_attr_setstacksize(netdata_threads_attr, stacksize); + if(threads_globals.attr && stacksize > (size_t)PTHREAD_STACK_MIN) { + i = pthread_attr_setstacksize(threads_globals.attr, stacksize); if(i != 0) nd_log(NDLS_DAEMON, NDLP_WARNING, "pthread_attr_setstacksize() to %zu bytes, failed with code %d.", stacksize, i); else @@ -169,7 +234,6 @@ void netdata_threads_init_for_external_plugins(size_t stacksize) { } // ---------------------------------------------------------------------------- -// netdata_thread_create void rrdset_thread_rda_free(void); void sender_thread_buffer_free(void); @@ -177,92 +241,99 @@ void query_target_free(void); void service_exits(void); void rrd_collector_finished(void); -static void thread_cleanup(void *ptr) { - if(netdata_thread != ptr) { - NETDATA_THREAD *info = (NETDATA_THREAD *)ptr; - nd_log(NDLS_DAEMON, NDLP_ERR, "THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag); - } +static void nd_thread_join_exited_detached_threads(void) { + while(1) { + spinlock_lock(&threads_globals.exited.spinlock); - if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP)) - nd_log(NDLS_DAEMON, NDLP_DEBUG, "thread with task id %d finished", gettid()); + ND_THREAD *nti = threads_globals.exited.list; + while (nti && nd_thread_status_check(nti, NETDATA_THREAD_OPTION_JOINABLE) == 0) + nti = nti->next; - rrd_collector_finished(); - sender_thread_buffer_free(); - rrdset_thread_rda_free(); - query_target_free(); - thread_cache_destroy(); - service_exits(); - worker_unregister(); + if(nti) + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next); - netdata_thread->tag[0] = '\0'; + spinlock_unlock(&threads_globals.exited.spinlock); - freez(netdata_thread); - netdata_thread = NULL; + if(nti) { + nd_log(NDLS_DAEMON, NDLP_INFO, "Joining detached thread '%s', tid %d", nti->tag, nti->tid); + nd_thread_join(nti); + } + else + break; + } } -void netdata_thread_set_tag(const char *tag) { - if(!tag || !*tag) - return; +static void nd_thread_exit(void *pptr) { + ND_THREAD *nti = CLEANUP_FUNCTION_GET_PTR(pptr); - int ret = 0; + if(nti != _nd_thread_info || !nti || !_nd_thread_info) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "THREADS: internal error - thread local variable does not match the one passed to this function. " + "Expected thread '%s', passed thread '%s'", + _nd_thread_info ? _nd_thread_info->tag : "(null)", nti ? nti->tag : "(null)"); - char threadname[NETDATA_THREAD_NAME_MAX+1]; - strncpyz(threadname, tag, NETDATA_THREAD_NAME_MAX); + if(!nti) nti = _nd_thread_info; + } -#if defined(__FreeBSD__) - pthread_set_name_np(pthread_self(), threadname); -#elif defined(__APPLE__) - ret = pthread_setname_np(threadname); -#else - ret = pthread_setname_np(pthread_self(), threadname); -#endif + if(!nti) return; - if (ret != 0) - nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread name of %d to %s. ErrCode: %d", gettid(), threadname, ret); - else - nd_log(NDLS_DAEMON, NDLP_DEBUG, "set name of thread %d to %s", gettid(), threadname); + internal_fatal(nti->rwlocks_read_locks != 0, + "THREAD '%s' WITH PID %d HAS %d RWLOCKS READ ACQUIRED WHILE EXITING !!!", + (nti) ? nti->tag : "(unset)", gettid_cached(), nti->rwlocks_read_locks); - if(netdata_thread) { - strncpyz(netdata_thread->tag, threadname, sizeof(netdata_thread->tag) - 1); - } -} + internal_fatal(nti->rwlocks_write_locks != 0, + "THREAD '%s' WITH PID %d HAS %d RWLOCKS WRITE ACQUIRED WHILE EXITING !!!", + (nti) ? nti->tag : "(unset)", gettid_cached(), nti->rwlocks_write_locks); -void uv_thread_set_name_np(uv_thread_t ut, const char* name) { - int ret = 0; + internal_fatal(nti->mutex_locks != 0, + "THREAD '%s' WITH PID %d HAS %d MUTEXES ACQUIRED WHILE EXITING !!!", + (nti) ? nti->tag : "(unset)", gettid_cached(), nti->mutex_locks); - char threadname[NETDATA_THREAD_NAME_MAX+1]; - strncpyz(threadname, name, NETDATA_THREAD_NAME_MAX); + internal_fatal(nti->spinlock_locks != 0, + "THREAD '%s' WITH PID %d HAS %d SPINLOCKS ACQUIRED WHILE EXITING !!!", + (nti) ? nti->tag : "(unset)", gettid_cached(), nti->spinlock_locks); -#if defined(__FreeBSD__) - pthread_set_name_np(ut ? ut : pthread_self(), threadname); -#elif defined(__APPLE__) - // Apple can only set its own name - UNUSED(ut); -#else - ret = pthread_setname_np(ut ? ut : pthread_self(), threadname); -#endif + internal_fatal(nti->rwspinlock_read_locks != 0, + "THREAD '%s' WITH PID %d HAS %d RWSPINLOCKS READ ACQUIRED WHILE EXITING !!!", + (nti) ? nti->tag : "(unset)", gettid_cached(), nti->rwspinlock_read_locks); - thread_name_get(true); + internal_fatal(nti->rwspinlock_write_locks != 0, + "THREAD '%s' WITH PID %d HAS %d RWSPINLOCKS WRITE ACQUIRED WHILE EXITING !!!", + (nti) ? nti->tag : "(unset)", gettid_cached(), nti->rwspinlock_write_locks); - if (ret) - nd_log(NDLS_DAEMON, NDLP_NOTICE, "cannot set libuv thread name to %s. Err: %d", threadname, ret); -} + if(nd_thread_status_check(nti, NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP) != NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP) + nd_log(NDLS_DAEMON, NDLP_DEBUG, "thread with task id %d finished", nti->tid); -void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]) -{ - threadname[0] = '\0'; -#if defined(__FreeBSD__) - pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1); -#elif defined(HAVE_PTHREAD_GETNAME_NP) /* Linux & macOS */ - (void)pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1); -#endif + rrd_collector_finished(); + sender_thread_buffer_free(); + rrdset_thread_rda_free(); + query_target_free(); + thread_cache_destroy(); + service_exits(); + worker_unregister(); + + nd_thread_status_set(nti, NETDATA_THREAD_STATUS_FINISHED); + + spinlock_lock(&threads_globals.running.spinlock); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next); + spinlock_unlock(&threads_globals.running.spinlock); + + if (nd_thread_status_check(nti, NETDATA_THREAD_OPTION_JOINABLE) != NETDATA_THREAD_OPTION_JOINABLE) { + spinlock_lock(&threads_globals.exited.spinlock); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next); + spinlock_unlock(&threads_globals.exited.spinlock); + } } -static void *netdata_thread_init(void *ptr) { - netdata_thread = (NETDATA_THREAD *)ptr; +static void *nd_thread_starting_point(void *ptr) { + ND_THREAD *nti = _nd_thread_info = (ND_THREAD *)ptr; + nd_thread_status_set(nti, NETDATA_THREAD_STATUS_STARTED); + + nti->tid = gettid_cached(); + nd_thread_tag_set(nti->tag); - if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_STARTUP)) - nd_log(NDLS_DAEMON, NDLP_DEBUG, "thread created with task id %d", gettid()); + if(nd_thread_status_check(nti, NETDATA_THREAD_OPTION_DONT_LOG_STARTUP) != NETDATA_THREAD_OPTION_DONT_LOG_STARTUP) + nd_log(NDLS_DAEMON, NDLP_DEBUG, "thread created with task id %d", gettid_cached()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread cancel type to DEFERRED."); @@ -270,72 +341,97 @@ static void *netdata_thread_init(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread cancel state to ENABLE."); - netdata_thread_set_tag(netdata_thread->tag); + CLEANUP_FUNCTION_REGISTER(nd_thread_exit) cleanup_ptr = nti; - if (!(netdata_thread->options & NETDATA_THREAD_OPTION_JOINABLE)) { - int rc = pthread_detach(pthread_self()); - if (rc != 0) - nd_log(NDLS_DAEMON, NDLP_WARNING, - "cannot request detach of newly created %s thread. pthread_detach() failed with code %d", - netdata_thread->tag, rc); - } + // run the thread code + nti->ret = nti->start_routine(nti->arg); + + return nti; +} + +ND_THREAD *nd_thread_self(void) { + return _nd_thread_info; +} - void *ret = NULL; - pthread_cleanup_push(thread_cleanup, ptr) { - ret = netdata_thread->start_routine(netdata_thread->arg); +bool nd_thread_is_me(ND_THREAD *nti) { + return nti && nti->thread == pthread_self(); +} + +ND_THREAD *nd_thread_create(const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine)(void *), void *arg) { + nd_thread_join_exited_detached_threads(); + + ND_THREAD *nti = callocz(1, sizeof(*nti)); + spinlock_init(&nti->canceller.spinlock); + nti->arg = arg; + nti->start_routine = start_routine; + nti->options = options & NETDATA_THREAD_OPTIONS_ALL; + strncpyz(nti->tag, tag, ND_THREAD_TAG_MAX); + + spinlock_lock(&threads_globals.running.spinlock); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next); + spinlock_unlock(&threads_globals.running.spinlock); + + int ret = pthread_create(&nti->thread, threads_globals.attr, nd_thread_starting_point, nti); + if(ret != 0) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "failed to create new thread for %s. pthread_create() failed with code %d", + tag, ret); + + spinlock_lock(&threads_globals.running.spinlock); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next); + spinlock_unlock(&threads_globals.running.spinlock); + freez(nti); + return NULL; } - pthread_cleanup_pop(1); - return ret; + return nti; } -int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg) { - NETDATA_THREAD *info = callocz(1, sizeof(NETDATA_THREAD)); - info->arg = arg; - info->start_routine = start_routine; - info->options = options; - strncpyz(info->tag, tag, NETDATA_THREAD_NAME_MAX); +// -------------------------------------------------------------------------------------------------------------------- - int ret = pthread_create(thread, netdata_threads_attr, netdata_thread_init, info); - if(ret != 0) - nd_log(NDLS_DAEMON, NDLP_ERR, "failed to create new thread for %s. pthread_create() failed with code %d", tag, ret); +void nd_thread_register_canceller(nd_thread_canceller cb, void *data) { + ND_THREAD *nti = _nd_thread_info; + if(!nti) return; - return ret; + spinlock_lock(&nti->canceller.spinlock); + nti->canceller.cb = cb; + nti->canceller.data = data; + spinlock_unlock(&nti->canceller.spinlock); } -// ---------------------------------------------------------------------------- -// netdata_thread_cancel -#ifdef NETDATA_INTERNAL_CHECKS -int netdata_thread_cancel_with_trace(netdata_thread_t thread, int line, const char *file, const char *function) { -#else -int netdata_thread_cancel(netdata_thread_t thread) { -#endif - int ret = pthread_cancel(thread); - if(ret != 0) -#ifdef NETDATA_INTERNAL_CHECKS - nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot cancel thread. pthread_cancel() failed with code %d at %d@%s, function %s()", ret, line, file, function); -#else - nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot cancel thread. pthread_cancel() failed with code %d.", ret); -#endif +void nd_thread_signal_cancel(ND_THREAD *nti) { + if(!nti) return; + + __atomic_store_n(&nti->cancel_atomic, true, __ATOMIC_RELAXED); + + spinlock_lock(&nti->canceller.spinlock); + if(nti->canceller.cb) + nti->canceller.cb(nti->canceller.data); + spinlock_unlock(&nti->canceller.spinlock); +} - return ret; +bool nd_thread_signaled_to_cancel(void) { + if(!_nd_thread_info) return false; + return __atomic_load_n(&_nd_thread_info->cancel_atomic, __ATOMIC_RELAXED); } // ---------------------------------------------------------------------------- -// netdata_thread_join +// nd_thread_join -int netdata_thread_join(netdata_thread_t thread, void **retval) { - int ret = pthread_join(thread, retval); +void nd_thread_join(ND_THREAD *nti) { + if(!nti) return; + + int ret = pthread_join(nti->thread, NULL); if(ret != 0) nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot join thread. pthread_join() failed with code %d.", ret); + else { + nd_thread_status_set(nti, NETDATA_THREAD_STATUS_JOINED); - return ret; -} - -int netdata_thread_detach(pthread_t thread) { - int ret = pthread_detach(thread); - if(ret != 0) - nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot detach thread. pthread_detach() failed with code %d.", ret); + spinlock_lock(&threads_globals.exited.spinlock); + if(nti->prev) + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next); + spinlock_unlock(&threads_globals.exited.spinlock); - return ret; + freez(nti); + } } diff --git a/src/libnetdata/threads/threads.h b/src/libnetdata/threads/threads.h index 4f1d06f00a3e5c..a7204e2a2ffbe4 100644 --- a/src/libnetdata/threads/threads.h +++ b/src/libnetdata/threads/threads.h @@ -5,22 +5,25 @@ #include "../libnetdata.h" -pid_t gettid(void); - -typedef enum { +typedef enum __attribute__((packed)) { NETDATA_THREAD_OPTION_DEFAULT = 0 << 0, NETDATA_THREAD_OPTION_JOINABLE = 1 << 0, NETDATA_THREAD_OPTION_DONT_LOG_STARTUP = 1 << 1, NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP = 1 << 2, - NETDATA_THREAD_OPTION_DONT_LOG = NETDATA_THREAD_OPTION_DONT_LOG_STARTUP|NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP, + NETDATA_THREAD_STATUS_STARTED = 1 << 3, + NETDATA_THREAD_STATUS_FINISHED = 1 << 4, + NETDATA_THREAD_STATUS_JOINED = 1 << 5, } NETDATA_THREAD_OPTIONS; +#define NETDATA_THREAD_OPTIONS_ALL (NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG_STARTUP | NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP) +#define NETDATA_THREAD_OPTION_DONT_LOG (NETDATA_THREAD_OPTION_DONT_LOG_STARTUP | NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP) + #define netdata_thread_cleanup_push(func, arg) pthread_cleanup_push(func, arg) #define netdata_thread_cleanup_pop(execute) pthread_cleanup_pop(execute) -void netdata_thread_set_tag(const char *tag); +void nd_thread_tag_set(const char *tag); -typedef pthread_t netdata_thread_t; +typedef struct nd_thread ND_THREAD; struct netdata_static_thread { // the name of the thread as it should appear in the logs @@ -36,7 +39,7 @@ struct netdata_static_thread { volatile sig_atomic_t enabled; // internal use, to maintain a pointer to the created thread - netdata_thread_t *thread; + ND_THREAD *thread; // an initialization function to run before spawning the thread void (*init_routine) (void); @@ -56,36 +59,56 @@ struct netdata_static_thread { #define NETDATA_MAIN_THREAD_EXITED CONFIG_BOOLEAN_NO #define NETDATA_THREAD_TAG_MAX 100 -const char *netdata_thread_tag(void); -int netdata_thread_tag_exists(void); +const char *nd_thread_tag(void); +int nd_thread_has_tag(void); #define THREAD_TAG_STREAM_RECEIVER "RCVR" #define THREAD_TAG_STREAM_SENDER "SNDR" - size_t netdata_threads_init(void); void netdata_threads_init_after_fork(size_t stacksize); void netdata_threads_init_for_external_plugins(size_t stacksize); -int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg); +ND_THREAD *nd_thread_create(const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg); +void nd_thread_join(ND_THREAD * nti); +ND_THREAD *nd_thread_self(void); +bool nd_thread_is_me(ND_THREAD *nti); -#ifdef NETDATA_INTERNAL_CHECKS -#define netdata_thread_cancel(thread) netdata_thread_cancel_with_trace(thread, __LINE__, __FILE__, __FUNCTION__) -int netdata_thread_cancel_with_trace(netdata_thread_t thread, int line, const char *file, const char *function); -#else -int netdata_thread_cancel(netdata_thread_t thread); -#endif - -int netdata_thread_join(netdata_thread_t thread, void **retval); -int netdata_thread_detach(pthread_t thread); - -#define NETDATA_THREAD_NAME_MAX 15 -void uv_thread_set_name_np(uv_thread_t ut, const char* name); -void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]); +typedef void (*nd_thread_canceller)(void *data); +void nd_thread_register_canceller(nd_thread_canceller cb, void *data); +void nd_thread_signal_cancel(ND_THREAD *nti); +bool nd_thread_signaled_to_cancel(void); +#define ND_THREAD_TAG_MAX 15 +void uv_thread_set_name_np(const char* name); void webrtc_set_thread_name(void); -#define netdata_thread_self pthread_self -#define netdata_thread_testcancel pthread_testcancel +#ifdef NETDATA_INTERNAL_CHECKS +void nd_thread_rwlock_read_locked(void); +void nd_thread_rwlock_read_unlocked(void); +void nd_thread_rwlock_write_locked(void); +void nd_thread_rwlock_write_unlocked(void); +void nd_thread_mutex_locked(void); +void nd_thread_mutex_unlocked(void); +void nd_thread_spinlock_locked(void); +void nd_thread_spinlock_unlocked(void); +void nd_thread_rwspinlock_read_locked(void); +void nd_thread_rwspinlock_read_unlocked(void); +void nd_thread_rwspinlock_write_locked(void); +void nd_thread_rwspinlock_write_unlocked(void); +#else +#define nd_thread_rwlock_read_locked() debug_dummy() +#define nd_thread_rwlock_read_unlocked() debug_dummy() +#define nd_thread_rwlock_write_locked() debug_dummy() +#define nd_thread_rwlock_write_unlocked() debug_dummy() +#define nd_thread_mutex_locked() debug_dummy() +#define nd_thread_mutex_unlocked() debug_dummy() +#define nd_thread_spinlock_locked() debug_dummy() +#define nd_thread_spinlock_unlocked() debug_dummy() +#define nd_thread_rwspinlock_read_locked() debug_dummy() +#define nd_thread_rwspinlock_read_unlocked() debug_dummy() +#define nd_thread_rwspinlock_write_locked() debug_dummy() +#define nd_thread_rwspinlock_write_unlocked() debug_dummy() +#endif #endif //NETDATA_THREADS_H diff --git a/src/libnetdata/url/url.c b/src/libnetdata/url/url.c index e8497171433cce..720a703d4ef705 100644 --- a/src/libnetdata/url/url.c +++ b/src/libnetdata/url/url.c @@ -25,7 +25,7 @@ char *url_encode(char *str) { pbuf = buf = mallocz(strlen(str) * 3 + 1); while (*str) { - if (isalnum(*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~') + if (isalnum((uint8_t)*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~') *pbuf++ = *str; else if (*str == ' ') @@ -267,9 +267,9 @@ url_is_request_complete_and_extract_payload(const char *begin, const char *end, const char *ct = strcasestr(begin, "Content-Type: "); if(ct) { ct = &ct[14]; - while (*ct && isspace(*ct)) ct++; + while (*ct && isspace((uint8_t)*ct)) ct++; const char *space = ct; - while (*space && !isspace(*space) && *space != ';') space++; + while (*space && !isspace((uint8_t)*space) && *space != ';') space++; size_t ct_len = space - ct; char ct_copy[ct_len + 1]; diff --git a/src/libnetdata/uuid/uuid.c b/src/libnetdata/uuid/uuid.c index f062d0bc569895..6b05229fc347d9 100644 --- a/src/libnetdata/uuid/uuid.c +++ b/src/libnetdata/uuid/uuid.c @@ -2,11 +2,10 @@ #include "../libnetdata.h" +ND_UUID UUID_generate_from_hash(const void *payload, size_t payload_len) { + assert(sizeof(XXH128_hash_t) == sizeof(ND_UUID)); -UUID UUID_generate_from_hash(const void *payload, size_t payload_len) { - assert(sizeof(XXH128_hash_t) == sizeof(UUID)); - - UUID uuid; + ND_UUID uuid = UUID_ZERO; XXH128_hash_t *xxh3_128 = (XXH128_hash_t *)&uuid; // Hash the payload using XXH128 @@ -22,7 +21,7 @@ UUID UUID_generate_from_hash(const void *payload, size_t payload_len) { return uuid; } -void uuid_unparse_lower_compact(const uuid_t uuid, char *out) { +void uuid_unparse_lower_compact(const nd_uuid_t uuid, char *out) { static const char *hex_chars = "0123456789abcdef"; for (int i = 0; i < 16; i++) { out[i * 2] = hex_chars[(uuid[i] >> 4) & 0x0F]; @@ -31,7 +30,29 @@ void uuid_unparse_lower_compact(const uuid_t uuid, char *out) { out[32] = '\0'; // Null-terminate the string } -inline int uuid_parse_compact(const char *in, uuid_t uuid) { +static inline void nd_uuid_unparse_full(const nd_uuid_t uuid, char *out, const char *hex_chars) { + int shifts = 0; + for (int i = 0; i < 16; i++) { + if (i == 4 || i == 6 || i == 8 || i == 10) { + out[i * 2 + shifts] = '-'; + shifts++; + } + out[i * 2 + shifts] = hex_chars[(uuid[i] >> 4) & 0x0F]; + out[i * 2 + 1 + shifts] = hex_chars[uuid[i] & 0x0F]; + } + out[36] = '\0'; // Null-terminate the string +} + +// Wrapper functions for lower and upper case hexadecimal representation +void nd_uuid_unparse_lower(const nd_uuid_t uuid, char *out) { + nd_uuid_unparse_full(uuid, out, "0123456789abcdef"); +} + +void nd_uuid_unparse_upper(const nd_uuid_t uuid, char *out) { + nd_uuid_unparse_full(uuid, out, "0123456789ABCDEF"); +} + +inline int uuid_parse_compact(const char *in, nd_uuid_t uuid) { if (strlen(in) != 32) return -1; // Invalid input length @@ -48,7 +69,7 @@ inline int uuid_parse_compact(const char *in, uuid_t uuid) { return 0; // Success } -int uuid_parse_flexi(const char *in, uuid_t uu) { +int uuid_parse_flexi(const char *in, nd_uuid_t uu) { if(!in || !*in) return -1; @@ -56,7 +77,7 @@ int uuid_parse_flexi(const char *in, uuid_t uu) { size_t hyphenCount = 0; const char *s = in; int byteIndex = 0; - uuid_t uuid; // work on a temporary place, to not corrupt the previous value of uu if we fail + nd_uuid_t uuid; // work on a temporary place, to not corrupt the previous value of uu if we fail while (*s && byteIndex < 16) { if (*s == '-') { @@ -68,11 +89,11 @@ int uuid_parse_flexi(const char *in, uuid_t uu) { return -2; } - if (likely(isxdigit(*s))) { + if (likely(isxdigit((uint8_t)*s))) { int high = hex_char_to_int(*s++); hexCharCount++; - if (likely(isxdigit(*s))) { + if (likely(isxdigit((uint8_t)*s))) { int low = hex_char_to_int(*s++); hexCharCount++; @@ -100,7 +121,7 @@ int uuid_parse_flexi(const char *in, uuid_t uu) { return -7; // copy the final value - memcpy(uu, uuid, sizeof(uuid_t)); + memcpy(uu, uuid, sizeof(nd_uuid_t)); return 0; } @@ -125,7 +146,7 @@ int uuid_unittest(void) { int i; for (i = 0; i < num_tests; i++) { - uuid_t original_uuid, parsed_uuid; + nd_uuid_t original_uuid, parsed_uuid; char uuid_str_with_hyphens[UUID_STR_LEN], uuid_str_without_hyphens[UUID_COMPACT_STR_LEN]; // Generate a random UUID diff --git a/src/libnetdata/uuid/uuid.h b/src/libnetdata/uuid/uuid.h index 6d5f024f86494d..cde457616e7a15 100644 --- a/src/libnetdata/uuid/uuid.h +++ b/src/libnetdata/uuid/uuid.h @@ -3,39 +3,60 @@ #ifndef NETDATA_UUID_H #define NETDATA_UUID_H -UUID_DEFINE(streaming_from_child_msgid, 0xed,0x4c,0xdb, 0x8f, 0x1b, 0xeb, 0x4a, 0xd3, 0xb5, 0x7c, 0xb3, 0xca, 0xe2, 0xd1, 0x62, 0xfa); -UUID_DEFINE(streaming_to_parent_msgid, 0x6e, 0x2e, 0x38, 0x39, 0x06, 0x76, 0x48, 0x96, 0x8b, 0x64, 0x60, 0x45, 0xdb, 0xf2, 0x8d, 0x66); -UUID_DEFINE(health_alert_transition_msgid, 0x9c, 0xe0, 0xcb, 0x58, 0xab, 0x8b, 0x44, 0xdf, 0x82, 0xc4, 0xbf, 0x1a, 0xd9, 0xee, 0x22, 0xde); +// for compatibility with libuuid +typedef unsigned char nd_uuid_t[16]; -// this is also defined in alarm-notify.sh.in -UUID_DEFINE(health_alert_notification_msgid, 0x6d, 0xb0, 0x01, 0x8e, 0x83, 0xe3, 0x43, 0x20, 0xae, 0x2a, 0x65, 0x9d, 0x78, 0x01, 0x9f, 0xb7); - -typedef struct { +// for quickly managing it as 2x 64-bit numbers +typedef struct _uuid { union { - uuid_t uuid; + nd_uuid_t uuid; struct { uint64_t hig64; uint64_t low64; } parts; }; -} UUID; -UUID UUID_generate_from_hash(const void *payload, size_t payload_len); +} ND_UUID; + +#ifdef __GNUC__ +#define ND_UUID_DEFINE(name,u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15) \ + static const nd_uuid_t name __attribute__ ((unused)) = {u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15} +#else +#define ND_UUID_DEFINE(name,u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15) \ + static const nd_uuid_t name = {u0,u1,u2,u3,u4,u5,u6,u7,u8,u9,u10,u11,u12,u13,u14,u15} +#endif + +static const ND_UUID UUID_ZERO = (ND_UUID){ { .parts = { .hig64 = 0, .low64 = 0 } }}; +ND_UUID_DEFINE(streaming_from_child_msgid, 0xed,0x4c,0xdb, 0x8f, 0x1b, 0xeb, 0x4a, 0xd3, 0xb5, 0x7c, 0xb3, 0xca, 0xe2, 0xd1, 0x62, 0xfa); +ND_UUID_DEFINE(streaming_to_parent_msgid, 0x6e, 0x2e, 0x38, 0x39, 0x06, 0x76, 0x48, 0x96, 0x8b, 0x64, 0x60, 0x45, 0xdb, 0xf2, 0x8d, 0x66); +ND_UUID_DEFINE(health_alert_transition_msgid, 0x9c, 0xe0, 0xcb, 0x58, 0xab, 0x8b, 0x44, 0xdf, 0x82, 0xc4, 0xbf, 0x1a, 0xd9, 0xee, 0x22, 0xde); + +// this is also defined in alarm-notify.sh.in +ND_UUID_DEFINE(health_alert_notification_msgid, 0x6d, 0xb0, 0x01, 0x8e, 0x83, 0xe3, 0x43, 0x20, 0xae, 0x2a, 0x65, 0x9d, 0x78, 0x01, 0x9f, 0xb7); + +ND_UUID UUID_generate_from_hash(const void *payload, size_t payload_len); #define UUIDeq(a, b) ((a).parts.hig64 == (b).parts.hig64 && (a).parts.low64 == (b).parts.low64) -static inline UUID uuid2UUID(uuid_t uu1) { - UUID *ret = (UUID *)uu1; - return *ret; +static inline ND_UUID uuid2UUID(const nd_uuid_t uu1) { + // uu1 may not be aligned, so copy it to the output + ND_UUID copy; + memcpy(copy.uuid, uu1, sizeof(nd_uuid_t)); + return copy; } +#ifndef UUID_STR_LEN +// CentOS 7 has older version that doesn't define this +// same goes for MacOS +#define UUID_STR_LEN 37 +#endif + #define UUID_COMPACT_STR_LEN 33 -void uuid_unparse_lower_compact(const uuid_t uuid, char *out); -int uuid_parse_compact(const char *in, uuid_t uuid); -int uuid_parse_flexi(const char *in, uuid_t uuid); -static inline int uuid_memcmp(const uuid_t *uu1, const uuid_t *uu2) { - return memcmp(uu1, uu2, sizeof(uuid_t)); -} +void uuid_unparse_lower_compact(const nd_uuid_t uuid, char *out); +int uuid_parse_compact(const char *in, nd_uuid_t uuid); + +int uuid_parse_flexi(const char *in, nd_uuid_t uuid); +#define uuid_parse(in, uuid) uuid_parse_flexi(in, uuid) static inline int hex_char_to_int(char c) { if (c >= '0' && c <= '9') return c - '0'; @@ -44,4 +65,48 @@ static inline int hex_char_to_int(char c) { return -1; // Invalid hexadecimal character } +static inline void nd_uuid_clear(nd_uuid_t uu) { + memset(uu, 0, sizeof(nd_uuid_t)); +} + +// Netdata does not need to sort UUIDs lexicographically and this kind +// of sorting does not need to be portable between little/big endian. +// So, any kind of sorting will work, as long as it compares UUIDs. +// The fastest possible, is good enough. +static inline int nd_uuid_compare(const nd_uuid_t uu1, const nd_uuid_t uu2) { + // IMPORTANT: + // uu1 or uu2 may not be aligned to word boundaries on this call, + // so casting this to a struct may give SIGBUS on some architectures. + return memcmp(uu1, uu2, sizeof(nd_uuid_t)); +} + +static inline void nd_uuid_copy(nd_uuid_t dst, const nd_uuid_t src) { + memcpy(dst, src, sizeof(nd_uuid_t)); +} + +static inline bool nd_uuid_eq(const nd_uuid_t uu1, const nd_uuid_t uu2) { + return nd_uuid_compare(uu1, uu2) == 0; +} + +static inline int nd_uuid_is_null(const nd_uuid_t uu) { + return nd_uuid_compare(uu, UUID_ZERO.uuid) == 0; +} + +void nd_uuid_unparse_lower(const nd_uuid_t uuid, char *out); +void nd_uuid_unparse_upper(const nd_uuid_t uuid, char *out); + +#define uuid_is_null(uu) nd_uuid_is_null(uu) +#define uuid_clear(uu) nd_uuid_clear(uu) +#define uuid_compare(uu1, uu2) nd_uuid_compare(uu1, uu2) +#define uuid_copy(dst, src) nd_uuid_copy(dst, src) +#define uuid_eq(uu1, uu2) nd_uuid_eq(uu1, uu2) + +#define uuid_generate(out) os_uuid_generate(out) +#define uuid_generate_random(out) os_uuid_generate_random(out) +#define uuid_generate_time(out) os_uuid_generate_time(out) + +#define uuid_unparse(uu, out) nd_uuid_unparse_lower(uu, out) +#define uuid_unparse_lower(uu, out) nd_uuid_unparse_lower(uu, out) +#define uuid_unparse_upper(uu, out) nd_uuid_unparse_upper(uu, out) + #endif //NETDATA_UUID_H diff --git a/src/libnetdata/worker_utilization/worker_utilization.c b/src/libnetdata/worker_utilization/worker_utilization.c index f39cea8a03c04e..4c61ea921ff9f8 100644 --- a/src/libnetdata/worker_utilization/worker_utilization.c +++ b/src/libnetdata/worker_utilization/worker_utilization.c @@ -92,8 +92,8 @@ void worker_register(const char *name) { return; worker = callocz(1, sizeof(struct worker)); - worker->pid = gettid(); - worker->tag = strdupz(netdata_thread_tag()); + worker->pid = gettid_cached(); + worker->tag = strdupz(nd_thread_tag()); worker->workname = strdupz(name); usec_t now = worker_now_monotonic_usec(); diff --git a/src/logsmanagement/db_api.c b/src/logsmanagement/db_api.c index 50018ea26df2d5..a3489b2dfb8ced 100644 --- a/src/logsmanagement/db_api.c +++ b/src/logsmanagement/db_api.c @@ -617,7 +617,7 @@ int db_init() { /* Create directory of collection of logs for the particular * log source (in the form of a UUID) and bind it. */ - uuid_t uuid; + nd_uuid_t uuid; uuid_generate(uuid); char uuid_str[UUID_STR_LEN]; // ex. "1b4e28ba-2fa1-11d2-883f-0016d3cca427" + "\0" uuid_unparse_lower(uuid, uuid_str); diff --git a/src/ml/ml-dummy.c b/src/ml/ml-dummy.c index 7058e21cdb9143..64a1a6b099e3c6 100644 --- a/src/ml/ml-dummy.c +++ b/src/ml/ml-dummy.c @@ -100,7 +100,7 @@ bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool return false; } -int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **stmp) { +int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **stmp __maybe_unused) { UNUSED(rd); return 0; } @@ -109,12 +109,12 @@ void ml_update_global_statistics_charts(uint64_t models_consulted) { UNUSED(models_consulted); } -bool ml_host_get_host_status(RRDHOST *rh, struct ml_metrics_statistics *mlm) { +bool ml_host_get_host_status(RRDHOST *rh __maybe_unused, struct ml_metrics_statistics *mlm) { memset(mlm, 0, sizeof(*mlm)); return false; } -bool ml_host_running(RRDHOST *rh) { +bool ml_host_running(RRDHOST *rh __maybe_unused) { return false; } diff --git a/src/ml/ml-private.h b/src/ml/ml-private.h index cfa6419efaef90..fc90589b37b95d 100644 --- a/src/ml/ml-private.h +++ b/src/ml/ml-private.h @@ -274,13 +274,13 @@ typedef struct { } ml_host_t; typedef struct { - uuid_t metric_uuid; + nd_uuid_t metric_uuid; ml_kmeans_t kmeans; } ml_model_info_t; typedef struct { size_t id; - netdata_thread_t nd_thread; + ND_THREAD *nd_thread; netdata_mutex_t nd_mutex; ml_queue_t *training_queue; @@ -347,7 +347,7 @@ typedef struct { std::vector random_nums; - netdata_thread_t detection_thread; + ND_THREAD *detection_thread; std::atomic detection_stop; size_t num_training_threads; diff --git a/src/ml/ml.cc b/src/ml/ml.cc index 33a01f2c6fe29b..88de840595784a 100644 --- a/src/ml/ml.cc +++ b/src/ml/ml.cc @@ -443,7 +443,7 @@ const char *db_models_prune = "WHERE after < @after LIMIT @n;"; static int -ml_dimension_add_model(const uuid_t *metric_uuid, const ml_kmeans_t *km) +ml_dimension_add_model(const nd_uuid_t *metric_uuid, const ml_kmeans_t *km) { static __thread sqlite3_stmt *res = NULL; int param = 0; @@ -520,7 +520,7 @@ ml_dimension_add_model(const uuid_t *metric_uuid, const ml_kmeans_t *km) } static int -ml_dimension_delete_models(const uuid_t *metric_uuid, time_t before) +ml_dimension_delete_models(const nd_uuid_t *metric_uuid, time_t before) { static __thread sqlite3_stmt *res = NULL; int rc = 0; @@ -1175,7 +1175,7 @@ ml_acquired_dimension_get(char *machine_guid, STRING *chart_id, STRING *dimensio } } - rrd_unlock(); + rrd_rdunlock(); ml_acquired_dimension_t acq_dim = { acq_rh, acq_rs, acq_rd, dim @@ -1232,7 +1232,7 @@ ml_detect_main(void *arg) ml_host_detect_once((ml_host_t *) rh->ml_host); } - rrd_unlock(); + rrd_rdunlock(); if (Cfg.enable_statistics_charts) { // collect and update training thread stats @@ -1833,12 +1833,14 @@ void ml_start_threads() { char tag[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(tag, NETDATA_THREAD_TAG_MAX, "%s", "PREDICT"); - netdata_thread_create(&Cfg.detection_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, ml_detect_main, NULL); + Cfg.detection_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, + ml_detect_main, NULL); for (size_t idx = 0; idx != Cfg.num_training_threads; idx++) { ml_training_thread_t *training_thread = &Cfg.training_threads[idx]; snprintfz(tag, NETDATA_THREAD_TAG_MAX, "TRAIN[%zu]", training_thread->id); - netdata_thread_create(&training_thread->nd_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, ml_train_main, training_thread); + training_thread->nd_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, + ml_train_main, training_thread); } } @@ -1853,7 +1855,7 @@ void ml_stop_threads() if (!Cfg.detection_thread) return; - netdata_thread_join(Cfg.detection_thread, NULL); + nd_thread_join(Cfg.detection_thread); Cfg.detection_thread = 0; // signal the training queue of each thread @@ -1863,18 +1865,11 @@ void ml_stop_threads() ml_queue_signal(training_thread->training_queue); } - // cancel training threads - for (size_t idx = 0; idx != Cfg.num_training_threads; idx++) { - ml_training_thread_t *training_thread = &Cfg.training_threads[idx]; - - netdata_thread_cancel(training_thread->nd_thread); - } - // join training threads for (size_t idx = 0; idx != Cfg.num_training_threads; idx++) { ml_training_thread_t *training_thread = &Cfg.training_threads[idx]; - netdata_thread_join(training_thread->nd_thread, NULL); + nd_thread_join(training_thread->nd_thread); } // clear training thread data diff --git a/src/registry/registry_init.c b/src/registry/registry_init.c index 79523e258198ad..c291c6f822573b 100644 --- a/src/registry/registry_init.c +++ b/src/registry/registry_init.c @@ -172,9 +172,6 @@ int registry_init(void) { &netdata_configured_cache_dir, use_mmap, true); - // disable cancelability to avoid enable/disable per item in the dictionary locks - netdata_thread_disable_cancelability(); - registry_log_open(); registry_db_load(); registry_log_load(); @@ -185,8 +182,6 @@ int registry_init(void) { // registry_db_stats(); // registry_generate_curl_urls(); // exit(0); - - netdata_thread_enable_cancelability(); } return 0; diff --git a/src/registry/registry_internals.c b/src/registry/registry_internals.c index dbe9c3e3eb45ab..54fad4254f24b6 100644 --- a/src/registry/registry_internals.c +++ b/src/registry/registry_internals.c @@ -11,7 +11,7 @@ struct registry registry; // parse a GUID and re-generated to be always lower case // this is used as a protection against the variations of GUIDs int regenerate_guid(const char *guid, char *result) { - uuid_t uuid; + nd_uuid_t uuid; if(unlikely(uuid_parse(guid, uuid) == -1)) { netdata_log_info("Registry: GUID '%s' is not a valid GUID.", guid); return -1; @@ -35,12 +35,12 @@ static inline char *registry_fix_machine_name(char *name, size_t *len) { char *s = name?name:""; // skip leading spaces - while(*s && isspace(*s)) s++; + while(*s && isspace((uint8_t)*s)) s++; // make sure all spaces are a SPACE char *t = s; while(*t) { - if(unlikely(isspace(*t))) + if(unlikely(isspace((uint8_t)*t))) *t = ' '; t++; @@ -298,7 +298,7 @@ const char *registry_get_this_machine_guid(void) { // generate a new one? if(!guid[0]) { - uuid_t uuid; + nd_uuid_t uuid; uuid_generate_time(uuid); uuid_unparse_lower(uuid, guid); diff --git a/src/registry/registry_person.c b/src/registry/registry_person.c index 4fd40fd75b1d51..a9d3ac88aec8ed 100644 --- a/src/registry/registry_person.c +++ b/src/registry/registry_person.c @@ -118,7 +118,7 @@ REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when) REGISTRY_PERSON *p = aral_mallocz(registry.persons_aral); if(!person_guid) { for(;;) { - uuid_t uuid; + nd_uuid_t uuid; uuid_generate(uuid); uuid_unparse_lower(uuid, p->guid); diff --git a/src/spawn/spawn.c b/src/spawn/spawn.c index 3d62df7965065e..a6e53718af4cbe 100644 --- a/src/spawn/spawn.c +++ b/src/spawn/spawn.c @@ -248,7 +248,6 @@ void spawn_init(void) /* wait for spawn client thread to initialize */ completion_wait_for(&completion); completion_destroy(&completion); - uv_thread_set_name_np(thread, "DAEMON_SPAWN"); if (spawn_thread_error) { error = uv_thread_join(&thread); diff --git a/src/spawn/spawn_client.c b/src/spawn/spawn_client.c index 8928a468c6bc98..f2af9842cad0f5 100644 --- a/src/spawn/spawn_client.c +++ b/src/spawn/spawn_client.c @@ -170,6 +170,8 @@ static void spawn_process_cmd(struct spawn_cmd_info *cmdinfo) void spawn_client(void *arg) { + uv_thread_set_name_np("DAEMON_SPAWN"); + int ret; struct completion *completion = (struct completion *)arg; diff --git a/src/spawn/spawn_server.c b/src/spawn/spawn_server.c index 1d79ef15d48971..f17669368f365a 100644 --- a/src/spawn/spawn_server.c +++ b/src/spawn/spawn_server.c @@ -127,7 +127,7 @@ static void wait_children(void *arg) while (!server_shutdown) { i.si_pid = 0; - if (waitid(P_ALL, (id_t) 0, &i, WEXITED) == -1) { + if (os_waitid(P_ALL, (id_t) 0, &i, WEXITED) == -1) { if (errno != ECHILD) fprintf(stderr, "SPAWN: Failed to wait: %s\n", strerror(errno)); break; diff --git a/src/streaming/receiver.c b/src/streaming/receiver.c index a4888aef4cc4ce..2cbf247dc48809 100644 --- a/src/streaming/receiver.c +++ b/src/streaming/receiver.c @@ -58,8 +58,12 @@ static inline int read_stream(struct receiver_state *r, char* buffer, size_t siz } #ifdef ENABLE_H2O - if (is_h2o_rrdpush(r)) + if (is_h2o_rrdpush(r)) { + if(nd_thread_signaled_to_cancel()) + return -4; + return (int)h2o_stream_read(r->h2o_ctx, buffer, size); + } #endif int tries = 100; @@ -68,6 +72,29 @@ static inline int read_stream(struct receiver_state *r, char* buffer, size_t siz do { errno = 0; + switch(wait_on_socket_or_cancel_with_timeout( +#ifdef ENABLE_HTTPS + &r->ssl, +#endif + r->fd, 0, POLLIN, NULL)) + { + case 0: // data are waiting + break; + + case 1: // timeout reached + netdata_log_error("STREAM: %s(): timeout while waiting for data on socket!", __FUNCTION__); + return -3; + + case -1: // thread cancelled + netdata_log_error("STREAM: %s(): thread has been cancelled timeout while waiting for data on socket!", __FUNCTION__); + return -4; + + default: + case 2: // error on socket + netdata_log_error("STREAM: %s() socket error!", __FUNCTION__); + return -2; + } + #ifdef ENABLE_HTTPS if (SSL_connection(&r->ssl)) bytes_read = netdata_ssl_read(&r->ssl, buffer, size); @@ -116,6 +143,10 @@ static inline STREAM_HANDSHAKE read_stream_error_to_reason(int code) { // timeout return STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_TIMEOUT; + case -4: + // the thread is cancelled + return STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN; + default: // anything else return STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR; @@ -261,6 +292,11 @@ static void receiver_set_exit_reason(struct receiver_state *rpt, STREAM_HANDSHAK static inline bool receiver_should_stop(struct receiver_state *rpt) { static __thread size_t counter = 0; + if(nd_thread_signaled_to_cancel()) { + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, false); + return true; + } + if(unlikely(rpt->exit.shutdown)) { receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, false); return true; @@ -271,11 +307,8 @@ static inline bool receiver_should_stop(struct receiver_state *rpt) { return true; } - if(unlikely((counter++ % 1000) == 0)) { - // check every 1000 lines read - netdata_thread_testcancel(); + if(unlikely((counter++ % 1000) == 0)) rpt->last_msg_t = now_monotonic_sec(); - } return false; } @@ -307,60 +340,58 @@ static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, i // this keeps the parser with its current value // so, parser needs to be allocated before pushing it - netdata_thread_cleanup_push(pluginsd_process_thread_cleanup, parser) { - bool compressed_connection = rrdpush_decompression_initialize(rpt); - buffered_reader_init(&rpt->reader); + CLEANUP_FUNCTION_REGISTER(pluginsd_process_thread_cleanup) parser_ptr = parser; + + bool compressed_connection = rrdpush_decompression_initialize(rpt); + buffered_reader_init(&rpt->reader); #ifdef NETDATA_LOG_STREAM_RECEIVE - { - char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "/tmp/stream-receiver-%s.txt", rpt->host ? rrdhost_hostname( - rpt->host) : "unknown" - ); - parser->user.stream_log_fp = fopen(filename, "w"); - parser->user.stream_log_repertoire = PARSER_REP_METADATA; - } + { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "/tmp/stream-receiver-%s.txt", rpt->host ? rrdhost_hostname( + rpt->host) : "unknown" + ); + parser->user.stream_log_fp = fopen(filename, "w"); + parser->user.stream_log_repertoire = PARSER_REP_METADATA; + } #endif - CLEAN_BUFFER *buffer = buffer_create(sizeof(rpt->reader.read_buffer), NULL); - - ND_LOG_STACK lgs[] = { - ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &parser->line), - ND_LOG_FIELD_CB(NDF_NIDL_NODE, parser_reconstruct_node, parser), - ND_LOG_FIELD_CB(NDF_NIDL_INSTANCE, parser_reconstruct_instance, parser), - ND_LOG_FIELD_CB(NDF_NIDL_CONTEXT, parser_reconstruct_context, parser), - ND_LOG_FIELD_END(), - }; - ND_LOG_STACK_PUSH(lgs); - - while(!receiver_should_stop(rpt)) { + CLEAN_BUFFER *buffer = buffer_create(sizeof(rpt->reader.read_buffer), NULL); - if(!buffered_reader_next_line(&rpt->reader, buffer)) { - STREAM_HANDSHAKE reason = STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR; + ND_LOG_STACK lgs[] = { + ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &parser->line), + ND_LOG_FIELD_CB(NDF_NIDL_NODE, parser_reconstruct_node, parser), + ND_LOG_FIELD_CB(NDF_NIDL_INSTANCE, parser_reconstruct_instance, parser), + ND_LOG_FIELD_CB(NDF_NIDL_CONTEXT, parser_reconstruct_context, parser), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); - bool have_new_data = compressed_connection ? receiver_read_compressed(rpt, &reason) - : receiver_read_uncompressed(rpt, &reason); + while(!receiver_should_stop(rpt)) { - if(unlikely(!have_new_data)) { - receiver_set_exit_reason(rpt, reason, false); - break; - } + if(!buffered_reader_next_line(&rpt->reader, buffer)) { + STREAM_HANDSHAKE reason = STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR; - continue; - } + bool have_new_data = compressed_connection ? receiver_read_compressed(rpt, &reason) + : receiver_read_uncompressed(rpt, &reason); - if(unlikely(parser_action(parser, buffer->buffer))) { - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false); + if(unlikely(!have_new_data)) { + receiver_set_exit_reason(rpt, reason, false); break; } - buffer->len = 0; - buffer->buffer[0] = '\0'; + continue; } - result = parser->user.data_collections_count; - } - netdata_thread_cleanup_pop(1); // free parser with the pop function + if(unlikely(parser_action(parser, buffer->buffer))) { + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false); + break; + } + + buffer->len = 0; + buffer->buffer[0] = '\0'; + } + result = parser->user.data_collections_count; return result; } @@ -480,7 +511,7 @@ bool stop_streaming_receiver(RRDHOST *host, STREAM_HANDSHAKE reason) { shutdown(host->receiver->fd, SHUT_RDWR); } - netdata_thread_cancel(host->receiver->thread); + nd_thread_signal_cancel(host->receiver->thread); } int count = 2000; @@ -843,20 +874,18 @@ static void rrdpush_receive(struct receiver_state *rpt) ; } -static void rrdpush_receiver_thread_cleanup(void *ptr) { - struct receiver_state *rpt = (struct receiver_state *) ptr; - worker_unregister(); - - rrdhost_clear_receiver(rpt); +static void rrdpush_receiver_thread_cleanup(void *pptr) { + struct receiver_state *rpt = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!rpt) return; netdata_log_info("STREAM '%s' [receive from [%s]:%s]: " "receive thread ended (task id %d)" - , rpt->hostname ? rpt->hostname : "-" - , rpt->client_ip ? rpt->client_ip : "-", rpt->client_port ? rpt->client_port : "-" - , gettid()); + , rpt->hostname ? rpt->hostname : "-" + , rpt->client_ip ? rpt->client_ip : "-", rpt->client_port ? rpt->client_port : "-", gettid_cached()); + worker_unregister(); + rrdhost_clear_receiver(rpt); receiver_state_free(rpt); - rrdhost_set_is_parent_label(); } @@ -883,42 +912,37 @@ static bool stream_receiver_log_transport(BUFFER *wb, void *ptr) { } void *rrdpush_receiver_thread(void *ptr) { - netdata_thread_cleanup_push(rrdpush_receiver_thread_cleanup, ptr); - - { - worker_register("STREAMRCV"); + CLEANUP_FUNCTION_REGISTER(rrdpush_receiver_thread_cleanup) cleanup_ptr = ptr; + worker_register("STREAMRCV"); - worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_READ, - "received bytes", "bytes/s", - WORKER_METRIC_INCREMENT); + worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_READ, + "received bytes", "bytes/s", + WORKER_METRIC_INCREMENT); - worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, - "uncompressed bytes", "bytes/s", - WORKER_METRIC_INCREMENT); + worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, + "uncompressed bytes", "bytes/s", + WORKER_METRIC_INCREMENT); - worker_register_job_custom_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, - "replication completion", "%", - WORKER_METRIC_ABSOLUTE); + worker_register_job_custom_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, + "replication completion", "%", + WORKER_METRIC_ABSOLUTE); - struct receiver_state *rpt = (struct receiver_state *) ptr; - rpt->tid = gettid(); - - ND_LOG_STACK lgs[] = { - ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->client_ip), - ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->client_port), - ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rpt->hostname), - ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt), - ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt), - ND_LOG_FIELD_END(), - }; - ND_LOG_STACK_PUSH(lgs); + struct receiver_state *rpt = (struct receiver_state *) ptr; + rpt->tid = gettid_cached(); - netdata_log_info("STREAM %s [%s]:%s: receive thread started", rpt->hostname, rpt->client_ip - , rpt->client_port); + ND_LOG_STACK lgs[] = { + ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->client_ip), + ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->client_port), + ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rpt->hostname), + ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt), + ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); - rrdpush_receive(rpt); - } + netdata_log_info("STREAM %s [%s]:%s: receive thread started", rpt->hostname, rpt->client_ip + , rpt->client_port); - netdata_thread_cleanup_pop(1); + rrdpush_receive(rpt); return NULL; } diff --git a/src/streaming/replication.c b/src/streaming/replication.c index 6f68fedae7d722..1f5aeb34c09180 100644 --- a/src/streaming/replication.c +++ b/src/streaming/replication.c @@ -1036,7 +1036,7 @@ static struct replication_thread { struct { size_t last_executed; // caching of the atomic.executed to report number of requests executed since last time - netdata_thread_t **threads_ptrs; + ND_THREAD **threads_ptrs; size_t threads; } main_thread; // access is allowed only by the main thread @@ -1445,8 +1445,6 @@ static bool replication_execute_request(struct replication_request *rq, bool wor goto cleanup; } - netdata_thread_disable_cancelability(); - if(!rq->q) { if(likely(workers)) worker_is_busy(WORKER_JOB_PREPARE_QUERY); @@ -1468,7 +1466,6 @@ static bool replication_execute_request(struct replication_request *rq, bool wor rq->q, (size_t)((unsigned long long)rq->sender->host->sender->buffer->max_size * MAX_REPLICATION_MESSAGE_PERCENT_SENDER_BUFFER / 100ULL)); rq->q = NULL; - netdata_thread_enable_cancelability(); __atomic_add_fetch(&replication_globals.atomic.executed, 1, __ATOMIC_RELAXED); @@ -1830,43 +1827,44 @@ static int replication_pipeline_execute_next(void) { return REQUEST_OK; } -static void replication_worker_cleanup(void *ptr __maybe_unused) { +static void replication_worker_cleanup(void *pptr) { + if(CLEANUP_FUNCTION_GET_PTR(pptr) != (void *)0x01) return; replication_pipeline_cancel_and_cleanup(); worker_unregister(); } -static void *replication_worker_thread(void *ptr) { +static void *replication_worker_thread(void *ptr __maybe_unused) { + CLEANUP_FUNCTION_REGISTER(replication_worker_cleanup) cleanup_ptr = (void *)0x1; replication_initialize_workers(false); - netdata_thread_cleanup_push(replication_worker_cleanup, ptr) { - while (service_running(SERVICE_REPLICATION)) { - if (unlikely(replication_pipeline_execute_next() == REQUEST_QUEUE_EMPTY)) { - sender_thread_buffer_free(); - worker_is_busy(WORKER_JOB_WAIT); - worker_is_idle(); - sleep_usec(1 * USEC_PER_SEC); - } + while (service_running(SERVICE_REPLICATION)) { + if (unlikely(replication_pipeline_execute_next() == REQUEST_QUEUE_EMPTY)) { + sender_thread_buffer_free(); + worker_is_busy(WORKER_JOB_WAIT); + worker_is_idle(); + sleep_usec(1 * USEC_PER_SEC); } } - netdata_thread_cleanup_pop(1); + return NULL; } -static void replication_main_cleanup(void *ptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; +static void replication_main_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; replication_pipeline_cancel_and_cleanup(); int threads = (int)replication_globals.main_thread.threads; for(int i = 0; i < threads ;i++) { - netdata_thread_join(*replication_globals.main_thread.threads_ptrs[i], NULL); - freez(replication_globals.main_thread.threads_ptrs[i]); - __atomic_sub_fetch(&replication_buffers_allocated, sizeof(netdata_thread_t), __ATOMIC_RELAXED); + nd_thread_join(replication_globals.main_thread.threads_ptrs[i]); + __atomic_sub_fetch(&replication_buffers_allocated, sizeof(ND_THREAD *), __ATOMIC_RELAXED); } freez(replication_globals.main_thread.threads_ptrs); replication_globals.main_thread.threads_ptrs = NULL; - __atomic_sub_fetch(&replication_buffers_allocated, threads * sizeof(netdata_thread_t), __ATOMIC_RELAXED); + __atomic_sub_fetch(&replication_buffers_allocated, threads * sizeof(ND_THREAD *), __ATOMIC_RELAXED); aral_destroy(replication_globals.aral_rse); replication_globals.aral_rse = NULL; @@ -1894,20 +1892,20 @@ void *replication_thread_main(void *ptr __maybe_unused) { if(--threads) { replication_globals.main_thread.threads = threads; - replication_globals.main_thread.threads_ptrs = mallocz(threads * sizeof(netdata_thread_t *)); - __atomic_add_fetch(&replication_buffers_allocated, threads * sizeof(netdata_thread_t), __ATOMIC_RELAXED); + replication_globals.main_thread.threads_ptrs = mallocz(threads * sizeof(ND_THREAD *)); + __atomic_add_fetch(&replication_buffers_allocated, threads * sizeof(ND_THREAD *), __ATOMIC_RELAXED); for(int i = 0; i < threads ;i++) { char tag[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(tag, NETDATA_THREAD_TAG_MAX, "REPLAY[%d]", i + 2); - replication_globals.main_thread.threads_ptrs[i] = mallocz(sizeof(netdata_thread_t)); - __atomic_add_fetch(&replication_buffers_allocated, sizeof(netdata_thread_t), __ATOMIC_RELAXED); - netdata_thread_create(replication_globals.main_thread.threads_ptrs[i], tag, - NETDATA_THREAD_OPTION_JOINABLE, replication_worker_thread, NULL); + replication_globals.main_thread.threads_ptrs[i] = mallocz(sizeof(ND_THREAD *)); + __atomic_add_fetch(&replication_buffers_allocated, sizeof(ND_THREAD *), __ATOMIC_RELAXED); + replication_globals.main_thread.threads_ptrs[i] = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, + replication_worker_thread, NULL); } } - netdata_thread_cleanup_push(replication_main_cleanup, ptr); + CLEANUP_FUNCTION_REGISTER(replication_main_cleanup) cleanup_ptr = ptr; // start from 100% completed worker_set_metric(WORKER_JOB_CUSTOM_METRIC_COMPLETION, 100.0); @@ -2030,6 +2028,5 @@ void *replication_thread_main(void *ptr __maybe_unused) { } } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/streaming/rrdpush.c b/src/streaming/rrdpush.c index 874d4eb2fc3661..1ce8e4ea8436f5 100644 --- a/src/streaming/rrdpush.c +++ b/src/streaming/rrdpush.c @@ -192,7 +192,7 @@ int configured_as_parent() { appconfig_wrlock(&stream_config); for (section = stream_config.first_section; section; section = section->next) { - uuid_t uuid; + nd_uuid_t uuid; if (uuid_parse(section->name, uuid) != -1 && appconfig_get_boolean_by_section(section, "enabled", 0)) { @@ -610,6 +610,9 @@ int connect_to_one_of_destinations( for (struct rrdpush_destinations *d = host->destinations; d; d = d->next) { time_t now = now_realtime_sec(); + if(nd_thread_signaled_to_cancel()) + return -1; + if(d->postpone_reconnection_until > now) continue; @@ -718,7 +721,7 @@ void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wai host->sender->exit.reason = reason; // signal it to cancel - netdata_thread_cancel(host->rrdpush_sender_thread); + nd_thread_signal_cancel(host->rrdpush_sender_thread); } sender_unlock(host->sender); @@ -744,7 +747,9 @@ static void rrdpush_sender_thread_spawn(RRDHOST *host) { char tag[NETDATA_THREAD_TAG_MAX + 1]; snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_SENDER "[%s]", rrdhost_hostname(host)); - if(netdata_thread_create(&host->rrdpush_sender_thread, tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_sender_thread, (void *) host->sender)) + host->rrdpush_sender_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, + rrdpush_sender_thread, (void *)host->sender); + if(!host->rrdpush_sender_thread) nd_log_daemon(NDLP_ERR, "STREAM %s [send]: failed to create new thread for client.", rrdhost_hostname(host)); else rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN); @@ -795,7 +800,7 @@ static void rrdpush_receiver_takeover_web_connection(struct web_client *w, struc } void *rrdpush_receiver_thread(void *ptr); -int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx) { +int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx __maybe_unused) { if(!service_running(ABILITY_STREAMING_CONNECTIONS)) return rrdpush_receiver_too_busy_now(w); @@ -1155,7 +1160,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri } netdata_mutex_unlock(&host->receiver_lock); } - rrd_unlock(); + rrd_rdunlock(); if (receiver_stale && stop_streaming_receiver(host, STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER)) { // we stopped the receiver @@ -1197,7 +1202,8 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_RECEIVER "[%s]", rpt->hostname); tag[NETDATA_THREAD_TAG_MAX] = '\0'; - if(netdata_thread_create(&rpt->thread, tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_receiver_thread, (void *)rpt)) { + rpt->thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_receiver_thread, (void *)rpt); + if(!rpt->thread) { rrdpush_receive_log_status( rpt, "can't create receiver thread", RRDPUSH_STATUS_INTERNAL_SERVER_ERROR, NDLP_ERR); diff --git a/src/streaming/rrdpush.h b/src/streaming/rrdpush.h index c7387713455bba..d55a07675e4683 100644 --- a/src/streaming/rrdpush.h +++ b/src/streaming/rrdpush.h @@ -201,7 +201,7 @@ typedef enum __attribute__((packed)) { struct sender_state { RRDHOST *host; - pid_t tid; // the thread id of the sender, from gettid() + pid_t tid; // the thread id of the sender, from gettid_cached() SENDER_FLAGS flags; int timeout; int default_port; @@ -337,7 +337,7 @@ typedef struct stream_node_instance { struct receiver_state { RRDHOST *host; pid_t tid; - netdata_thread_t thread; + ND_THREAD *thread; int fd; char *key; char *hostname; diff --git a/src/streaming/sender.c b/src/streaming/sender.c index bb617c5fd74ec1..bae2af795e3443 100644 --- a/src/streaming/sender.c +++ b/src/streaming/sender.c @@ -883,9 +883,8 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p return false; } - ssize_t bytes, len = (ssize_t)strlen(http); - - bytes = send_timeout( + ssize_t len = (ssize_t)strlen(http); + ssize_t bytes = send_timeout( #ifdef ENABLE_HTTPS &host->sender->ssl, #endif @@ -1015,8 +1014,10 @@ static bool attempt_to_connect(struct sender_state *state) { usec_t now_ut = now_monotonic_usec(); usec_t end_ut = now_ut + USEC_PER_SEC * state->reconnect_delay; while(now_ut < end_ut) { - netdata_thread_testcancel(); - sleep_usec(500 * USEC_PER_MS); // seconds + if(nd_thread_signaled_to_cancel()) + return false; + + sleep_usec(100 * USEC_PER_MS); // seconds now_ut = now_monotonic_usec(); } @@ -1386,7 +1387,7 @@ static bool rrdpush_sender_pipe_close(RRDHOST *host, int *pipe_fds, bool reopen) } void rrdpush_signal_sender_to_wake_up(struct sender_state *s) { - if(unlikely(s->tid == gettid())) + if(unlikely(s->tid == gettid_cached())) return; RRDHOST *host = s->host; @@ -1409,7 +1410,7 @@ static bool rrdhost_set_sender(RRDHOST *host) { rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED | RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS); rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN); host->rrdpush_sender_connection_counter++; - host->sender->tid = gettid(); + host->sender->tid = gettid_cached(); host->sender->last_state_since_t = now_realtime_sec(); host->sender->exit.reason = STREAM_HANDSHAKE_NEVER; ret = true; @@ -1424,7 +1425,7 @@ static bool rrdhost_set_sender(RRDHOST *host) { static void rrdhost_clear_sender___while_having_sender_mutex(RRDHOST *host) { if(unlikely(!host->sender)) return; - if(host->sender->tid == gettid()) { + if(host->sender->tid == gettid_cached()) { host->sender->tid = 0; host->sender->exit.shutdown = false; rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN | RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED | RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS); @@ -1439,8 +1440,11 @@ static void rrdhost_clear_sender___while_having_sender_mutex(RRDHOST *host) { } static bool rrdhost_sender_should_exit(struct sender_state *s) { - // check for outstanding cancellation requests - netdata_thread_testcancel(); + if(unlikely(nd_thread_signaled_to_cancel())) { + if(!s->exit.reason) + s->exit.reason = STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN; + return true; + } if(unlikely(!service_running(SERVICE_STREAMING))) { if(!s->exit.reason) @@ -1469,8 +1473,10 @@ static bool rrdhost_sender_should_exit(struct sender_state *s) { return false; } -static void rrdpush_sender_thread_cleanup_callback(void *ptr) { - struct rrdpush_sender_thread_data *s = ptr; +static void rrdpush_sender_thread_cleanup_callback(void *pptr) { + struct rrdpush_sender_thread_data *s = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!s) return; + worker_unregister(); RRDHOST *host = s->host; @@ -1615,19 +1621,19 @@ void *rrdpush_sender_thread(void *ptr) { !*s->host->rrdpush_send_destination || !s->host->rrdpush_send_api_key || !*s->host->rrdpush_send_api_key) { netdata_log_error("STREAM %s [send]: thread created (task id %d), but host has streaming disabled.", - rrdhost_hostname(s->host), gettid()); + rrdhost_hostname(s->host), gettid_cached()); return NULL; } if(!rrdhost_set_sender(s->host)) { netdata_log_error("STREAM %s [send]: thread created (task id %d), but there is another sender running for this host.", - rrdhost_hostname(s->host), gettid()); + rrdhost_hostname(s->host), gettid_cached()); return NULL; } rrdpush_initialize_ssl_ctx(s->host); - netdata_log_info("STREAM %s [send]: thread created (task id %d)", rrdhost_hostname(s->host), gettid()); + netdata_log_info("STREAM %s [send]: thread created (task id %d)", rrdhost_hostname(s->host), gettid_cached()); s->timeout = (int)appconfig_get_number( &stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 600); @@ -1670,7 +1676,7 @@ void *rrdpush_sender_thread(void *ptr) { thread_data->pipe_buffer = mallocz(pipe_buffer_size); thread_data->host = s->host; - netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, thread_data); + CLEANUP_FUNCTION_REGISTER(rrdpush_sender_thread_cleanup_callback) cleanup_ptr = thread_data; size_t iterations = 0; time_t now_s = now_monotonic_sec(); @@ -1794,7 +1800,6 @@ void *rrdpush_sender_thread(void *ptr) { // Spurious wake-ups without error - loop again if (poll_rc == 0 || ((poll_rc == -1) && (errno == EAGAIN || errno == EINTR))) { - netdata_thread_testcancel(); netdata_log_debug(D_STREAM, "Spurious wakeup"); now_s = now_monotonic_sec(); continue; @@ -1888,6 +1893,5 @@ void *rrdpush_sender_thread(void *ptr) { worker_set_metric(WORKER_SENDER_JOB_REPLAY_DICT_SIZE, (NETDATA_DOUBLE) dictionary_entries(s->replication.requests)); } - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/web/api/badges/web_buffer_svg.c b/src/web/api/badges/web_buffer_svg.c index 23dc96d10120cc..747c46d5eeba40 100644 --- a/src/web/api/badges/web_buffer_svg.c +++ b/src/web/api/badges/web_buffer_svg.c @@ -255,7 +255,7 @@ static inline char *format_value_with_precision_and_unit(char *value_string, siz value = 0.0; char *separator = ""; - if(unlikely(isalnum(*units))) + if(unlikely(isalnum((uint8_t)*units))) separator = " "; if(precision < 0) { diff --git a/src/web/api/formatters/charts2json.c b/src/web/api/formatters/charts2json.c index cab4debaec1377..2a3deeae31a403 100644 --- a/src/web/api/formatters/charts2json.c +++ b/src/web/api/formatters/charts2json.c @@ -96,7 +96,7 @@ void charts2json(RRDHOST *host, BUFFER *wb) { buffer_json_object_close(wb); } } - rrd_unlock(); + rrd_rdunlock(); } buffer_json_array_close(wb); diff --git a/src/web/api/http_auth.c b/src/web/api/http_auth.c index 3afab12e9f8eda..ec0520304fe978 100644 --- a/src/web/api/http_auth.c +++ b/src/web/api/http_auth.c @@ -8,7 +8,7 @@ bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at static DICTIONARY *netdata_authorized_bearers = NULL; struct bearer_token { - uuid_t cloud_account_id; + nd_uuid_t cloud_account_id; char cloud_user_name[CLOUD_USER_NAME_LENGTH]; HTTP_ACCESS access; HTTP_USER_ROLE user_role; @@ -59,7 +59,7 @@ void bearer_tokens_init(void) { NULL, sizeof(struct bearer_token)); } -time_t bearer_create_token(uuid_t *uuid, struct web_client *w) { +time_t bearer_create_token(nd_uuid_t *uuid, struct web_client *w) { char uuid_str[UUID_COMPACT_STR_LEN]; uuid_generate_random(*uuid); diff --git a/src/web/api/http_auth.h b/src/web/api/http_auth.h index ef06728f21ee19..f339a44cf0d26e 100644 --- a/src/web/api/http_auth.h +++ b/src/web/api/http_auth.h @@ -11,7 +11,7 @@ extern bool netdata_is_protected_by_bearer; bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len); -time_t bearer_create_token(uuid_t *uuid, struct web_client *w); +time_t bearer_create_token(nd_uuid_t *uuid, struct web_client *w); bool web_client_bearer_token_auth(struct web_client *w, const char *v); static inline bool http_access_user_has_enough_access_level_for_endpoint(HTTP_ACCESS user, HTTP_ACCESS endpoint) { diff --git a/src/web/api/http_header.c b/src/web/api/http_header.c index 5b0b1b651d822d..983d2436653ef0 100644 --- a/src/web/api/http_header.c +++ b/src/web/api/http_header.c @@ -170,7 +170,7 @@ static void http_header_x_netdata_auth(struct web_client *w, const char *v, size if(strncasecmp(v, "Bearer ", 7) == 0) { v = &v[7]; - while(*v && isspace(*v)) v++; + while(*v && isspace((uint8_t)*v)) v++; web_client_bearer_token_auth(w, v); } } diff --git a/src/web/api/queries/countif/countif.h b/src/web/api/queries/countif/countif.h index 896b9d873ae449..af204a95af06d0 100644 --- a/src/web/api/queries/countif/countif.h +++ b/src/web/api/queries/countif/countif.h @@ -28,7 +28,7 @@ static inline void tg_countif_create(RRDR *r, const char *options __maybe_unused if(options && *options) { // skip any leading spaces - while(isspace(*options)) options++; + while(isspace((uint8_t)*options)) options++; // find the comparison function switch(*options) { @@ -73,7 +73,7 @@ static inline void tg_countif_create(RRDR *r, const char *options __maybe_unused if(*options) options++; // skip everything up to the first digit - while(isspace(*options)) options++; + while(isspace((uint8_t)*options)) options++; g->target = str2ndd(options, NULL); } diff --git a/src/web/api/queries/weights.h b/src/web/api/queries/weights.h index a93519b6fed8a9..be7e5a8b3f6f3a 100644 --- a/src/web/api/queries/weights.h +++ b/src/web/api/queries/weights.h @@ -58,7 +58,7 @@ typedef struct query_weights_request { weights_interrupt_callback_t interrupt_callback; void *interrupt_callback_data; - uuid_t *transaction; + nd_uuid_t *transaction; } QUERY_WEIGHTS_REQUEST; int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr); diff --git a/src/web/api/web_api.h b/src/web/api/web_api.h index 6d30630dc8fd5f..634e59657a1c88 100644 --- a/src/web/api/web_api.h +++ b/src/web/api/web_api.h @@ -28,7 +28,7 @@ static inline void fix_google_param(char *s) { if(unlikely(!s || !*s)) return; for( ; *s ;s++) { - if(!isalnum(*s) && *s != '.' && *s != '_' && *s != '-') + if(!isalnum((uint8_t)*s) && *s != '.' && *s != '_' && *s != '-') *s = '_'; } } diff --git a/src/web/api/web_api_v1.c b/src/web/api/web_api_v1.c index 386221d6198c7b..1d197697ca91ee 100644 --- a/src/web/api/web_api_v1.c +++ b/src/web/api/web_api_v1.c @@ -142,7 +142,7 @@ void web_client_api_v1_init(void) { time_grouping_init(); - uuid_t uuid; + nd_uuid_t uuid; // generate uuid_generate(uuid); @@ -181,7 +181,7 @@ char *get_mgmt_api_key(void) { // generate a new one? if(!guid[0]) { - uuid_t uuid; + nd_uuid_t uuid; uuid_generate_time(uuid); uuid_unparse_lower(uuid, guid); @@ -1233,7 +1233,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) { } buffer_json_array_close(wb); - rrd_unlock(); + rrd_rdunlock(); } void host_labels2json(RRDHOST *host, BUFFER *wb, const char *key) { diff --git a/src/web/api/web_api_v2.c b/src/web/api/web_api_v2.c index 4f24e8c6806f77..c62ed9ed37cf22 100644 --- a/src/web/api/web_api_v2.c +++ b/src/web/api/web_api_v2.c @@ -99,7 +99,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may return HTTP_RESP_BAD_REQUEST; } - uuid_t uuid; + nd_uuid_t uuid; time_t expires_s = bearer_create_token(&uuid, w); BUFFER *wb = w->response.data; @@ -575,7 +575,7 @@ static int web_client_api_request_v2_progress(RRDHOST *host __maybe_unused, stru if(!strcmp(name, "transaction")) transaction = value; } - uuid_t tr; + nd_uuid_t tr; uuid_parse_flexi(transaction, tr); rrd_function_call_progresser(&tr); diff --git a/src/web/rtc/webrtc.c b/src/web/rtc/webrtc.c index 027b168b3a84bd..eb73f0b5e45f5d 100644 --- a/src/web/rtc/webrtc.c +++ b/src/web/rtc/webrtc.c @@ -366,7 +366,7 @@ static void myOpenCallback(int id __maybe_unused, void *user_ptr) { WEBRTC_DC *chan = user_ptr; internal_fatal(chan->dc != id, "WEBRTC[%d],DC[%d]: dc mismatch, expected %d, got %d", chan->conn->pc, chan->dc, chan->dc, id); - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' OPEN", chan->conn->pc, chan->dc, gettid(), chan->label); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' OPEN", chan->conn->pc, chan->dc, gettid_cached(), chan->label); internal_error(true, "WEBRTC[%d],DC[%d]: data channel opened.", chan->conn->pc, chan->dc); chan->open = true; } @@ -384,7 +384,7 @@ static void myClosedCallback(int id __maybe_unused, void *user_ptr) { DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(chan->conn->channels.head, chan, link.prev, link.next); spinlock_unlock(&chan->conn->channels.spinlock); - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' CLOSED", chan->conn->pc, chan->dc, gettid(), chan->label); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' CLOSED", chan->conn->pc, chan->dc, gettid_cached(), chan->label); freez(chan->label); freez(chan); @@ -566,27 +566,27 @@ static void myStateChangeCallback(int pc __maybe_unused, rtcState state, void *u break; case RTC_CONNECTING: - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTING", conn->pc, gettid()); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTING", conn->pc, gettid_cached()); internal_error(true, "WEBRTC[%d]: connecting...", conn->pc); break; case RTC_CONNECTED: - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTED", conn->pc, gettid()); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTED", conn->pc, gettid_cached()); internal_error(true, "WEBRTC[%d]: connected!", conn->pc); break; case RTC_DISCONNECTED: - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d DISCONNECTED", conn->pc, gettid()); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d DISCONNECTED", conn->pc, gettid_cached()); internal_error(true, "WEBRTC[%d]: disconnected.", conn->pc); break; case RTC_FAILED: - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTION FAILED", conn->pc, gettid()); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTION FAILED", conn->pc, gettid_cached()); internal_error(true, "WEBRTC[%d]: failed.", conn->pc); break; case RTC_CLOSED: - nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTION CLOSED", conn->pc, gettid()); + nd_log(NDLS_ACCESS, NDLP_DEBUG, "WEBRTC[%d]: %d CONNECTION CLOSED", conn->pc, gettid_cached()); internal_error(true, "WEBRTC[%d]: closed.", conn->pc); spinlock_lock(&webrtc_base.unsafe.spinlock); webrtc_destroy_connection_unsafe(conn); diff --git a/src/web/server/h2o/http_server.c b/src/web/server/h2o/http_server.c index 96bc91192ea86b..a079c6afe1a517 100644 --- a/src/web/server/h2o/http_server.c +++ b/src/web/server/h2o/http_server.c @@ -363,8 +363,6 @@ void *h2o_main(void *ptr) { h2o_pathconf_t *pathconf; h2o_hostconf_t *hostconf; - netdata_thread_disable_cancelability(); - const char *bind_addr = config_get(HTTPD_CONFIG_SECTION, "bind to", "127.0.0.1"); int bind_port = config_get_number(HTTPD_CONFIG_SECTION, "port", 19998); diff --git a/src/web/server/static/static-threaded.c b/src/web/server/static/static-threaded.c index f2a07e405f3bfd..a4b24c9ac74035 100644 --- a/src/web/server/static/static-threaded.c +++ b/src/web/server/static/static-threaded.c @@ -61,7 +61,7 @@ static struct web_client *web_client_create_on_fd(POLLINFO *pi) { // the main socket listener - STATIC-THREADED struct web_server_static_threaded_worker { - netdata_thread_t thread; + ND_THREAD *thread; int id; int running; @@ -394,8 +394,9 @@ static int web_server_snd_callback(POLLINFO *pi, short int *events) { // ---------------------------------------------------------------------------- // web server worker thread -static void socket_listen_main_static_threaded_worker_cleanup(void *ptr) { - worker_private = (struct web_server_static_threaded_worker *)ptr; +static void socket_listen_main_static_threaded_worker_cleanup(void *pptr) { + worker_private = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!worker_private) return; netdata_log_info("stopped after %zu connects, %zu disconnects (max concurrent %zu), %zu receptions and %zu sends", worker_private->connected, @@ -414,7 +415,7 @@ static bool web_server_should_stop(void) { } void *socket_listen_main_static_threaded_worker(void *ptr) { - worker_private = (struct web_server_static_threaded_worker *)ptr; + worker_private = ptr; worker_private->running = 1; worker_register("WEB"); worker_register_job_name(WORKER_JOB_ADD_CONNECTION, "connect"); @@ -427,26 +428,24 @@ void *socket_listen_main_static_threaded_worker(void *ptr) { worker_register_job_name(WORKER_JOB_SND_DATA, "send"); worker_register_job_name(WORKER_JOB_PROCESS, "process"); - netdata_thread_cleanup_push(socket_listen_main_static_threaded_worker_cleanup, ptr); - - poll_events(&api_sockets - , web_server_add_callback - , web_server_del_callback - , web_server_rcv_callback - , web_server_snd_callback - , NULL - , web_server_should_stop - , web_allow_connections_from - , web_allow_connections_dns - , NULL - , web_client_first_request_timeout - , web_client_timeout - , default_rrd_update_every * 1000 // timer_milliseconds - , ptr // timer_data - , worker_private->max_sockets - ); - - netdata_thread_cleanup_pop(1); + CLEANUP_FUNCTION_REGISTER(socket_listen_main_static_threaded_worker_cleanup) cleanup_ptr = worker_private; + poll_events(&api_sockets + , web_server_add_callback + , web_server_del_callback + , web_server_rcv_callback + , web_server_snd_callback + , NULL + , web_server_should_stop + , web_allow_connections_from + , web_allow_connections_dns + , NULL + , web_client_first_request_timeout + , web_client_timeout + , default_rrd_update_every * 1000 // timer_milliseconds + , ptr // timer_data + , worker_private->max_sockets + ); + return NULL; } @@ -454,8 +453,10 @@ void *socket_listen_main_static_threaded_worker(void *ptr) { // ---------------------------------------------------------------------------- // web server main thread - also becomes a worker -static void socket_listen_main_static_threaded_cleanup(void *ptr) { - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; +static void socket_listen_main_static_threaded_cleanup(void *pptr) { + struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); + if(!static_thread) return; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; // int i, found = 0; @@ -466,7 +467,7 @@ static void socket_listen_main_static_threaded_cleanup(void *ptr) { // if(static_workers_private_data[i].running) { // found++; // netdata_log_info("stopping worker %d", i + 1); -// netdata_thread_cancel(static_workers_private_data[i].thread); +// nd_thread_signal_cancel(static_workers_private_data[i].thread); // } // else // netdata_log_info("found stopped worker %d", i + 1); @@ -496,7 +497,7 @@ static void socket_listen_main_static_threaded_cleanup(void *ptr) { } void *socket_listen_main_static_threaded(void *ptr) { - netdata_thread_cleanup_push(socket_listen_main_static_threaded_cleanup, ptr); + CLEANUP_FUNCTION_REGISTER(socket_listen_main_static_threaded_cleanup) cleanup_ptr = ptr; web_server_mode = WEB_SERVER_MODE_STATIC_THREADED; if(!api_sockets.opened) @@ -548,14 +549,14 @@ void *socket_listen_main_static_threaded(void *ptr) { snprintfz(tag, sizeof(tag) - 1, "WEB[%d]", i+1); netdata_log_info("starting worker %d", i+1); - netdata_thread_create(&static_workers_private_data[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, - socket_listen_main_static_threaded_worker, (void *)&static_workers_private_data[i]); + static_workers_private_data[i].thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, + socket_listen_main_static_threaded_worker, + (void *)&static_workers_private_data[i]); } // and the main one static_workers_private_data[0].max_sockets = max_sockets / static_threaded_workers_count; socket_listen_main_static_threaded_worker((void *)&static_workers_private_data[0]); - netdata_thread_cleanup_pop(1); return NULL; } diff --git a/src/web/server/web_client.c b/src/web/server/web_client.c index 4020d701f44a92..d0fe544e5c9349 100644 --- a/src/web/server/web_client.c +++ b/src/web/server/web_client.c @@ -130,7 +130,7 @@ static inline char *strip_control_characters(char *url) { if(!url) return ""; for(char *s = url; *s ;s++) - if(iscntrl(*s)) *s = ' '; + if(iscntrl((uint8_t)*s)) *s = ' '; return url; } @@ -479,7 +479,7 @@ static int mysendfile(struct web_client *w, char *filename) { // if the filename contains "strange" characters, refuse to serve it char *s; for(s = filename; *s ;s++) { - if( !isalnum(*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') { + if( !isalnum((uint8_t)*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') { netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename); w->response.data->content_type = CT_TEXT_HTML; buffer_sprintf(w->response.data, "Filename contains invalid characters: "); @@ -1065,7 +1065,7 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch if(!host) { // we didn't find it, but it may be a uuid case mismatch for MACHINE_GUID // so, recreate the machine guid in lower-case. - uuid_t uuid; + nd_uuid_t uuid; char txt[UUID_STR_LEN]; if (uuid_parse(tok, uuid) == 0) { uuid_unparse_lower(uuid, txt); diff --git a/src/web/server/web_client.h b/src/web/server/web_client.h index 90e19e8a6c428d..650ddb3eb55210 100644 --- a/src/web/server/web_client.h +++ b/src/web/server/web_client.h @@ -165,7 +165,7 @@ struct web_client { unsigned long long id; size_t use_count; - uuid_t transaction; + nd_uuid_t transaction; WEB_CLIENT_FLAGS flags; // status flags for the client HTTP_REQUEST_MODE mode; // the operational mode of the client @@ -207,8 +207,8 @@ struct web_client { #endif struct { - uuid_t bearer_token; - uuid_t cloud_account_id; + nd_uuid_t bearer_token; + nd_uuid_t cloud_account_id; char client_name[CLOUD_USER_NAME_LENGTH]; } auth;