From 477602b18c82111ef9d94b2b91475c5d506d4be1 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 7 Sep 2022 13:42:35 +0200 Subject: [PATCH 1/5] tests: mark failing tests "broken_with_enabled_sanitzers" --- core/src/tests/CMakeLists.txt | 5 +++++ systemtests/tests/reload/CMakeLists.txt | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/core/src/tests/CMakeLists.txt b/core/src/tests/CMakeLists.txt index 3c067858dca..0a9d69c731f 100644 --- a/core/src/tests/CMakeLists.txt +++ b/core/src/tests/CMakeLists.txt @@ -278,6 +278,10 @@ if(NOT client-only) test_config_parser_dir LINK_LIBRARIES dird_objects bareos bareosfind bareossql GTest::gtest_main ) + set_tests_properties( + gtest:ConfigParser_Dir.bareos_configparser_tests + PROPERTIES LABELS broken_with_enabled_sanitzers + ) bareos_add_test( test_crc32 ADDITIONAL_SOURCES ../stored/crc32/crc32.cc @@ -399,6 +403,7 @@ bareos_add_test( ) bareos_add_test(thread_list LINK_LIBRARIES bareos GTest::gtest_main) + set_tests_properties( gtest:thread_list.thread_list_startup_and_shutdown PROPERTIES LABELS broken ) diff --git a/systemtests/tests/reload/CMakeLists.txt b/systemtests/tests/reload/CMakeLists.txt index cfd54bc5868..581c570d904 100644 --- a/systemtests/tests/reload/CMakeLists.txt +++ b/systemtests/tests/reload/CMakeLists.txt @@ -19,3 +19,15 @@ get_filename_component(BASENAME ${CMAKE_CURRENT_BINARY_DIR} NAME) create_systemtest(${SYSTEMTEST_PREFIX} ${BASENAME}) + +get_property( + test_names + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + PROPERTY TESTS +) +set_tests_properties( + system:reload:add-second-director system:reload:add-uncommented-string + system:reload:unchanged-config PROPERTIES LABELS + broken_with_enabled_sanitzers +) +# message(FATAL_ERROR ${test_names}) From b99efecb25071dce97aa3c4a053d12caaf2f6743 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 7 Sep 2022 13:43:05 +0200 Subject: [PATCH 2/5] mntent_cache.cc: fix memleak when dev number is not unique The code before did not check for the return value of binary_insert(), so that the failed inserts were not freed. --- core/src/lib/mntent_cache.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/lib/mntent_cache.cc b/core/src/lib/mntent_cache.cc index a685dc73b60..5799c67a354 100644 --- a/core/src/lib/mntent_cache.cc +++ b/core/src/lib/mntent_cache.cc @@ -102,7 +102,7 @@ static time_t last_rescan = 0; static const char* skipped_fs_types[] = { #if defined(HAVE_LINUX_OS) - "rootfs", + "rootfs", // "nsfs", "tmpfs", #endif NULL}; @@ -152,8 +152,14 @@ static mntent_cache_entry_t* add_mntent_mapping(uint32_t dev, mce->fstype = strdup(fstype); if (mntopts) { mce->mntopts = strdup(mntopts); } - mntent_cache_entries->binary_insert(mce, CompareMntentMapping); + auto retval = mntent_cache_entries->binary_insert(mce, CompareMntentMapping); + if (retval != mce) { + Dmsg1(0, "failed to insert: %s %p, already exists!\n", mce->mountpoint, + mce->mountpoint); + DestroyMntentCacheEntry(mce); + free(mce); + } return mce; } From 3fce29f96134a89cb335535eec1b0aa40b241ff6 Mon Sep 17 00:00:00 2001 From: Bruno Friedmann Date: Fri, 9 Sep 2022 11:16:37 +0200 Subject: [PATCH 3/5] activate multithread in dist-tarball.sh Signed-off-by: Bruno Friedmann --- devtools/dist-tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/dist-tarball.sh b/devtools/dist-tarball.sh index a360ace69bc..1f5806ffcda 100755 --- a/devtools/dist-tarball.sh +++ b/devtools/dist-tarball.sh @@ -123,7 +123,7 @@ fi (echo -ne 'cmake/BareosVersion.cmake\0'; "$git" ls-files -z) | \ "$sort" -u -z | \ "$tar" "${args[@]}" -cf - --files-from - | \ -"$xz" -c -6 > "${archive_file}" +"$xz" --threads=0 -c -6 > "${archive_file}" echo -n "Wrote tarball: " "${list_file_cmd[@]}" "${archive_file}" From b6fd36b37af676a873f81fbe7a2a65ae1be34d4d Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 7 Sep 2022 13:41:59 +0200 Subject: [PATCH 4/5] .matrix.yml: add sanitize stage Add a stage with F37 building with a custom script, enabling sanitizers and running tests. Also cleanup the .matrix.yml a bit. Co-Authored-By: Andreas Rogge --- .matrix.yml | 29 ++++++------------- devtools/build-and-test-with-sanitize.sh | 37 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 20 deletions(-) create mode 100755 devtools/build-and-test-with-sanitize.sh diff --git a/.matrix.yml b/.matrix.yml index 273d9678532..7cf07eccd93 100644 --- a/.matrix.yml +++ b/.matrix.yml @@ -3,6 +3,14 @@ JENKINS_TESTTYPES: - univention-bareos OS: + Sanitize: + "F37": + TYPE: scripted + IMAGE: fedora37 + BUILD_SCRIPT: bareos/devtools/build-and-test-with-sanitize.sh + FINISH_SCRIPT: CD/sanitize/finish-sanitize.sh + ARCH: + - x86_64 ULC_deb: "OpenSSL_1.1": TYPE: scripted @@ -11,9 +19,6 @@ OS: FINISH_SCRIPT: CD/deb/finish-ulc.sh ARCH: - x86_64 - PROJECTPACKAGES: - x86_64: - - bareos xUbuntu: "22.04": @@ -220,28 +225,19 @@ OS: TYPE: freebsd ARCH: - amd64 - PROJECTPACKAGES: - amd64: - - bareos-freebsd MacOS: "10.13.6": TYPE: macos ARCH: - x64 - PROJECTPACKAGES: - x64: + Solaris: "11.4": TYPE: solaris ARCH: - i386 - sparc - PROJECTPACKAGES: - i386: - - bareos - sparc: - - bareos win: cross: @@ -251,13 +247,6 @@ OS: - win-64-release ARCH: - x86_64 - PROJECTPACKAGES: - x86_64: - - mingw-debugsrc - - mingw32-winbareos - - mingw64-winbareos - - winbareos-nsi - - winbareos-opsi WINVERSIONS: - windows-10-32 - windows-10-64 diff --git a/devtools/build-and-test-with-sanitize.sh b/devtools/build-and-test-with-sanitize.sh new file mode 100755 index 00000000000..53460ec7e0d --- /dev/null +++ b/devtools/build-and-test-with-sanitize.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -u +set -e +set -x +shopt -s nullglob + +# make sure we're in bareos's toplevel dir +if [ ! -f core/src/include/bareos.h ]; then + echo "$0: Invoke from Bareos' toplevel directory" >&2 + exit 2 +fi + +nproc="$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)" + +if [ -z ${CTEST_PARALLEL_LEVEL+x} ]; then + export CTEST_PARALLEL_LEVEL="$nproc" +fi +if [ -z ${CMAKE_BUILD_PARALLEL_LEVEL+x} ]; then + export CMAKE_BUILD_PARALLEL_LEVEL="$nproc" +fi + +rm -rf cmake-build + +cmake \ + -S . \ + -B cmake-build \ + -DENABLE_SANITIZERS=yes \ + -Dpostgresql=yes +cmake --build cmake-build + +cd cmake-build +export REGRESS_DEBUG=1 +ctest \ + --script CTestScript.cmake \ + --verbose \ + --label-exclude "broken.*" \ + || echo "ctest failed" From c9f80b8d7e582bb9af088693f7a7eeaefcff091b Mon Sep 17 00:00:00 2001 From: Andreas Rogge Date: Tue, 4 Oct 2022 12:10:09 +0200 Subject: [PATCH 5/5] Update CHANGELOG.md --- CHANGELOG.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2266c6030c2..568a83219b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,10 +45,11 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: - core/platform: Adding Bareos firewalld service xml files [PR #1237] - dird: Added `FS Type = vfat` in LinuxAll.conf for UEFI partition [PR #1236] - bareos tools: reintegrate testfind binary [PR #1176] -- fd: add support for role switching on PostgreSQL add-on [BUG #4607] [PR #1178] +- fd: add support for role switching on PostgreSQL add-on [Issue #4607] [PR #1178] - build: switch from FreeBSD 13.0 to 13.1 [PR #1253] -- stored: dird: added backup checkpoints that save backup metadata to the Catalog during the execution of the backup. [PR#1074] +- stored: dird: added backup checkpoints that save backup metadata to the Catalog during the execution of the backup. [PR #1074] - stored: dird: add backup checkpoints that save backup metadata to the Catalog during the execution of the backup. [PR #1074] +- build: run a build and test with sanitizers enabled [PR #1244] ### Fixed - dird: RunScript fixes [PR #1217] @@ -82,7 +83,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: - cats: make `.bvfs_update` and `.bvfs_versions` take archive jobs into consideration [PR #1152] - Fix `always-incremental-consolidate` systemtest sporadic fails, and rename it. [PR #1154] - packaging: FreeBSD place all scripts into "normal" location /usr/local/lib/bareos/scripts [PR #1163] -- [BUG #1445] adding quotes to director name when using `configure export`. [PR #1171] +- [Issue #1445] adding quotes to director name when using `configure export`. [PR #1171] - FreeBSD packages: add missing ddl/update 2171_2192 and 2192_2210 files [PR #1147] - Fix director connects to client while `Connection From Director To Client` is disabled. [PR #1099] - dir: miscalculation when using `always incremental keep number` [PR #1159] @@ -93,7 +94,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: - devtools/dist-tarball.sh: fix name if version contains "~pre" [PR #1221] - dird: fix odd-even weeks parsing bug in schedule [PR #1210] - bcopy: fix crash in bcopy when using certain cli arguments [PR #1211] -- webui: fix password string length limitation [BUG #1480][PR #1251] +- webui: fix password string length limitation [Issue #1480][PR #1251] - systemtest: fixed issues with systemtests not succeeding on first try [PR #1186] - btape: dumplabel only when label is valid [PR #1266] @@ -155,10 +156,10 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: - split the very long `Plugins.rst` file into one file per Bareos plugin [PR #1046] - rework SD plugin scsicrypto linux sg_io ioctl subsection for cap_sys_rawio [PR #1057] - improve action Python plugin documentation, by removing File in Fileset example [PR #1079] -- improve Mysql - PostgreSQL howto [PR #1093] fixing [BUG #1429] +- improve Mysql - PostgreSQL howto [PR #1093] fixing [Issue #1429] - clarifies Sphinx bareos-extension parallel_read_safe status to False [PR #1037] -- fix incorrect link in contrib PythonFdPlugin [BUG #1450] [PR #1065] -- clarifies CheckFileChanges option not intended to be used with plugin [BUG #1452][PR #1180] +- fix incorrect link in contrib PythonFdPlugin [Issue #1450] [PR #1065] +- clarifies CheckFileChanges option not intended to be used with plugin [Issue #1452][PR #1180] - fix broken links with sphinx linkcheck [PR #1200] - add Security FIPS section [PR #1181] - add instruction to run sphinx linkcheck before release [PR #1218] @@ -167,7 +168,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: - improve Wild(*) section with links to example [PR #1228] - add example to selection type = sql query [PR #1229] - dir-job-RescheduleTimes remove wrong default sentence [PR #1225] -- update Appendix/HardwareSizing DB size with new numbers and formulas [BUG #1477][PR #1231] +- update Appendix/HardwareSizing DB size with new numbers and formulas [Issue #1477][PR #1231] - add description to fileset signature sha256 and sha512 parameter [PR #1230] - improve troubleshooting and debugging chapter [PR #1233] - mssql: after PITR restore, a full backup is required [PR #1235] @@ -182,6 +183,13 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: - obsolete comments removed [PR #1268] - fix bsmtp get-usage.sh call [PR #1267] +[Issue #1429]: https://bugs.bareos.org/view.php?id=1429 +[Issue #1445]: https://bugs.bareos.org/view.php?id=1445 +[Issue #1450]: https://bugs.bareos.org/view.php?id=1450 +[Issue #1452]: https://bugs.bareos.org/view.php?id=1452 +[Issue #1477]: https://bugs.bareos.org/view.php?id=1477 +[Issue #1480]: https://bugs.bareos.org/view.php?id=1480 +[Issue #4607]: https://bugs.bareos.org/view.php?id=4607 [PR #698]: https://github.com/bareos/bareos/pull/698 [PR #768]: https://github.com/bareos/bareos/pull/768 [PR #1010]: https://github.com/bareos/bareos/pull/1010 @@ -299,6 +307,7 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: [PR #1238]: https://github.com/bareos/bareos/pull/1238 [PR #1242]: https://github.com/bareos/bareos/pull/1242 [PR #1243]: https://github.com/bareos/bareos/pull/1243 +[PR #1244]: https://github.com/bareos/bareos/pull/1244 [PR #1247]: https://github.com/bareos/bareos/pull/1247 [PR #1248]: https://github.com/bareos/bareos/pull/1248 [PR #1251]: https://github.com/bareos/bareos/pull/1251 @@ -308,4 +317,5 @@ and since Bareos version 20 this project adheres to [Semantic Versioning](https: [PR #1260]: https://github.com/bareos/bareos/pull/1260 [PR #1266]: https://github.com/bareos/bareos/pull/1266 [PR #1267]: https://github.com/bareos/bareos/pull/1267 +[PR #1268]: https://github.com/bareos/bareos/pull/1268 [unreleased]: https://github.com/bareos/bareos/tree/master