libcloud: split testrunner, improve cmake message

- cmake adjust python version to less than 3.8
  make the comment less confusing
- split testrunner to local test-cleanup and test-setup
- add the import of empty dir to cover the case by uploading an
  empty file with same name as dir
- improve comments, remove obsolete parts

Signed-off-by: Bruno Friedmann <bruno.friedmann@bareos.com>
Co-authored-by: Sebastian Sura <124262655+sebsura@users.noreply.github.com>

systemtests/tests/py3plug-fd-libcloud/CMakeLists.txt

@@ -23,13 +23,13 @@ if(TARGET python3-fd)
      AND MINIO
      AND PYMODULE_3_LIBCLOUD_FOUND
      AND (${Python3_VERSION_MAJOR} EQUAL 3)
-     AND (${Python3_VERSION_MINOR} LESS 10)
+     AND (${Python3_VERSION_MINOR} LESS 8)
   )
     create_systemtest(${SYSTEMTEST_PREFIX} "py3plug-fd-libcloud")
   else()
     create_systemtest(
       ${SYSTEMTEST_PREFIX} "py3plug-fd-libcloud" DISABLED COMMENT
-      "S3CMD, MINIO or LIBCLOUD MODULE not found or Python > 3.7"
+      "S3CMD, MINIO or LIBCLOUD MODULE not found or Python >= 3.8"
     )
   endif()
 else()

systemtests/tests/py3plug-fd-libcloud/test-cleanup

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+
+#   BAREOS® - Backup Archiving REcovery Open Sourced
+#
+#   Copyright (C) 2021-2023 Bareos GmbH & Co. KG
+#
+#   This program is Free Software; you can redistribute it and/or
+#   modify it under the terms of version three of the GNU Affero General Public
+#   License as published by the Free Software Foundation and included
+#   in the file LICENSE.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#   Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+#   02110-1301, USA.
+
+set -e
+set -o pipefail
+set -u
+
+TestName="$(basename "$(pwd)")"
+export TestName
+bucket_name=bareos-test
+
+#shellcheck source=../environment.in
+. ./environment
+
+"${rscripts}"/stop_minio.sh "$TestName"
+
+#shellcheck source=../scripts/functions
+. "${rscripts}"/functions
+"${rscripts}"/cleanup

systemtests/tests/py3plug-fd-libcloud/testrunner → systemtests/tests/py3plug-fd-libcloud/test-setup

@@ -1,40 +1,56 @@
 #!/bin/bash
-set -e
-set -o pipefail
-set -u
+
+#   BAREOS® - Backup Archiving REcovery Open Sourced
 #
-# This systemtest tests the plugin functionality
-# of the Bareos FD by using the supplied module
-#   bareos-fd-local-fileset.py
+#   Copyright (C) 2021-2023 Bareos GmbH & Co. KG
 #
-# The module will backup some files.
-# This plugin is not intended for production,
-# but is only a minimal example that shows
-# how to use the python plugin interface.
-# File attributes like uses and times will not be saved.
+#   This program is Free Software; you can redistribute it and/or
+#   modify it under the terms of version three of the GNU Affero General Public
+#   License as published by the Free Software Foundation and included
+#   in the file LICENSE.
 #
-TestName="$(basename "$(pwd)")"
-export TestName
-bucket_name=bareos-test
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#   Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+#   02110-1301, USA.
+
+set -e
+set -o pipefail
+set -u
 
-JobName=backup-bareos-fd
 #shellcheck source=../environment.in
 . ./environment
 
-JobName=backup-bareos-fd
 #shellcheck source=../scripts/functions
 . "${rscripts}"/functions
 "${rscripts}"/cleanup
 "${rscripts}"/setup
 
-# shortcut for s3cmd
-S3="${S3CMD} --no-check-certificate --config ${S3CFG}"
+bin/bareos start
+bin/bareos status
+
+# make sure, director is up and running.
+print_debug "$(bin/bconsole <<< "status dir")"
+
+TestName="$(basename "$(pwd)")"
+export TestName
+bucket_name=bareos-test
 
 
+# local test-setup extending
+print_debug "Data preparation starting"
 
 # Fill ${BackupDirectory} with data.
 setup_data
 
+# shortcut for s3cmd
+S3="${S3CMD} --no-check-certificate --config ${S3CFG}"
+
 # create files to test the temporary-file and the stream-download path
 prefetch_size=$(( $(grep prefetch_size etc/libcloud_config.ini | cut -d '=' -f 2) ))
 
@@ -53,7 +69,7 @@ dd if=/dev/random \
 # create s3 content for test
 ${S3} rb --recursive --force s3://$bucket_name || echo "s3://$bucket_name does not exist"
 ${S3} mb s3://$bucket_name
-
+print_debug "s3 ready for operation"
 
 # this test does not work with links and some other weird files as they would already
 # have a changed name by syncing to S3 using s3cmd
@@ -64,52 +80,17 @@ rm ${tmp}/data/weird-files/newline*
 rm ${tmp}/data/weird-files/tab*
 # the following file also makes problems
 rm ${tmp}/data/weird-files/filename-with-non-utf8-bytestring*
-# s3cmd does not sync empty dirs
+
+# s3cmd does not sync empty dirs, but you can create them by placing
+# into them an empty dummy file with same name
+# Most end-user expect to be able to restore those empty dirs.
 rmdir ${tmp}/data/weird-files/big-X
 rmdir ${tmp}/data/weird-files/subdir
-
-${S3} sync "$BackupDirectory" s3://$bucket_name
-
-start_test
-
-cat <<END_OF_DATA >$tmp/bconcmds
-@$out /dev/null
-messages
-@$out $tmp/log1.out
-setdebug level=100 storage=File
-setdebug level=200 client=bareos-fd trace=1 timestamp=1
-label volume=TestVolume001 storage=File pool=Full
-run job=$JobName yes
-status director
-status client
-status storage=File
-wait
-messages
-@#
-@# now do a restore
-@#
-@$out $tmp/log2.out
-wait
-restore client=bareos-fd fileset=PluginTest where=$tmp/bareos-restores select all done
-yes
-wait
-messages
-quit
-END_OF_DATA
-
-run_bareos "$@"
-check_for_zombie_jobs storage=File
-
-check_two_logs
-list=( $(find "${BackupDirectory}" -type f) )
-# Using check_restore_only_files_diff instead of check_restore_diff
-# to don'"t diff the file attributes, because they are not saved
-#check_restore_only_files_diff "${list[@]}"
-
-if ! diff -r tmp/data tmp/bareos-restores/$bucket_name/data; then
-  export estat=1
-fi
-
-"${rscripts}"/stop_minio.sh "$TestName"
-
-end_test
+mkdir -p ${tmp}/data/empty_subdir/
+touch ${tmp}/data/empty_subdir/empty_subdir
+${S3} sync "${BackupDirectory}" s3://${bucket_name}
+# create directly an empty directory in the s3 (would be a plus) but
+# unfortunately doesn't work
+# ${S3} put remote_empty s3://${bucket_name}/remote_empty/
+
+print_debug "Data preparation done"

systemtests/tests/py3plug-fd-libcloud/testrunner-default

@@ -0,0 +1,71 @@
+#!/bin/bash
+set -e
+set -o pipefail
+set -u
+#
+# This systemtest tests the plugin functionality
+# of the Bareos FD plugin bareos-fd-libcloud.py
+#
+# This test will start a s3minio server and populate it with some data.
+# Then it runs a backup and a restore with the plugin.
+# A full comparison is made afterwards.
+# File attributes like uses and times will not be saved.
+
+TestName="$(basename "$(pwd)")"
+export TestName
+bucket_name=bareos-test
+
+JobName=backup-bareos-fd
+#shellcheck source=../environment.in
+. ./environment
+#shellcheck source=../scripts/functions
+. "${rscripts}"/functions
+
+start_test
+
+cat <<END_OF_DATA >${tmp}/bconcmds
+@$out /dev/null
+messages
+@$out ${tmp}/log1.out
+setdebug level=100 trace=1 timestamp=1 storage=File
+setdebug level=200 trace=1 timestamp=1 client=bareos-fd
+run job=${JobName} yes
+wait
+setdebug level=0 client=bareos-fd
+status director
+status client
+status storage=File
+messages
+END_OF_DATA
+
+echo "Full backup stage"
+run_bconsole
+expect_grep "Backup OK" "${tmp}/log1.out" "Full Backup not found!"
+if [ ${estat} -ne 0 ]; then
+    exit ${estat}
+fi
+
+cat <<END_OF_DATA >${tmp}/bconcmds
+@$out ${tmp}/log2.out
+setdebug level=200 trace=1 timestamp=1 client=bareos-fd
+restore client=bareos-fd fileset=PluginTest where=${tmp}/bareos-restores select all done yes
+wait
+messages
+quit
+END_OF_DATA
+echo "Restore backup stage"
+run_bconsole
+expect_grep "Restore OK" "${tmp}/log2.out" "Full Restore not ok!"
+if [ ${estat} -ne 0 ]; then
+    exit ${estat}
+fi
+
+check_for_zombie_jobs storage=File
+
+check_two_logs
+
+if ! diff -r tmp/data tmp/bareos-restores/${bucket_name}/data; then
+  export estat=1
+fi
+
+end_test