From f44bb9b12e2de830c04c328302edddb6d3562d96 Mon Sep 17 00:00:00 2001 From: Frank Ueberschar Date: Fri, 22 May 2020 11:05:10 +0200 Subject: [PATCH] systemtests: fix systemtest-fd-plugin-postgresql --- .../plugins/filed/BareosFdPluginPostgres.py | 1 + .../database/setup_local_db.sh | 72 +++++++++++++++---- .../bareos-dir.d/fileset/PluginTest.conf.in | 2 +- .../python-fd-plugin-postgres-test/testrunner | 43 ++++++----- 4 files changed, 86 insertions(+), 32 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginPostgres.py b/core/src/plugins/filed/BareosFdPluginPostgres.py index 78d5e4c3f82..d467f7dc6a8 100644 --- a/core/src/plugins/filed/BareosFdPluginPostgres.py +++ b/core/src/plugins/filed/BareosFdPluginPostgres.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +#!/usr/bin/env python # -*- coding: utf-8 -*- # BAREOS - Backup Archiving REcovery Open Sourced # diff --git a/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh b/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh index e54b27cdb48..b4361fe0d01 100755 --- a/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh +++ b/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh @@ -1,23 +1,65 @@ +#!/bin/bash -pg_ctl -D data stop +local_db_stop_server() { + echo "Stop db server" + pg_ctl --silent --pgdata=data stop + tries=10 + while psql --host="$1" --list > /dev/null 2>&1; do + echo " -- $tries -- " + [ $((tries-=1)) -eq 0 ] && { + echo "Could not stop postgres server" + return 1 + } + sleep 0.3 + done +} +local_db_prepare_files() { + echo "Prepare files" + rm --recursive --force tmp data log + mkdir tmp data log wal_archive + LANG= pg_ctl --silent --pgdata=data --log=log/postgres.log initdb -rm --recursive --force tmp data log -mkdir tmp data log wal_archive -LANG= pg_ctl -D data -l log/postgres.log initdb + sed -i.bak "s@#listen_addresses.*@listen_addresses = ''@g" data/postgresql.conf + sed -i.bak "s@#unix_socket_directories.*@unix_socket_directories = \'$1\'@g" data/postgresql.conf -sed -i.bak "s@#listen_addresses.*@listen_addresses = ''@g" data/postgresql.conf -sed -i.bak "s@#unix_socket_directories.*@unix_socket_directories = \'$(pwd)/tmp\'@g" data/postgresql.conf + { + # for online backups we need wal_archiving + echo "wal_level = archive" + echo "archive_mode = on" + echo "archive_command = 'cp %p ../wal_archive'" + echo "max_wal_senders = 10" + } >> data/postgresql.conf +} -# for online backups we need wal_archiving -echo "wal_level = archive" >> data/postgresql.conf -echo "archive_mode = on" >> data/postgresql.conf -echo "archive_command = 'cp %p @current_test_directory@/database/wal_archive'" >> data/postgresql.conf -echo "max_wal_senders = 10" >> data/postgresql.conf +local_db_start_server() { + echo "Start db server" + pg_ctl --silent --pgdata=data --log=log/logfile start -pg_ctl -D data -l log/logfile start -sleep 10 + tries=10 + while ! psql --host="$1" --list > /dev/null 2>&1; do + [ $((tries-=1)) -eq 0 ] && { + echo "Could not start postgres server" + return 1 + } + sleep 0.1 + done -echo "CREATE ROLE root WITH SUPERUSER CREATEDB CREATEROLE REPLICATION LOGIN" |psql -h @current_test_directory@/database/tmp postgres + return 0 +} + +local_db_create_superuser_role() { + echo "CREATE ROLE root WITH SUPERUSER CREATEDB CREATEROLE REPLICATION LOGIN" |psql -h "$1" postgres +} + +setup_local_db() { + local_db_stop_server "$1" + local_db_prepare_files "$1" + if ! local_db_start_server "$1"; then return 1; fi + local_db_create_superuser_role "$1" + + echo stop server with "pg_ctl --pgdata=data stop" + + return 0 +} -echo stop server with "pg_ctl -D data stop" diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in index 970a8ef197d..6950f35934c 100644 --- a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in @@ -5,6 +5,6 @@ FileSet { Options { signature = MD5 } - Plugin = "python:module_path=/home/maik/git/bareos/build/systemtests/tests/python-fd-plugin-postgres-test/python-modules:module_name=bareos-fd-postgres:dbHost=@current_test_directory@/database/tmp:postgresDataDir=@current_test_directory@/database/data:walArchive=@current_test_directory@/database/wal_archive/" + Plugin = "python:module_path=@current_test_directory@/python-modules:module_name=bareos-fd-postgres:dbHost=/tmp/python-fd-plugin-postgres-test:postgresDataDir=@current_test_directory@/database/data:walArchive=@current_test_directory@/database/wal_archive/" } } diff --git a/systemtests/tests/python-fd-plugin-postgres-test/testrunner b/systemtests/tests/python-fd-plugin-postgres-test/testrunner index 7168a6512de..1482ab8de2f 100755 --- a/systemtests/tests/python-fd-plugin-postgres-test/testrunner +++ b/systemtests/tests/python-fd-plugin-postgres-test/testrunner @@ -16,25 +16,24 @@ export TestName JobName=backup-bareos-fd #shellcheck source=../environment.in . ./environment - -JobName=backup-bareos-fd +. ./database/setup_local_db.sh # setup local database server -pushd "$current_test_directory"/database || exit 1 -sh -x setup_local_db.sh -popd - -TESTPGHOST="$current_test_directory"/database/tmp -PSQL="psql -h $TESTPGHOST" DBNAME="backuptest" +TESTPGHOST="/tmp/${TestName}" +PSQL="psql --host $TESTPGHOST" + +[ -d "$TESTPGHOST" ] || mkdir -p "$TESTPGHOST" + +pushd database > /dev/null || exit 1 +setup_local_db "$TESTPGHOST" || exit +popd > /dev/null #shellcheck source=../scripts/functions . "${rscripts}"/functions "${rscripts}"/cleanup "${rscripts}"/setup - - # Directory to backup. # This directory will be created by setup_data "$@"(). BackupDirectory="${tmp}/data" @@ -93,9 +92,13 @@ run_bconsole # Now stop database and try a restore # ... # shut down database and delete directories -pg_ctl -D database/data stop -rm -Rf database/data -rm -Rf database/wal_archive +pushd database/ > /dev/null +local_db_stop_server "$TESTPGHOST" +rm -Rf data +rm -Rf wal_archive +echo "------------ stopped" +ps -e | grep postgres +popd > /dev/null cat <$tmp/bconcmds @$out /dev/null @@ -109,21 +112,29 @@ run_bconsole check_for_zombie_jobs storage=File stop_bareos +sleep 1 + +#sometimes the pid file remains +rm -f database/data/postmaster.pid # Create a recovery.conf # This may become a plugin feature later on echo "restore_command = 'cp $current_test_directory/database/wal_archive/%f %p'" > $current_test_directory/database/data/recovery.conf # start DB again - shall recover to latest possible state -pg_ctl -D database/data -l database/log/logfile start -sleep 10 +pushd database > /dev/null +echo "------------ to be started" +ps -e | grep postgres +local_db_start_server "$TESTPGHOST" || exit 1 +popd > /dev/null echo "SELECT * FROM t" | ${PSQL} ${DBNAME} > $tmp/sql.log check_two_logs if (grep -q "for INCR" $tmp/sql.log) -then +then estat=0 else + echo "key not found" estat=1 fi