From 80ec507c10f47683f7732f25ad474a85d38c131b Mon Sep 17 00:00:00 2001 From: aussendorf Date: Thu, 16 Apr 2020 17:00:01 +0200 Subject: [PATCH 1/9] python-plugins: move baseclass io operations into separate methods - LocalFilesetPlugin: Support for file attributes, links and dirs --- .../plugins/filed/BareosFdPluginBaseclass.py | 188 ++++++++++++------ .../filed/BareosFdPluginLocalFileset.py | 88 +++++++- 2 files changed, 206 insertions(+), 70 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginBaseclass.py b/core/src/plugins/filed/BareosFdPluginBaseclass.py index 758b5b51c97..018cff2b41c 100644 --- a/core/src/plugins/filed/BareosFdPluginBaseclass.py +++ b/core/src/plugins/filed/BareosFdPluginBaseclass.py @@ -58,6 +58,7 @@ def __init__(self, context, plugindef, mandatory_options=None): self.jobName = bareosfd.GetValue(context, bVariable["bVarJobName"]) self.workingdir = bareosfd.GetValue(context, bVariable["bVarWorkingDir"]) self.FNAME = "undef" + self.filetype = "undef" self.file = None bareosfd.DebugMessage( context, 100, "FDName = %s - BareosFdPluginBaseclass\n" % (self.fdname) @@ -113,10 +114,8 @@ def check_options(self, context, mandatory_options=None): Here we just verify that eventual mandatory options are set. If you have more to veriy, just overwrite ths method in your class """ - if mandatory_options is None: return bRCs["bRC_OK"] - for option in mandatory_options: if option not in self.options: bareosfd.DebugMessage( @@ -128,99 +127,149 @@ def check_options(self, context, mandatory_options=None): "Mandatory option '%s' not defined.\n" % (option), ) return bRCs["bRC_Error"] - bareosfd.DebugMessage( context, 100, "Using Option %s=%s\n" % (option, self.options[option]) ) - return bRCs["bRC_OK"] - def plugin_io(self, context, IOP): - bareosfd.DebugMessage( - context, 100, "plugin_io called with function %s\n" % (IOP.func) - ) - bareosfd.DebugMessage(context, 100, "FNAME is set to %s\n" % (self.FNAME)) - - if IOP.func == bIOPS["IO_OPEN"]: - self.FNAME = IOP.fname - try: - if IOP.flags & (os.O_CREAT | os.O_WRONLY): - bareosfd.DebugMessage( - context, - 100, - "Open file %s for writing with %s\n" % (self.FNAME, IOP), - ) - - dirname = os.path.dirname(self.FNAME) - if not os.path.exists(dirname): - bareosfd.DebugMessage( - context, - 100, - "Directory %s does not exist, creating it now\n" - % (dirname), - ) - os.makedirs(dirname) - self.file = open(self.FNAME, "wb") - else: + def plugin_io_open(self, context, IOP): + self.FNAME = IOP.fname + if os.path.isdir(self.FNAME): + bareosfd.DebugMessage(context, 100, "%s is a directory\n" % (IOP.fname)) + self.fileType = "FT_DIR" + bareosfd.DebugMessage( + context, + 100, + "Did not open file %s of type %s\n" % (self.FNAME, self.fileType), + ) + return bRCs["bRC_OK"] + elif os.path.islink(self.FNAME): + self.fileType = "FT_LNK" + bareosfd.DebugMessage( + context, + 100, + "Did not open file %s of type %s\n" % (self.FNAME, self.fileType), + ) + return bRCs["bRC_OK"] + else: + self.fileType = "FT_REG" + bareosfd.DebugMessage( + context, + 150, + "file %s has type %s - trying to open it\n" + % (self.FNAME, self.fileType), + ) + try: + if IOP.flags & (os.O_CREAT | os.O_WRONLY): + bareosfd.DebugMessage( + context, + 100, + "Open file %s for writing with %s\n" % (self.FNAME, IOP), + ) + dirname = os.path.dirname(self.FNAME) + if not os.path.exists(dirname): bareosfd.DebugMessage( context, 100, - "Open file %s for reading with %s\n" % (self.FNAME, IOP), + "Directory %s does not exist, creating it now\n" % (dirname), ) - self.file = open(self.FNAME, "rb") - except: - IOP.status = -1 - return bRCs["bRC_Error"] - - return bRCs["bRC_OK"] + os.makedirs(dirname) + self.file = open(self.FNAME, "wb") + else: + bareosfd.DebugMessage( + context, + 100, + "Open file %s for reading with %s\n" % (self.FNAME, IOP), + ) + self.file = open(self.FNAME, "rb") + except: + IOP.status = -1 + return bRCs["bRC_Error"] + return bRCs["bRC_OK"] - elif IOP.func == bIOPS["IO_CLOSE"]: - bareosfd.DebugMessage(context, 100, "Closing file " + "\n") + def plugin_io_close(self, context, IOP): + bareosfd.DebugMessage(context, 100, "Closing file " + "\n") + if self.fileType == "FT_REG": self.file.close() - return bRCs["bRC_OK"] + return bRCs["bRC_OK"] - elif IOP.func == bIOPS["IO_SEEK"]: - return bRCs["bRC_OK"] + def plugin_io_seek(self, context, IOP): + return bRCs["bRC_OK"] - elif IOP.func == bIOPS["IO_READ"]: + def plugin_io_read(self, context, IOP): + if self.fileType == "FT_REG": bareosfd.DebugMessage( context, 200, "Reading %d from file %s\n" % (IOP.count, self.FNAME) ) IOP.buf = bytearray(IOP.count) IOP.status = self.file.readinto(IOP.buf) IOP.io_errno = 0 - return bRCs["bRC_OK"] - - elif IOP.func == bIOPS["IO_WRITE"]: + else: bareosfd.DebugMessage( - context, 200, "Writing buffer to file %s\n" % (self.FNAME) + context, + 100, + "Did not read from file %s of type %s\n" % (self.FNAME, self.fileType), ) - self.file.write(IOP.buf) - IOP.status = IOP.count + IOP.buf = bytearray() + IOP.status = 0 IOP.io_errno = 0 - return bRCs["bRC_OK"] + return bRCs["bRC_OK"] + + def plugin_io_write(self, context, IOP): + bareosfd.DebugMessage( + context, 200, "Writing buffer to file %s\n" % (self.FNAME) + ) + self.file.write(IOP.buf) + IOP.status = IOP.count + IOP.io_errno = 0 + return bRCs["bRC_OK"] + + def plugin_io(self, context, IOP): + """ + Basic IO operations. Some tweaks here: IOP.fname is only set on file-open + We need to capture it on open and keep it for the remaining procedures + Now (since 2020) separated into sub-methods to ease overloading in derived classes + """ + bareosfd.DebugMessage( + context, + 250, + "plugin_io called with function %s filename %s\n" % (IOP.func, IOP.fname), + ) + bareosfd.DebugMessage(context, 250, "self.FNAME is set to %s\n" % (self.FNAME)) + if IOP.func == bIOPS["IO_OPEN"]: + return self.plugin_io_open(context, IOP) + elif IOP.func == bIOPS["IO_CLOSE"]: + return self.plugin_io_close(context, IOP) + elif IOP.func == bIOPS["IO_SEEK"]: + return self.plugin_io_seek(context, IOP) + elif IOP.func == bIOPS["IO_READ"]: + return self.plugin_io_read(context, IOP) + elif IOP.func == bIOPS["IO_WRITE"]: + return self.plugin_io_write(context, IOP) def handle_plugin_event(self, context, event): if event == bEventType["bEventJobEnd"]: bareosfd.DebugMessage( context, 100, "handle_plugin_event called with bEventJobEnd\n" ) + return self.end_job(context) elif event == bEventType["bEventEndBackupJob"]: bareosfd.DebugMessage( context, 100, "handle_plugin_event called with bEventEndBackupJob\n" ) + return self.end_backup_job(context) elif event == bEventType["bEventEndFileSet"]: bareosfd.DebugMessage( context, 100, "handle_plugin_event called with bEventEndFileSet\n" ) + return self.end_fileset(context) elif event == bEventType["bEventStartBackupJob"]: bareosfd.DebugMessage( context, 100, "handle_plugin_event() called with bEventStartBackupJob\n" ) - return self.start_backup_job(context) elif event == bEventType["bEventStartRestoreJob"]: @@ -229,14 +278,12 @@ def handle_plugin_event(self, context, event): 100, "handle_plugin_event() called with bEventStartRestoreJob\n", ) - return self.start_restore_job(context) else: bareosfd.DebugMessage( context, 100, "handle_plugin_event called with event %s\n" % (event) ) - return bRCs["bRC_OK"] def start_backup_job(self, context): @@ -246,6 +293,20 @@ def start_backup_job(self, context): """ return bRCs["bRC_OK"] + def end_job(self, context): + """ + Called if job ends regularyly (not for cancelled jobs) + Overload this to arrange whatever you have to do at this time. + """ + return bRCs["bRC_OK"] + + def end_backup_job(self, context): + """ + Called if backup job ends, before ClientAfterJob + Overload this to arrange whatever you have to do at this time. + """ + return bRCs["bRC_OK"] + def start_backup_file(self, context, savepkt): """ Base method, we do not add anything, overload this method with your @@ -260,6 +321,12 @@ def end_backup_file(self, context): ) return bRCs["bRC_OK"] + def end_fileset(self, context): + bareosfd.DebugMessage( + context, 100, "end_fileset() entry point in Python called\n" + ) + return bRCs["bRC_OK"] + def start_restore_job(self, context): """ Start of Restore Job. Called , if you have Restore objects. @@ -299,7 +366,7 @@ def create_file(self, context, restorepkt): "create_file() entry point in Python called with %s\n" % (restorepkt), ) FNAME = restorepkt.ofname - dirname = os.path.dirname(FNAME) + dirname = os.path.dirname(FNAME.rstrip("/")) if not os.path.exists(dirname): bareosfd.DebugMessage( context, 200, "Directory %s does not exist, creating it now\n" % dirname @@ -307,12 +374,15 @@ def create_file(self, context, restorepkt): os.makedirs(dirname) # open creates the file, if not yet existing, we close it again right # aways it will be opened again in plugin_io. - # But: only do this for regular files, prevent from - # IOError: (21, 'Is a directory', '/tmp/bareos-restores/my/dir/') - # if it's a directory if restorepkt.type == bFileType["FT_REG"]: open(FNAME, "wb").close() - restorepkt.create_status = bCFs["CF_EXTRACT"] + elif restorepkt.type == bFileType["FT_LNK"]: + if not os.path.exists(FNAME.rstrip("/")): + os.symlink(restorepkt.olname, FNAME.rstrip("/")) + elif restorepkt.type == bFileType["FT_DIREND"]: + if not os.path.exists(FNAME): + os.makedirs(FNAME) + restorepkt.create_status = bCFs["CF_EXTRACT"] return bRCs["bRC_OK"] def set_file_attributes(self, context, restorepkt): diff --git a/core/src/plugins/filed/BareosFdPluginLocalFileset.py b/core/src/plugins/filed/BareosFdPluginLocalFileset.py index a99fc4f548a..a4fdac9424e 100644 --- a/core/src/plugins/filed/BareosFdPluginLocalFileset.py +++ b/core/src/plugins/filed/BareosFdPluginLocalFileset.py @@ -39,16 +39,18 @@ class BareosFdPluginLocalFileset( listed there Filename is taken from plugin argument 'filename' """ - def __init__(self, context, plugindef): + def __init__(self, context, plugindef, mandatory_options=None): bareosfd.DebugMessage( context, 100, "Constructor called in module %s with plugindef=%s\n" % (__name__, plugindef), ) + if mandatory_options is None: + mandatory_options = ["filename"] # Last argument of super constructor is a list of mandatory arguments super(BareosFdPluginLocalFileset, self).__init__( - context, plugindef, ["filename"] + context, plugindef, mandatory_options ) self.files_to_backup = [] self.allow = None @@ -88,11 +90,10 @@ def start_backup_job(self, context): We try to read from filename and setup the list of file to backup in self.files_to_backup """ - bareosfd.DebugMessage( context, 100, - "Using %s to search for local files\n" % (self.options["filename"]), + "Using %s to search for local files\n" % self.options["filename"], ) if os.path.exists(self.options["filename"]): try: @@ -121,6 +122,11 @@ def start_backup_job(self, context): ): self.files_to_backup.append(listItem) if os.path.isdir(listItem): + fullDirName = listItem + # FD requires / at the end of a directory name + if not fullDirName.endswith("/"): + fullDirName += "/" + self.files_to_backup.append(fullDirName) for topdir, dirNames, fileNames in os.walk(listItem): for fileName in fileNames: if self.filename_is_allowed( @@ -130,6 +136,13 @@ def start_backup_job(self, context): self.deny, ): self.files_to_backup.append(os.path.join(topdir, fileName)) + for dirName in dirNames: + fullDirName = os.path.join(topdir, dirName) + "/" + self.files_to_backup.append(fullDirName) + bareosfd.DebugMessage( + context, 150, "Filelist: %s\n" % (self.files_to_backup), + ) + if not self.files_to_backup: bareosfd.JobMessage( context, @@ -138,7 +151,7 @@ def start_backup_job(self, context): ) return bRCs["bRC_Error"] else: - return bRCs["bRC_Cancel"] + return bRCs["bRC_OK"] def start_backup_file(self, context, savepkt): """ @@ -153,16 +166,69 @@ def start_backup_file(self, context, savepkt): file_to_backup = self.files_to_backup.pop() bareosfd.DebugMessage(context, 100, "file: " + file_to_backup + "\n") - statp = bareosfd.StatPacket() - savepkt.statp = statp + mystatp = bareosfd.StatPacket() + statp = os.stat(file_to_backup) + # As of Bareos 19.2.7 attribute names in bareosfd.StatPacket differ from os.stat + # In this case we have to translate names + # For future releases consistent names are planned, allowing to assign the + # complete stat object in one rush + if hasattr(mystatp, "st_uid"): + mystatp = statp + else: + mystatp.mode = statp.st_mode + mystatp.ino = statp.st_ino + mystatp.dev = statp.st_dev + mystatp.nlink = statp.st_nlink + mystatp.uid = statp.st_uid + mystatp.gid = statp.st_gid + mystatp.size = statp.st_size + mystatp.atime = statp.st_atime + mystatp.mtime = statp.st_mtime + mystatp.ctime = statp.st_ctime savepkt.fname = file_to_backup - savepkt.type = bFileType["FT_REG"] + # os.islink will detect links to directories only when + # there is no trailing slash - we need to perform checks + # on the stripped name but use it with trailing / for the backup itself + if os.path.islink(file_to_backup.rstrip("/")): + savepkt.type = bFileType["FT_LNK"] + savepkt.link = os.readlink(file_to_backup.rstrip("/")) + bareosfd.DebugMessage(context, 150, "file type is: FT_LNK\n") + elif os.path.isfile(file_to_backup): + savepkt.type = bFileType["FT_REG"] + bareosfd.DebugMessage(context, 150, "file type is: FT_REG\n") + elif os.path.isdir(file_to_backup): + savepkt.type = bFileType["FT_DIREND"] + savepkt.link = file_to_backup + bareosfd.DebugMessage( + context, 150, "file %s type is: FT_DIREND\n" % file_to_backup + ) + else: + bareosfd.JobMessage( + context, + bJobMessageType["M_WARNING"], + "File %s of unknown type" % (file_to_backup), + ) + return bRCs["bRC_Skip"] + + savepkt.statp = mystatp + bareosfd.DebugMessage(context, 150, "file statpx " + str(savepkt.statp) + "\n") - bareosfd.JobMessage( + return bRCs["bRC_OK"] + + def set_file_attributes(self, context, restorepkt): + # Python attribute setting does not work properly with links + if restorepkt.type == bFileType["FT_LNK"]: + return bRCs["bRC_OK"] + file_name = restorepkt.ofname + file_attr = restorepkt.statp + bareosfd.DebugMessage( context, - bJobMessageType["M_INFO"], - "Starting backup of %s\n" % (file_to_backup), + 150, + "Restore file " + file_name + " with stat " + str(file_attr) + "\n", ) + os.chown(file_name, file_attr.uid, file_attr.gid) + os.chmod(file_name, file_attr.mode) + os.utime(file_name, (file_attr.atime, file_attr.mtime)) return bRCs["bRC_OK"] def end_backup_file(self, context): From 26e7a8f1ee570e4f48f756c8e8ea901b3514096c Mon Sep 17 00:00:00 2001 From: aussendorf Date: Thu, 16 Apr 2020 17:05:59 +0200 Subject: [PATCH 2/9] postgres-plugin: full / incr. backup of Postgres databases - tested with Postgres >= 9.2 supposed to work with any Postgres version supporting pg_start_backup() function. - WAL switching supported since Postgres 9. --- .../plugins/filed/BareosFdPluginBaseclass.py | 45 +- .../filed/BareosFdPluginLocalFileset.py | 25 +- .../plugins/filed/BareosFdPluginPostgres.py | 514 ++++++++++++++++++ core/src/plugins/filed/bareos-fd-postgres.py | 57 ++ .../source/TasksAndConcepts/Plugins.rst | 150 +++++ 5 files changed, 765 insertions(+), 26 deletions(-) create mode 100644 core/src/plugins/filed/BareosFdPluginPostgres.py create mode 100644 core/src/plugins/filed/bareos-fd-postgres.py diff --git a/core/src/plugins/filed/BareosFdPluginBaseclass.py b/core/src/plugins/filed/BareosFdPluginBaseclass.py index 018cff2b41c..fe3297c7a49 100644 --- a/core/src/plugins/filed/BareosFdPluginBaseclass.py +++ b/core/src/plugins/filed/BareosFdPluginBaseclass.py @@ -202,8 +202,17 @@ def plugin_io_read(self, context, IOP): context, 200, "Reading %d from file %s\n" % (IOP.count, self.FNAME) ) IOP.buf = bytearray(IOP.count) - IOP.status = self.file.readinto(IOP.buf) - IOP.io_errno = 0 + try: + IOP.status = self.file.readinto(IOP.buf) + IOP.io_errno = 0 + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could net read %d bytes from file %s. \"%s\"" % (IOP.count, file_to_backup, e.message), + ) + IOP.io_errno = e.errno + return bRCs["bRC_Error"] else: bareosfd.DebugMessage( context, @@ -219,7 +228,17 @@ def plugin_io_write(self, context, IOP): bareosfd.DebugMessage( context, 200, "Writing buffer to file %s\n" % (self.FNAME) ) - self.file.write(IOP.buf) + try: + self.file.write(IOP.buf) + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could net write to file %s. \"%s\"" % (file_to_backup, e.message), + ) + IOP.io_errno = e.errno + IOP.status = 0 + return bRCs["bRC_Error"] IOP.status = IOP.count IOP.io_errno = 0 return bRCs["bRC_OK"] @@ -365,24 +384,8 @@ def create_file(self, context, restorepkt): 100, "create_file() entry point in Python called with %s\n" % (restorepkt), ) - FNAME = restorepkt.ofname - dirname = os.path.dirname(FNAME.rstrip("/")) - if not os.path.exists(dirname): - bareosfd.DebugMessage( - context, 200, "Directory %s does not exist, creating it now\n" % dirname - ) - os.makedirs(dirname) - # open creates the file, if not yet existing, we close it again right - # aways it will be opened again in plugin_io. - if restorepkt.type == bFileType["FT_REG"]: - open(FNAME, "wb").close() - elif restorepkt.type == bFileType["FT_LNK"]: - if not os.path.exists(FNAME.rstrip("/")): - os.symlink(restorepkt.olname, FNAME.rstrip("/")) - elif restorepkt.type == bFileType["FT_DIREND"]: - if not os.path.exists(FNAME): - os.makedirs(FNAME) - restorepkt.create_status = bCFs["CF_EXTRACT"] + # We leave file creation up to the core for the default case + restorepkt.create_status = bCFs["CF_CORE"] return bRCs["bRC_OK"] def set_file_attributes(self, context, restorepkt): diff --git a/core/src/plugins/filed/BareosFdPluginLocalFileset.py b/core/src/plugins/filed/BareosFdPluginLocalFileset.py index a4fdac9424e..5ceac6bf36c 100644 --- a/core/src/plugins/filed/BareosFdPluginLocalFileset.py +++ b/core/src/plugins/filed/BareosFdPluginLocalFileset.py @@ -167,7 +167,14 @@ def start_backup_file(self, context, savepkt): bareosfd.DebugMessage(context, 100, "file: " + file_to_backup + "\n") mystatp = bareosfd.StatPacket() - statp = os.stat(file_to_backup) + try: + statp = os.stat(file_to_backup) + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could net get stat-info for file %s: \"%s\"" % (file_to_backup, e.message), + ) # As of Bareos 19.2.7 attribute names in bareosfd.StatPacket differ from os.stat # In this case we have to translate names # For future releases consistent names are planned, allowing to assign the @@ -224,11 +231,19 @@ def set_file_attributes(self, context, restorepkt): bareosfd.DebugMessage( context, 150, - "Restore file " + file_name + " with stat " + str(file_attr) + "\n", + "Set file attributes " + file_name + " with stat " + str(file_attr) + "\n", ) - os.chown(file_name, file_attr.uid, file_attr.gid) - os.chmod(file_name, file_attr.mode) - os.utime(file_name, (file_attr.atime, file_attr.mtime)) + try: + os.chown(file_name, file_attr.uid, file_attr.gid) + os.chmod(file_name, file_attr.mode) + os.utime(file_name, (file_attr.atime, file_attr.mtime)) + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_WARNING"], + "Could net set attributes for file %s: \"%s\"" % (file_to_backup, e.message), + ) + return bRCs["bRC_OK"] def end_backup_file(self, context): diff --git a/core/src/plugins/filed/BareosFdPluginPostgres.py b/core/src/plugins/filed/BareosFdPluginPostgres.py new file mode 100644 index 00000000000..2f1bcb488c9 --- /dev/null +++ b/core/src/plugins/filed/BareosFdPluginPostgres.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# BAREOS - Backup Archiving REcovery Open Sourced +# +# Copyright (C) 2014-2014 Bareos GmbH & Co. KG +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of version three of the GNU Affero General Public +# License as published by the Free Software Foundation, which is +# listed in the file LICENSE. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# Author: Maik Aussendorf +# +# Bareos python plugins class that adds files from a local list to +# the backup fileset + +from bareosfd import * +from bareos_fd_consts import bJobMessageType, bFileType, bRCs +import os +import sys +import re +import psycopg2 +import time +import datetime +from dateutil import parser +import dateutil +import json +import BareosFdPluginLocalFileset +from BareosFdPluginBaseclass import * + + +class BareosFdPluginPostgres( + BareosFdPluginLocalFileset.BareosFdPluginLocalFileset +): # noqa + """ + Simple Bareos-FD-Plugin-Class that parses a file and backups all files + listed there Filename is taken from plugin argument 'filename' + """ + + def __init__(self, context, plugindef): + bareosfd.DebugMessage( + context, + 100, + "Constructor called in module %s with plugindef=%s\n" + % (__name__, plugindef), + ) + # Last argument of super constructor is a list of mandatory arguments + super(BareosFdPluginPostgres, self).__init__( + context, plugindef, ["postgresDataDir", "walArchive"] + ) + self.ignoreSubdirs = ["pg_wal", "pg_log", "pg_xlog"] + + self.dbCon = None + self.dbCursor = None + # This will be set to True between SELCET pg_start_backup + # and SELECT pg_stop_backup. We backup database file during + # this time + self.PostgressFullBackupRunning = False + # Here we store items found in file backup_label, produced by Postgres + self.labelItems = dict() + # We will store the starttime from backup_label here + self.backupStartTime = None + # Postgresql backup stop time + self.lastBackupStopTime = 0 + # Our label, will be used for SELECT pg_start_backup and there + # be used as backup_label + self.backupLabelString = "Bareos.pgplugin.jobid.%d" % self.jobId + # Raw restore object data (json-string) + self.row_rop_raw = None + # Dictionary to store passed restore object data + self.rop_data = {} + # we need our timezone information for some timestamp comparisons + # this one respects daylight saving timezones + self.tzOffset = -( + time.altzone + if (time.daylight and time.localtime().tm_isdst) + else time.timezone + ) + + def check_options(self, context, mandatory_options=None): + """ + Check for mandatory options and verify database connection + """ + result = super(BareosFdPluginPostgres, self).check_options( + context, mandatory_options + ) + if not result == bRCs["bRC_OK"]: + return result + if not self.options["postgresDataDir"].endswith("/"): + self.options["postgresDataDir"] += "/" + self.labelFileName = self.options["postgresDataDir"] + "/backup_label" + if not self.options["walArchive"].endswith("/"): + self.options["walArchive"] += "/" + if "ignoreSubdirs" in self.options: + self.ignoreSubdirs = self.options["ignoreSubdirs"] + if "dbname" in self.options: + self.dbname = self.options["dbname"] + else: + self.dbname = "postgres" + if "dbuser" in self.options: + self.dbuser = self.options["dbuser"] + else: + self.dbuser = "root" + if not "switchWal" in self.options: + self.switchWal = True + else: + self.switchWal = self.options["switchWal"].lower() == "true" + return bRCs["bRC_OK"] + + def execute_SQL(self, context, sqlStatement): + """ + Executes the SQL statement using the classes dbCursor + """ + try: + self.dbCursor.execute(sqlStatement) + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Query \"%s\" failed: \"%s\"" % (sqlStatement, e.message), + ) + return False + return True + + def start_backup_job(self, context): + """ + Make filelist in super class and tell Postgres + that we start a backup now + """ + bareosfd.DebugMessage( + context, 100, "start_backup_job in PostgresPlugin called", + ) + try: + self.dbCon = psycopg2.connect( + "dbname=%s user=%s" % (self.dbname, self.dbuser) + ) + self.dbCursor = self.dbCon.cursor() + self.dbCursor.execute("SELECT current_setting('server_version_num')") + self.pgVersion = int(self.dbCursor.fetchone()[0]) + #bareosfd.DebugMessage( + # context, 1, "Connected to Postgres version %d\n" % self.pgVersion, + #) + ## WARNING: JobMessages cause fatal errors at this stage + JobMessage( + context, + bJobMessageType["M_INFO"], + "Connected to Postgres version %d\n" + % (self.pgVersion), + ) + except: + bareosfd.JobMessage( + context, + bJobMessageType["M_FATAL"], + "Could not connect to database %s, user %s\n" + % (self.dbname, self.dbuser), + ) + return bRCs["bRC_Error"] + if chr(self.level) == "F": + # For Full we backup the Postgres data directory + # Restore object ROP comes later, after file backup + # is done. + startDir = self.options["postgresDataDir"] + self.files_to_backup.append(startDir) + bareosfd.DebugMessage( + context, 100, "dataDir: %s\n" % self.options["postgresDataDir"], + ) + else: + # If level is not Full, we only backup WAL files + # and create a restore object ROP with timestamp information. + startDir = self.options["walArchive"] + self.files_to_backup.append("ROP") + # get current Log Sequence Number (LSN) + # PG8: not supported + # PG9: pg_get_current_xlog_location + # PG10: pg_current_wal_lsn + pgMajorVersion = self.pgVersion // 10000 + if pgMajorVersion >= 10: + getLsnStmt = "SELECT pg_current_wal_lsn()" + switchLsnStmt = "SELECT pg_switch_wal()" + elif pgMajorVersion >= 9: + getLsnStmt = "SELECT pg_current_xlog_location()" + switchLsnStmt = "SELECT pg_switch_xlog()" + if pgMajorVersion < 9: + bareosfd.JobMessage( + context, + bJobMessageType["M_INFO"], + "WAL switching not supported on Postgres Version < 9\n", + ) + else: + if self.execute_SQL(context, getLsnStmt): + currentLSN = self.dbCursor.fetchone()[0].zfill(17) + bareosfd.DebugMessage( + context, + 100, + "Current LSN %s, last LSN: %s\n" % (currentLSN, self.lastLSN), + ) + else: + currrentLSN = 0 + bareosfd.JobMessage( + context, + bJobMessageType["M_WARNING"], + "Could not get current LSN, last LSN was: %s\n" % self.lastLSN, + ) + if currentLSN > self.lastLSN and self.switchWal: + # Let Postgres write latest transaction into a new WAL file now + if not self.execute_SQL(context, switchLsnStmt): + bareosfd.JobMessage( + context, + bJobMessageType["M_WARNING"], + "Could not switch to next WAL segment\n", + ) + if self.execute_SQL(context, getLsnStmt): + currentLSN = self.dbCursor.fetchone()[0].zfill(17) + self.lastLSN = currentLSN + # wait some seconds to make sure WAL file gets written + time.sleep(10) + else: + bareosfd.JobMessage( + context, + bJobMessageType["M_WARNING"], + "Could not read LSN after switching to new WAL segment\n", + ) + else: + # Nothing has changed since last backup - only send ROP this time + bareosfd.JobMessage( + context, + bJobMessageType["M_INFO"], + "Same LSN %s as last time - nothing to do\n" % currentLSN, + ) + return bRCs["bRC_OK"] + + # Gather files from startDir (Postgres data dir or walArchive for incr/diff jobs) + for fileName in os.listdir(startDir): + fullName = os.path.join(startDir, fileName) + # We need a trailing '/' for directories + if os.path.isdir(fullName) and not fullName.endswith("/"): + fullName += "/" + bareosfd.DebugMessage( + context, 100, "fullName: %s\n" % fullName, + ) + # Usually Bareos takes care about timestamps when doing incremental backups + # but here we have to compare against last BackupPostgres timestamp + try: + mTime = os.stat(fullName).st_mtime + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could net get stat-info for file %s: \"%s\"" % (file_to_backup, e.message), + ) + bareosfd.DebugMessage( + context, + 150, + "%s fullTime: %d mtime: %d\n" + % (fullName, self.lastBackupStopTime, mTime), + ) + if mTime > self.lastBackupStopTime + 1: + bareosfd.DebugMessage( + context, + 150, + "file: %s, fullTime: %d mtime: %d\n" + % (fullName, self.lastBackupStopTime, mTime), + ) + self.files_to_backup.append(fullName) + if os.path.isdir(fullName) and fileName not in self.ignoreSubdirs: + for topdir, dirNames, fileNames in os.walk(fullName): + for fileName in fileNames: + self.files_to_backup.append(os.path.join(topdir, fileName)) + for dirName in dirNames: + fullDirName = os.path.join(topdir, dirName) + "/" + self.files_to_backup.append(fullDirName) + + # If level is not Full, we are done here and set the new + # lastBackupStopTime as reference for future jobs + # Will be written into the Restore Object + if not chr(self.level) == "F": + self.lastBackupStopTime = int(time.time()) + return bRCs["bRC_OK"] + + # For Full we check for a running job and tell Postgres that + # we want to backup the DB files now. + if os.path.exists(self.labelFileName): + self.parseBackupLabelFile(context) + bareosfd.JobMessage( + context, + bJobMessageType["M_FATAL"], + 'Another Postgres Backup Operation "%s" is in progress. ' + % self.labelItems["LABEL"] + + "You may stop it using SELECT pg_stop_backup()", + ) + return bRCs["bRC_Error"] + + bareosfd.DebugMessage( + context, 100, "Send 'SELECT pg_start_backup' to Postgres\n" + ) + # We tell Postgres that we want to start to backup file now + self.backupStartTime = datetime.datetime.now( + tz=dateutil.tz.tzoffset(None, self.tzOffset) + ) + if not self.execute_SQL(context, "SELECT pg_start_backup('%s');" % self.backupLabelString): + bareosfd.JobMessage( + context, + bJobMessageType["M_FATAL"], + 'pg_start_backup statement failed.', + ) + return bRCs["bRC_Error"] + results = self.dbCursor.fetchall() + bareosfd.DebugMessage(context, 150, "Start response: %s\n" % str(results)) + bareosfd.DebugMessage( + context, 150, "Adding label file %s to fileset\n" % self.labelFileName + ) + self.files_to_backup.append(self.labelFileName) + bareosfd.DebugMessage( + context, 150, "Filelist: %s\n" % (self.files_to_backup), + ) + self.PostgressFullBackupRunning = True + return bRCs["bRC_OK"] + + def parseBackupLabelFile(self, context): + try: + labelFile = open(self.labelFileName, "rb") + except: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could not open Label File %s" % (self.labelFileName), + ) + + for labelItem in labelFile.read().splitlines(): + print labelItem + k, v = labelItem.split(":", 1) + self.labelItems.update({k.strip(): v.strip()}) + labelFile.close() + bareosfd.DebugMessage(context, 150, "Labels read: %s\n" % str(self.labelItems)) + + def start_backup_file(self, context, savepkt): + """ + For normal files we call the super method + Special objects are treated here + """ + if not self.files_to_backup: + bareosfd.DebugMessage(context, 100, "No files to backup\n") + return bRCs["bRC_Skip"] + + # Plain files are handled by super class + if self.files_to_backup[-1] not in ["ROP"]: + return super(BareosFdPluginPostgres, self).start_backup_file( + context, savepkt + ) + + # Here we create the restore object + self.file_to_backup = self.files_to_backup.pop() + bareosfd.DebugMessage(context, 100, "file: " + self.file_to_backup + "\n") + savepkt.statp = StatPacket() + if self.file_to_backup == "ROP": + self.rop_data["lastBackupStopTime"] = self.lastBackupStopTime + self.rop_data["lastLSN"] = self.lastLSN + savepkt.fname = "/_bareos_postgres_plugin/metadata" + savepkt.type = bFileType["FT_RESTORE_FIRST"] + savepkt.object_name = savepkt.fname + bareosfd.DebugMessage(context, 150, "fname: " + savepkt.fname + "\n") + bareosfd.DebugMessage(context, 150, "rop " + str(self.rop_data) + "\n") + savepkt.object = bytearray(json.dumps(self.rop_data)) + savepkt.object_len = len(savepkt.object) + savepkt.object_index = int(time.time()) + else: + # should not happen + JobMessage( + context, + bJobMessageType["M_FATAL"], + "Unknown error. Don't know how to handle %s\n" % self.file_to_backup, + ) + return bRCs["bRC_OK"] + + def restore_object_data(self, context, ROP): + """ + Called on restore and on diff/inc jobs. + """ + # Improve: sanity / consistence check of restore object + self.row_rop_raw = ROP.object + try: + self.rop_data[ROP.jobid] = json.loads(str(self.row_rop_raw)) + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could net parse restore object json-data \"%s\ / \"%s\"" % (self.row_rop_raw, e.message), + ) + + if "lastBackupStopTime" in self.rop_data[ROP.jobid]: + self.lastBackupStopTime = int( + self.rop_data[ROP.jobid]["lastBackupStopTime"] + ) + JobMessage( + context, + bJobMessageType["M_INFO"], + "Got lastBackupStopTime %d from restore object of job %d\n" + % (self.lastBackupStopTime, ROP.jobid), + ) + if "lastLSN" in self.rop_data[ROP.jobid]: + self.lastLSN = self.rop_data[ROP.jobid]["lastLSN"] + JobMessage( + context, + bJobMessageType["M_INFO"], + "Got lastLSN %s from restore object of job %d\n" + % (self.lastLSN, ROP.jobid), + ) + return bRCs["bRC_OK"] + + def closeDbConnection(self, context): + # TODO Error Handling + # Get Backup Start Date + self.parseBackupLabelFile(context) + # self.execute_SQL("SELECT pg_backup_start_time()") + # self.backupStartTime = self.dbCursor.fetchone()[0] + # Tell Postgres we are done + if self.execute_SQL(context, "SELECT pg_stop_backup();"): + self.lastLSN = self.dbCursor.fetchone()[0].zfill(17) + self.lastBackupStopTime = int(time.time()) + results = self.dbCursor.fetchall() + bareosfd.JobMessage( + context, + bJobMessageType["M_INFO"], + "Database connection closed. " + + "CHECKPOINT LOCATION: %s, " % self.labelItems["CHECKPOINT LOCATION"] + + "START WAL LOCATION: %s\n" % self.labelItems["START WAL LOCATION"], + ) + self.PostgressFullBackupRunning = False + else: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + 'pg_stop_backup statement failed.', + ) + + + def checkForWalFiles(self, context): + """ + Look for new WAL files and backup + Backup start time is timezone aware, we need to add timezone + to files' mtime to make them comparable + """ + # We have to add local timezone to the file's timestamp in order + # to compare them with the backup starttime, which has a timezone + walArchive = self.options["walArchive"] + self.files_to_backup.append(walArchive) + for fileName in os.listdir(walArchive): + fullPath = os.path.join(walArchive, fileName) + try: + st = os.stat(fullPath) + except Exception as e: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "Could net get stat-info for file %s: \"%s\"" % (fullPath, e.message), + ) + continue + fileMtime = datetime.datetime.fromtimestamp(st.st_mtime) + if ( + fileMtime.replace(tzinfo=dateutil.tz.tzoffset(None, self.tzOffset)) + > self.backupStartTime + ): + bareosfd.DebugMessage( + context, 150, "Adding WAL file %s for backup\n" % fileName + ) + self.files_to_backup.append(fullPath) + + if self.files_to_backup: + return bRCs["bRC_More"] + else: + return bRCs["bRC_OK"] + + def end_backup_file(self, context): + """ + Here we return 'bRC_More' as long as our list files_to_backup is not + empty and bRC_OK when we are done + """ + bareosfd.DebugMessage( + context, 100, "end_backup_file() entry point in Python called\n" + ) + if self.files_to_backup: + return bRCs["bRC_More"] + else: + if self.PostgressFullBackupRunning: + self.closeDbConnection(context) + # Now we can also create the Restore object with the right timestamp + self.files_to_backup.append("ROP") + return self.checkForWalFiles(context) + else: + return bRCs["bRC_OK"] + + def end_backup_job(self, context): + """ + Called if backup job ends, before ClientAfterJob + Make sure that dbconnection was closed in any way, + especially when job was cancelled + """ + if self.PostgressFullBackupRunning: + self.closeDbConnection(context) + self.PostgressFullBackupRunning = False + return bRCs["bRC_OK"] + + +# vim: ts=4 tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/core/src/plugins/filed/bareos-fd-postgres.py b/core/src/plugins/filed/bareos-fd-postgres.py new file mode 100644 index 00000000000..d2330186562 --- /dev/null +++ b/core/src/plugins/filed/bareos-fd-postgres.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# BAREOS - Backup Archiving REcovery Open Sourced +# +# Copyright (C) 2014-2014 Bareos GmbH & Co. KG +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of version three of the GNU Affero General Public +# License as published by the Free Software Foundation, which is +# listed in the file LICENSE. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# Bareos-fd-postgres is a plugin to backup Postgres SQL databases using +# BareosFdPluginPostgres. +# TODO: The plugin argument 'filename' is used to read +# all files listed in that file and add it to the fileset +# +# Author: Maik Aussendorf +# + +# Provided by the Bareos FD Python plugin interface +import bareos_fd_consts + +# This module contains the wrapper functions called by the Bareos-FD, the +# functions call the corresponding methods from your plugin class +import BareosFdWrapper + +# from BareosFdWrapper import parse_plugin_definition, handle_plugin_event, start_backup_file, end_backup_file, start_restore_file, end_restore_file, restore_object_data, plugin_io, create_file, check_file, handle_backup_file # noqa +from BareosFdWrapper import * # noqa + +# This module contains the used plugin class +import BareosFdPluginPostgres + + +def load_bareos_plugin(context, plugindef): + """ + This function is called by the Bareos-FD to load the plugin + We use it to instantiate the plugin class + """ + # BareosFdWrapper.bareos_fd_plugin_object is the module attribute that + # holds the plugin class object + BareosFdWrapper.bareos_fd_plugin_object = BareosFdPluginPostgres.BareosFdPluginPostgres( + context, plugindef + ) + return bareos_fd_consts.bRCs["bRC_OK"] + + +# the rest is done in the Plugin module diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index 0d3954d119a..e6752ce5561 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1052,6 +1052,7 @@ Percona XtraBackup Plugin :index:`\ ` :index:`\ ` :index:`\ ` +:index:`\ ` This plugin uses Perconas XtraBackup tool, to make full and incremental backups of Mysql / MariaDB databases. @@ -1164,7 +1165,156 @@ If things don't work as expected, make sure that - Make sure *XtraBackup* works as user root, MySQL access needs to be configured properly +PostgreSQL Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:index:`\ ` + + +This plugin uses the standard API |postgresql| backup routines based on *pg_start_backup()* and *pg_stop_backup()*. + +The key features are: + +* Incremental backups +* Point in time recovery +* Backups that complete quickly and reliably +* Uninterrupted transaction processing during backups +* Savings on disk space and network bandwidth +* Higher uptime due to faster restore time + +Requires |postgresql| Version 9.x or newer. + + +Concept +^^^^^^^^^^^^^ + +Please make sure to read the |postgresql| documentation about the backup and restore process: https://www.postgresql.org/docs/current/continuous-archiving.html + +This is just a short outline of the tasks performed by the plugin. + +#. Notify Postgres that we want to start backup the database files using the *SELECT pg_start_backup()* statement +#. Backup database files +#. Notify Postgres when done with file backups using the *SELECT pg_stop_backup()* statement +#. Postgres will write *Write-Ahead-Logfiles* - WAL - into the WAL Archive. These transaction logs contain transactions done while the file backup proceeded +#. Backup fresh created WAL files + +Incremental and Differential backups will only have to backup WAL files created since last reference backup. + +The restore basically works like this: + +#. Restore all files to the original Postgres location +#. Create a recovery.conf file (see below) +#. Start Postgres +#. Postgres will restore the latest possible consistent point in time. You can manage to restore to any other point in in time available in the WAL files, please refer to the Postgres documentation for more details. + + +Prerequisites +^^^^^^^^^^^^^ + +As it is a Python plugin, it will also require to have the package **bareos-filedaemon-python-plugin** installed on the |fd|, where you run it. + +The plugin requires the Python module psycopg2 to be installed in your python2 environment. + +**You have to enable WAL-Archiving** - the process and the plugin depend on it. + +Installation +^^^^^^^^^^^^ + +Make sure you have met the prerequisites, after that install the package **bareos-filedaemon-postgres-python-plugin**. + +The plugin must be installed on the same server where the |postgresql| database runs. + +Configuration +^^^^^^^^^^^^^ + +Activate your plugin directory in the |fd| configuration. See :ref:`fdPlugins` for more about plugins in general. + +.. code-block:: bareosconfig + :caption: bareos-fd.d/client/myself.conf + + Client { + ... + Plugin Directory = /usr/lib64/bareos/plugins + Plugin Names = "python" + } + +Now include the plugin as command-plugin in the fileset resource and define a job using this fileset: + +.. code-block:: bareosconfig + :caption: bareos-dir.d/fileset/postgres.conf + + FileSet { + Name = "postgres" + Include { + Options { + compression=GZIP + signature = MD5 + } + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-postgres:postgresDataDir=/var/lib/pgsql/data:walArchive=/var/lib/pgsql/wal_archive/" + } + } + + + +You can append options to the plugin call as key=value pairs, separated by ’:’. The following options are available: + +- :strong:`postgresDataDir` the Postgres data directory. Default: :file:`/var/lib/pgsql/data` + +- :strong:`walArchive` directory where Postgres archives the WAL files as defined in your :file:`postgresql.conf` with the *archive_command* directive. This is a **mandatory** option, there is no default set. + +- :strong:`dbuser` with this user the plugin will try to connect to the database. Default: *root* + +- :strong:`dbname` there needs to be a named database for the connection. Default: *postgres* + +- :strong:`ignoreSubdirs` a list of comma separated directories below the *postgresDataDir*, that will not be backed up. Default: *pg_wal,pg_log,pg_xlog* + +- :strong:`switchWal` If set to *true* (default), the plugin will let Postgres write a new wal file, if the current Log Sequence Number (LSN) is greater than the LSN from the previous job to make sure changes will go into the backup. + +Restore +^^^^^^^ + +With the usual Bareos restore mechanism a file-hierarchy will be created on the restore client under the default restore location according to the options set: + +- :file:`//` +- :file:`//` + +You need to place a minimal :file:`recovery.conf` in your Postgres datadir, Example: + +.. code-block:: cfg + :caption: recovery.conf + + restore_command = 'cp /var/lib/pgsql/wal_archive/%f %p' + + +Where :file:`/var/lib/pgsql/wal_archive/` is the *walArchive* directory. Starting the |postgresql| server shall now initiate the recovery process. Make sure that the user *postgres* is allowed to rename the :file:`recovery.conf` file. You might have to disable or adapt your SELINUX configuration on some installations. + +Troubleshooting +^^^^^^^^^^^^^^^ + +If things don't work as expected, make sure that + +- the |fd| (FD) works in general, so that you can make simple file backups and restores +- the Bareos FD Python plugins work in general, try one of + the shipped simple sample plugins +- check your Postgres data directory for a file named backup_label. If it exists, another backup-process is already running. This file contains an entry like *LABEL: SomeLabel*. If the backup was triggered by this plugin, the label will look like: *LABEL: Bareos.pgplugin.jobid.*. + You may want to stop it using the *SELECT pg_stop_backup()* statement. +- make sure your *dbuser* can connect to the database *dbname* and is allowed to issue the following statements: + +.. code-block:: sql + + SELECT current_setting('server_version_num') + -- Postgres version >= 9 + SELECT pg_start_backup() + SELECT pg_backup_start_time()" + SELECT pg_stop_backup() + -- Postgres version >=10: + SELECT pg_current_wal_lsn() + SELECT pg_switch_wal() + -- Postgres version 9 only: + SELECT pg_current_xlog_location() + SELECT pg_switch_xlog() + Support is available here: https://www.bareos.com From 59ea1d59499b5d0db3f033c7c3a11e3848ca628d Mon Sep 17 00:00:00 2001 From: Frank Ueberschar Date: Mon, 20 Apr 2020 15:59:01 +0200 Subject: [PATCH 3/9] systemtests: perform complete Postgres full/incremental backups and restore validation --- .../plugins/filed/BareosFdPluginPostgres.py | 6 +- systemtests/CMakeLists.txt | 15 + .../database/setup_local_db.sh | 23 ++ .../bareos-dir.d/catalog/MyCatalog.conf.in | 11 + .../bareos-dir.d/client/bareos-fd.conf.in | 7 + .../bareos-dir.d/console/bareos-mon.conf.in | 7 + .../bareos-dir.d/director/bareos-dir.conf.in | 27 ++ .../bareos-dir.d/fileset/Catalog.conf.in | 11 + .../bareos-dir.d/fileset/PluginTest.conf.in | 10 + .../bareos-dir.d/fileset/SelfTest.conf.in | 11 + .../bareos-dir.d/job/BackupCatalog.conf.in | 20 + .../bareos-dir.d/job/RestoreFiles.conf.in | 11 + .../bareos-dir.d/job/backup-bareos-fd.conf.in | 6 + .../bareos-dir.d/jobdefs/DefaultJob.conf.in | 15 + .../bareos-dir.d/messages/Daemon.conf.in | 7 + .../bareos-dir.d/messages/Standard.conf.in | 7 + .../bareos-dir.d/pool/Differential.conf | 10 + .../etc/bareos/bareos-dir.d/pool/Full.conf | 10 + .../bareos/bareos-dir.d/pool/Incremental.conf | 10 + .../etc/bareos/bareos-dir.d/pool/Scratch.conf | 4 + .../bareos/bareos-dir.d/profile/operator.conf | 18 + .../bareos/bareos-dir.d/storage/File.conf.in | 8 + .../bareos/bareos-fd.d/client/myself.conf.in | 19 + .../bareos-fd.d/director/bareos-dir.conf.in | 5 + .../bareos-fd.d/director/bareos-mon.conf.in | 6 + .../bareos/bareos-fd.d/messages/Standard.conf | 5 + .../bareos-sd.d/device/FileStorage.conf | 11 + .../bareos-sd.d/director/bareos-dir.conf.in | 5 + .../bareos-sd.d/director/bareos-mon.conf.in | 6 + .../bareos/bareos-sd.d/messages/Standard.conf | 5 + .../bareos-sd.d/storage/bareos-sd.conf.in | 14 + .../etc/bareos/bconsole.conf.in | 10 + .../client/FileDaemon-local.conf.in | 5 + .../director/Director-local.conf.in | 4 + .../tray-monitor.d/monitor/bareos-mon.conf.in | 7 + .../storage/StorageDaemon-local.conf.in | 5 + ...sFdPluginLocalFilesetWithRestoreObjects.py | 353 ++++++++++++++++++ ...os-fd-local-fileset-with-restoreobjects.py | 59 +++ .../python-fd-plugin-postgres-test/testrunner | 130 +++++++ 39 files changed, 902 insertions(+), 1 deletion(-) create mode 100755 systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Differential.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Full.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Incremental.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Scratch.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/profile/operator.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/storage/File.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/client/myself.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/messages/Standard.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/device/FileStorage.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/messages/Standard.conf create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bconsole.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/python-modules/BareosFdPluginLocalFilesetWithRestoreObjects.py create mode 100644 systemtests/tests/python-fd-plugin-postgres-test/python-modules/bareos-fd-local-fileset-with-restoreobjects.py create mode 100755 systemtests/tests/python-fd-plugin-postgres-test/testrunner diff --git a/core/src/plugins/filed/BareosFdPluginPostgres.py b/core/src/plugins/filed/BareosFdPluginPostgres.py index 2f1bcb488c9..8576e7b2680 100644 --- a/core/src/plugins/filed/BareosFdPluginPostgres.py +++ b/core/src/plugins/filed/BareosFdPluginPostgres.py @@ -62,6 +62,8 @@ def __init__(self, context, plugindef): self.dbCon = None self.dbCursor = None + # Additional db options for psycopg2 connectino + self.dbOpts = "" # This will be set to True between SELCET pg_start_backup # and SELECT pg_stop_backup. We backup database file during # this time @@ -115,6 +117,8 @@ def check_options(self, context, mandatory_options=None): self.switchWal = True else: self.switchWal = self.options["switchWal"].lower() == "true" + if 'dbHost' in self.options: + self.dbOpts += " host='%s'" %self.options["dbHost"] return bRCs["bRC_OK"] def execute_SQL(self, context, sqlStatement): @@ -142,7 +146,7 @@ def start_backup_job(self, context): ) try: self.dbCon = psycopg2.connect( - "dbname=%s user=%s" % (self.dbname, self.dbuser) + "dbname=%s user=%s %s" % (self.dbname, self.dbuser, self.dbOpts) ) self.dbCursor = self.dbCon.cursor() self.dbCursor.execute("SELECT current_setting('server_version_num')") diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index 1011b66d4e5..3ce64ddc9ae 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -171,6 +171,14 @@ macro(handle_python_plugin_modules test_name) ) endif() + if(${test_name} STREQUAL python-fd-plugin-postgres-test) + list(APPEND FD_PYMODULES_TO_LINK_TO_SRC bareos-fd-postgres.py + BareosFdPluginLocalFileset.py + BareosFdPluginPostgres.py + BareosFdPluginBaseclass.py + ) + endif() + # still missing: filed/BareosFdPluginLDAP.py filed/bareos-fd-ldap.py if(NOT EXISTS ${python_plugin_module_src_test_dir}) @@ -334,6 +342,7 @@ macro(prepare_test) string(REPLACE "-" "_" db_name "regress-${TEST_NAME}") # set(db_name "regress-${TEST_NAME}") set(db_user "regress") + set(db_address "$current_test_directory"/database/tmp) set(job_email root@localhost) @@ -672,6 +681,12 @@ else() list(APPEND SYSTEM_TESTS_DISABLED "python-fd-plugin-local-fileset-test") endif() +if(TARGET python-fd) + list(APPEND SYSTEM_TESTS "python-fd-plugin-postgres-test") +else() + list(APPEND SYSTEM_TESTS_DISABLED "python-fd-plugin-postgres-test") +endif() + if(TARGET python-fd AND ovirt_server) list(APPEND SYSTEM_TESTS "python-fd-ovirt-plugin-test") else() diff --git a/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh b/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh new file mode 100755 index 00000000000..e54b27cdb48 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/database/setup_local_db.sh @@ -0,0 +1,23 @@ + +pg_ctl -D data stop + + +rm --recursive --force tmp data log +mkdir tmp data log wal_archive +LANG= pg_ctl -D data -l log/postgres.log initdb + +sed -i.bak "s@#listen_addresses.*@listen_addresses = ''@g" data/postgresql.conf +sed -i.bak "s@#unix_socket_directories.*@unix_socket_directories = \'$(pwd)/tmp\'@g" data/postgresql.conf + +# for online backups we need wal_archiving +echo "wal_level = archive" >> data/postgresql.conf +echo "archive_mode = on" >> data/postgresql.conf +echo "archive_command = 'cp %p @current_test_directory@/database/wal_archive'" >> data/postgresql.conf +echo "max_wal_senders = 10" >> data/postgresql.conf + +pg_ctl -D data -l log/logfile start +sleep 10 + +echo "CREATE ROLE root WITH SUPERUSER CREATEDB CREATEROLE REPLICATION LOGIN" |psql -h @current_test_directory@/database/tmp postgres + +echo stop server with "pg_ctl -D data stop" diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in new file mode 100644 index 00000000000..2995cbcdfcf --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in @@ -0,0 +1,11 @@ +Catalog { + Name = MyCatalog + #dbdriver = "@DEFAULT_DB_TYPE@" + dbdriver = "XXX_REPLACE_WITH_DATABASE_DRIVER_XXX" + dbname = "@db_name@" + dbuser = "@db_user@" + dbpassword = "@db_password@" + # this would rund the catalog in the local database + # for now we use it for backup and restore tests, only + #dbaddress = "@dbaddress@" +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in new file mode 100644 index 00000000000..470ca702035 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in @@ -0,0 +1,7 @@ +Client { + Name = bareos-fd + Description = "Client resource of the Director itself." + Address = localhost + Password = "@fd_password@" # password for FileDaemon + FD PORT = @fd_port@ +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in new file mode 100644 index 00000000000..d276adcb87d --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in @@ -0,0 +1,7 @@ +Console { + Name = bareos-mon + Description = "Restricted console used by tray-monitor to get the status of the director." + Password = "@mon_dir_password@" + CommandACL = status, .status + JobACL = *all* +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in new file mode 100644 index 00000000000..8346e62ddc7 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in @@ -0,0 +1,27 @@ +Director { # define myself + Name = bareos-dir + QueryFile = "@scriptdir@/query.sql" + Maximum Concurrent Jobs = 10 + Password = "@dir_password@" # Console password + Messages = Daemon + Auditing = yes + + # Enable the Heartbeat if you experience connection losses + # (eg. because of your router or firewall configuration). + # Additionally the Heartbeat can be enabled in bareos-sd and bareos-fd. + # + # Heartbeat Interval = 1 min + + # remove comment in next line to load dynamic backends from specified directory + Backend Directory = @backenddir@ + + # remove comment from "Plugin Directory" to load plugins from specified directory. + # if "Plugin Names" is defined, only the specified plugins will be loaded, + # otherwise all director plugins (*-dir.so) from the "Plugin Directory". + # + # Plugin Directory = "@dir_plugin_binary_path@" + # Plugin Names = "" + Working Directory = "@working_dir@" + Pid Directory = "@piddir@" + DirPort = @dir_port@ +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in new file mode 100644 index 00000000000..c7cdc433fe1 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in @@ -0,0 +1,11 @@ +FileSet { + Name = "Catalog" + Description = "Backup the catalog dump and Bareos configuration files." + Include { + Options { + signature = MD5 + } + File = "@working_dir@/@db_name@.sql" # database dump + File = "@confdir@" # configuration + } +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in new file mode 100644 index 00000000000..970a8ef197d --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/PluginTest.conf.in @@ -0,0 +1,10 @@ +FileSet { + Name = "PluginTest" + Description = "Test the Plugin functionality with a Python Plugin." + Include { + Options { + signature = MD5 + } + Plugin = "python:module_path=/home/maik/git/bareos/build/systemtests/tests/python-fd-plugin-postgres-test/python-modules:module_name=bareos-fd-postgres:dbHost=@current_test_directory@/database/tmp:postgresDataDir=@current_test_directory@/database/data:walArchive=@current_test_directory@/database/wal_archive/" + } +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in new file mode 100644 index 00000000000..ba39719ea3f --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in @@ -0,0 +1,11 @@ +FileSet { + Name = "SelfTest" + Description = "fileset just to backup some files for selftest" + Include { + Options { + Signature = MD5 # calculate md5 checksum per file + } + #File = "@sbindir@" + File=<@tmpdir@/file-list + } +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in new file mode 100644 index 00000000000..1da2a7af657 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in @@ -0,0 +1,20 @@ +Job { + Name = "BackupCatalog" + Description = "Backup the catalog database (after the nightly save)" + JobDefs = "DefaultJob" + Level = Full + FileSet="Catalog" + + # This creates an ASCII copy of the catalog + # Arguments to make_catalog_backup.pl are: + # make_catalog_backup.pl + RunBeforeJob = "@scriptdir@/make_catalog_backup.pl MyCatalog" + + # This deletes the copy of the catalog + RunAfterJob = "@scriptdir@/delete_catalog_backup" + + # This sends the bootstrap via mail for disaster recovery. + # Should be sent to another system, please change recipient accordingly + Write Bootstrap = "|@bindir@/bsmtp -h @smtp_host@ -f \"\(Bareos\) \" -s \"Bootstrap for Job %j\" @job_email@" # (#01) + Priority = 11 # run after main backup +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in new file mode 100644 index 00000000000..89256864d9a --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in @@ -0,0 +1,11 @@ +Job { + Name = "RestoreFiles" + Description = "Standard Restore template. Only one such job is needed for all standard Jobs/Clients/Storage ..." + Type = Restore + Client = bareos-fd + FileSet = SelfTest + Storage = File + Pool = Incremental + Messages = Standard + Where = @tmp@/bareos-restores +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in new file mode 100644 index 00000000000..d54d7ff8e17 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in @@ -0,0 +1,6 @@ +Job { + Name = "backup-bareos-fd" + JobDefs = "DefaultJob" + Client = "bareos-fd" + FileSet = "PluginTest" +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in new file mode 100644 index 00000000000..563126477c9 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in @@ -0,0 +1,15 @@ +JobDefs { + Name = "DefaultJob" + Type = Backup + Level = Incremental + Client = bareos-fd + FileSet = "SelfTest" + Storage = File + Messages = Standard + Pool = Incremental + Priority = 10 + Write Bootstrap = "@working_dir@/%c.bsr" + Full Backup Pool = Full # write Full Backups into "Full" Pool + Differential Backup Pool = Differential # write Diff Backups into "Differential" Pool + Incremental Backup Pool = Incremental # write Incr Backups into "Incremental" Pool +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in new file mode 100644 index 00000000000..cf6a8cfa1e2 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in @@ -0,0 +1,7 @@ +Messages { + Name = Daemon + Description = "Message delivery for daemon messages (no job)." + console = all, !skipped, !saved, !audit + append = "@logdir@/bareos.log" = all, !skipped, !audit + append = "@logdir@/bareos-audit.log" = audit +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in new file mode 100644 index 00000000000..b3556ba8c23 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in @@ -0,0 +1,7 @@ +Messages { + Name = Standard + Description = "Reasonable message delivery -- send most everything to email address and to the console." + console = all, !skipped, !saved, !audit + append = "@logdir@/bareos.log" = all, !skipped, !saved, !audit + catalog = all, !skipped, !saved, !audit +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Differential.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Differential.conf new file mode 100644 index 00000000000..25ce24821ab --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Differential.conf @@ -0,0 +1,10 @@ +Pool { + Name = Differential + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 90 days # How long should the Differential Backups be kept? (#09) + Maximum Volume Bytes = 10G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Differential-" # Volumes will be labeled "Differential-" +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Full.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Full.conf new file mode 100644 index 00000000000..867fc66b483 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Full.conf @@ -0,0 +1,10 @@ +Pool { + Name = Full + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365 days # How long should the Full Backups be kept? (#06) + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Full-" # Volumes will be labeled "Full-" +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Incremental.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Incremental.conf new file mode 100644 index 00000000000..f4dbbab6650 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Incremental.conf @@ -0,0 +1,10 @@ +Pool { + Name = Incremental + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 30 days # How long should the Incremental Backups be kept? (#12) + Maximum Volume Bytes = 1G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Incremental-" # Volumes will be labeled "Incremental-" +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Scratch.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Scratch.conf new file mode 100644 index 00000000000..3a489b19871 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/pool/Scratch.conf @@ -0,0 +1,4 @@ +Pool { + Name = Scratch + Pool Type = Scratch +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/profile/operator.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/profile/operator.conf new file mode 100644 index 00000000000..6edd0166dca --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/profile/operator.conf @@ -0,0 +1,18 @@ +Profile { + Name = operator + Description = "Profile allowing normal Bareos operations." + + Command ACL = !.bvfs_clear_cache, !.exit, !.sql + Command ACL = !configure, !create, !delete, !purge, !prune, !sqlquery, !umount, !unmount + Command ACL = *all* + + Catalog ACL = *all* + Client ACL = *all* + FileSet ACL = *all* + Job ACL = *all* + Plugin Options ACL = *all* + Pool ACL = *all* + Schedule ACL = *all* + Storage ACL = *all* + Where ACL = *all* +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/storage/File.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/storage/File.conf.in new file mode 100644 index 00000000000..4058ddc7edc --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-dir.d/storage/File.conf.in @@ -0,0 +1,8 @@ +Storage { + Name = File + Address = @hostname@ # N.B. Use a fully qualified name here (do not use "localhost" here). + Password = "@sd_password@" + Device = FileStorage + Media Type = File + SD Port = @sd_port@ +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/client/myself.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/client/myself.conf.in new file mode 100644 index 00000000000..49039c29d18 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/client/myself.conf.in @@ -0,0 +1,19 @@ +Client { + Name = @basename@-fd + Maximum Concurrent Jobs = 20 + + # remove comment from "Plugin Directory" to load plugins from specified directory. + # if "Plugin Names" is defined, only the specified plugins will be loaded, + # otherwise all filedaemon plugins (*-fd.so) from the "Plugin Directory". + # + Plugin Directory = "@fd_plugin_binary_path@" + Plugin Names = "python" + + # if compatible is set to yes, we are compatible with bacula + # if set to no, new bareos features are enabled which is the default + # compatible = yes + + Working Directory = "@working_dir@" + Pid Directory = "@piddir@" + FD Port = @fd_port@ +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in new file mode 100644 index 00000000000..c8dc7085a45 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in @@ -0,0 +1,5 @@ +Director { + Name = bareos-dir + Password = "@fd_password@" + Description = "Allow the configured Director to access this file daemon." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in new file mode 100644 index 00000000000..630c3a9abd3 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in @@ -0,0 +1,6 @@ +Director { + Name = bareos-mon + Password = "@mon_fd_password@" + Monitor = yes + Description = "Restricted Director, used by tray-monitor to get the status of this file daemon." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/messages/Standard.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/messages/Standard.conf new file mode 100644 index 00000000000..97788e00573 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-fd.d/messages/Standard.conf @@ -0,0 +1,5 @@ +Messages { + Name = Standard + Director = bareos-dir = all, !skipped, !restored + Description = "Send relevant messages to the Director." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/device/FileStorage.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/device/FileStorage.conf new file mode 100644 index 00000000000..11a639bc688 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/device/FileStorage.conf @@ -0,0 +1,11 @@ +Device { + Name = FileStorage + Media Type = File + Archive Device = storage + LabelMedia = yes; # lets Bareos label unlabeled media + Random Access = yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; + Description = "File device. A connecting Director must have the same Name and MediaType." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in new file mode 100644 index 00000000000..deef3360c2d --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in @@ -0,0 +1,5 @@ +Director { + Name = bareos-dir + Password = "@sd_password@" + Description = "Director, who is permitted to contact this storage daemon." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in new file mode 100644 index 00000000000..e3cfdee6315 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in @@ -0,0 +1,6 @@ +Director { + Name = bareos-mon + Password = "@mon_sd_password@" + Monitor = yes + Description = "Restricted Director, used by tray-monitor to get the status of this storage daemon." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/messages/Standard.conf b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/messages/Standard.conf new file mode 100644 index 00000000000..468348e62fc --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/messages/Standard.conf @@ -0,0 +1,5 @@ +Messages { + Name = Standard + Director = bareos-dir = all + Description = "Send all messages to the Director." +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in new file mode 100644 index 00000000000..3e1723fa60c --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in @@ -0,0 +1,14 @@ +Storage { + Name = bareos-sd + Maximum Concurrent Jobs = 20 + + # remove comment from "Plugin Directory" to load plugins from specified directory. + # if "Plugin Names" is defined, only the specified plugins will be loaded, + # otherwise all storage plugins (*-sd.so) from the "Plugin Directory". + # + # Plugin Directory = "@python_plugin_module_src_sd@" + # Plugin Names = "" + Working Directory = "@working_dir@" + Pid Directory = "@piddir@" + SD Port = @sd_port@ +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bconsole.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bconsole.conf.in new file mode 100644 index 00000000000..ecb6ad00dce --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/bconsole.conf.in @@ -0,0 +1,10 @@ +# +# Bareos User Agent (or Console) Configuration File +# + +Director { + Name = @basename@-dir + DIRport = @dir_port@ + address = @hostname@ + Password = "@dir_password@" +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in new file mode 100644 index 00000000000..dd00dd9f103 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in @@ -0,0 +1,5 @@ +Client { + Name = @basename@-fd + Address = localhost + Password = "@mon_fd_password@" # password for FileDaemon +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in new file mode 100644 index 00000000000..55dae492250 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in @@ -0,0 +1,4 @@ +Director { + Name = bareos-dir + Address = localhost +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in new file mode 100644 index 00000000000..cddfa253945 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in @@ -0,0 +1,7 @@ +Monitor { + # Name to establish connections to Director Console, Storage Daemon and File Daemon. + Name = bareos-mon + # Password to access the Director + Password = "@mon_dir_password@" # password for the Directors + RefreshInterval = 30 seconds +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in new file mode 100644 index 00000000000..6538f4fd248 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in @@ -0,0 +1,5 @@ +Storage { + Name = bareos-sd + Address = localhost + Password = "@mon_sd_password@" # password for StorageDaemon +} diff --git a/systemtests/tests/python-fd-plugin-postgres-test/python-modules/BareosFdPluginLocalFilesetWithRestoreObjects.py b/systemtests/tests/python-fd-plugin-postgres-test/python-modules/BareosFdPluginLocalFilesetWithRestoreObjects.py new file mode 100644 index 00000000000..7a5c2e5e7a4 --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/python-modules/BareosFdPluginLocalFilesetWithRestoreObjects.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# BAREOS - Backup Archiving REcovery Open Sourced +# +# Copyright (C) 2014-2019 Bareos GmbH & Co. KG +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of version three of the GNU Affero General Public +# License as published by the Free Software Foundation, which is +# listed in the file LICENSE. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# Author: Maik Aussendorf +# +# Bareos python plugins class that adds files from a local list to +# the backup fileset + +import bareosfd +from bareos_fd_consts import bJobMessageType, bFileType, bRCs +import os +import re +import hashlib +import time +import BareosFdPluginBaseclass + + +class BareosFdPluginLocalFilesetWithRestoreObjects( + BareosFdPluginBaseclass.BareosFdPluginBaseclass +): + """ + This Bareos-FD-Plugin-Class was created for automated test purposes only. + It is based on the BareosFdPluginLocalFileset class that parses a file + and backups all files listed there. + Filename is taken from plugin argument 'filename'. + In addition to the file backup and restore, this plugin also tests + restore objects of different sizes. As restore objects are compressed + automatically, when they are greater then 1000 bytes, both uncompressed + and compressed restore objects are tested. + """ + + def __init__(self, context, plugindef): + bareosfd.DebugMessage( + context, + 100, + "Constructor called in module %s with plugindef=%s\n" + % (__name__, plugindef), + ) + # Last argument of super constructor is a list of mandatory arguments + super(BareosFdPluginLocalFilesetWithRestoreObjects, self).__init__( + context, plugindef, ["filename"] + ) + self.files_to_backup = [] + self.allow = None + self.deny = None + self.object_index_seq = int((time.time() - 1546297200) * 10) + self.sha256sums_by_filename = {} + + def filename_is_allowed(self, context, filename, allowregex, denyregex): + """ + Check, if filename is allowed. + True, if matches allowreg and not denyregex. + If allowreg is None, filename always matches + If denyreg is None, it never matches + """ + if allowregex is None or allowregex.search(filename): + allowed = True + else: + allowed = False + if denyregex is None or not denyregex.search(filename): + denied = False + else: + denied = True + if not allowed or denied: + bareosfd.DebugMessage( + context, 100, "File %s denied by configuration\n" % (filename) + ) + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "File %s denied by configuration\n" % (filename), + ) + return False + else: + return True + + def start_backup_job(self, context): + """ + At this point, plugin options were passed and checked already. + We try to read from filename and setup the list of file to backup + in self.files_to_backup + """ + + bareosfd.DebugMessage( + context, + 100, + "Using %s to search for local files\n" % (self.options["filename"]), + ) + if os.path.exists(self.options["filename"]): + try: + config_file = open(self.options["filename"], "rb") + except: + bareosfd.DebugMessage( + context, + 100, + "Could not open file %s\n" % (self.options["filename"]), + ) + return bRCs["bRC_Error"] + else: + bareosfd.DebugMessage( + context, 100, "File %s does not exist\n" % (self.options["filename"]) + ) + return bRCs["bRC_Error"] + # Check, if we have allow or deny regular expressions defined + if "allow" in self.options: + self.allow = re.compile(self.options["allow"]) + if "deny" in self.options: + self.deny = re.compile(self.options["deny"]) + + for listItem in config_file.read().splitlines(): + if os.path.isfile(listItem) and self.filename_is_allowed( + context, listItem, self.allow, self.deny + ): + self.files_to_backup.append(listItem) + if os.path.isdir(listItem): + for topdir, dirNames, fileNames in os.walk(listItem): + for fileName in fileNames: + if self.filename_is_allowed( + context, + os.path.join(topdir, fileName), + self.allow, + self.deny, + ): + self.files_to_backup.append(os.path.join(topdir, fileName)) + if os.path.isfile(os.path.join(topdir, fileName)): + self.files_to_backup.append( + os.path.join(topdir, fileName) + ".sha256sum" + ) + self.files_to_backup.append( + os.path.join(topdir, fileName) + ".abspath" + ) + else: + if os.path.isfile(listItem): + self.files_to_backup.append(listItem + ".sha256sum") + self.files_to_backup.append(listItem + ".abspath") + + for longrestoreobject_length in range(998, 1004): + self.files_to_backup.append( + "%s.longrestoreobject" % longrestoreobject_length + ) + + if not self.files_to_backup: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "No (allowed) files to backup found\n", + ) + return bRCs["bRC_Error"] + else: + return bRCs["bRC_Cancel"] + + def start_backup_file(self, context, savepkt): + """ + Defines the file to backup and creates the savepkt. In this example + only files (no directories) are allowed + """ + bareosfd.DebugMessage(context, 100, "start_backup_file() called\n") + if not self.files_to_backup: + bareosfd.DebugMessage(context, 100, "No files to backup\n") + return bRCs["bRC_Skip"] + + file_to_backup = self.files_to_backup.pop() + bareosfd.DebugMessage(context, 100, "file: " + file_to_backup + "\n") + + statp = bareosfd.StatPacket() + savepkt.statp = statp + + if file_to_backup.endswith(".sha256sum"): + checksum = self.get_sha256sum(context, os.path.splitext(file_to_backup)[0]) + savepkt.type = bFileType["FT_RESTORE_FIRST"] + savepkt.fname = file_to_backup + savepkt.object_name = file_to_backup + savepkt.object = bytearray(checksum) + savepkt.object_len = len(savepkt.object) + savepkt.object_index = self.object_index_seq + self.object_index_seq += 1 + + elif file_to_backup.endswith(".abspath"): + savepkt.type = bFileType["FT_RESTORE_FIRST"] + savepkt.fname = file_to_backup + savepkt.object_name = file_to_backup + savepkt.object = bytearray(os.path.splitext(file_to_backup)[0]) + savepkt.object_len = len(savepkt.object) + savepkt.object_index = self.object_index_seq + self.object_index_seq += 1 + + elif file_to_backup.endswith(".longrestoreobject"): + teststring_length = int(os.path.splitext(file_to_backup)[0]) + savepkt.type = bFileType["FT_RESTORE_FIRST"] + savepkt.fname = file_to_backup + savepkt.object_name = file_to_backup + savepkt.object = bytearray("a" * teststring_length) + savepkt.object_len = len(savepkt.object) + savepkt.object_index = self.object_index_seq + self.object_index_seq += 1 + + else: + savepkt.fname = file_to_backup + savepkt.type = bFileType["FT_REG"] + + bareosfd.JobMessage( + context, + bJobMessageType["M_INFO"], + "Starting backup of %s\n" % (file_to_backup), + ) + return bRCs["bRC_OK"] + + def end_backup_file(self, context): + """ + Here we return 'bRC_More' as long as our list files_to_backup is not + empty and bRC_OK when we are done + """ + bareosfd.DebugMessage( + context, 100, "end_backup_file() entry point in Python called\n" + ) + if self.files_to_backup: + return bRCs["bRC_More"] + else: + return bRCs["bRC_OK"] + + def set_file_attributes(self, context, restorepkt): + bareosfd.DebugMessage( + context, + 100, + "set_file_attributes() entry point in Python called with %s\n" + % (str(restorepkt)), + ) + + orig_fname = "/" + os.path.relpath(restorepkt.ofname, restorepkt.where) + restoreobject_sha256sum = self.sha256sums_by_filename[orig_fname] + + file_sha256sum = self.get_sha256sum(context, orig_fname) + bareosfd.DebugMessage( + context, + 100, + "set_file_attributes() orig_fname: %s restoreobject_sha256sum: %s file_sha256sum: %s\n" + % (orig_fname, repr(restoreobject_sha256sum), repr(file_sha256sum)), + ) + if file_sha256sum != restoreobject_sha256sum: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "bad restoreobject orig_fname: %s restoreobject_sha256sum: %s file_sha256sum: %s\n" + % (orig_fname, repr(restoreobject_sha256sum), repr(file_sha256sum)), + ) + + return bRCs["bRC_OK"] + + def end_restore_file(self, context): + bareosfd.DebugMessage( + context, 100, "end_restore_file() self.FNAME: %s\n" % self.FNAME + ) + return bRCs["bRC_OK"] + + def restore_object_data(self, context, ROP): + """ + Note: + This is called in two cases: + - on diff/inc backup (should be called only once) + - on restore (for every job id being restored) + But at the point in time called, it is not possible + to distinguish which of them it is, because job type + is "I" until the bEventStartBackupJob event + """ + bareosfd.DebugMessage( + context, + 100, + "BareosFdPluginLocalFilesetWithRestoreObjects:restore_object_data() called with ROP:%s\n" + % (ROP), + ) + bareosfd.DebugMessage( + context, + 100, + "ROP.object_name(%s): %s\n" % (type(ROP.object_name), ROP.object_name), + ) + bareosfd.DebugMessage( + context, + 100, + "ROP.plugin_name(%s): %s\n" % (type(ROP.plugin_name), ROP.plugin_name), + ) + bareosfd.DebugMessage( + context, + 100, + "ROP.object_len(%s): %s\n" % (type(ROP.object_len), ROP.object_len), + ) + bareosfd.DebugMessage( + context, + 100, + "ROP.object_full_len(%s): %s\n" + % (type(ROP.object_full_len), ROP.object_full_len), + ) + bareosfd.DebugMessage( + context, 100, "ROP.object(%s): %s\n" % (type(ROP.object), repr(ROP.object)) + ) + orig_filename = os.path.splitext(ROP.object_name)[0] + if ROP.object_name.endswith(".sha256sum"): + self.sha256sums_by_filename[orig_filename] = str(ROP.object) + elif ROP.object_name.endswith(".abspath"): + if str(ROP.object) != orig_filename: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "bad restoreobject orig_fname: %s restoreobject_fname: %s\n" + % (orig_filename, repr(str(ROP.object))), + ) + elif ROP.object_name.endswith(".longrestoreobject"): + stored_length = int(os.path.splitext(ROP.object_name)[0]) + if str(ROP.object) != "a" * stored_length: + bareosfd.JobMessage( + context, + bJobMessageType["M_ERROR"], + "bad long restoreobject %s does not match stored object\n" + % (ROP.object_name), + ) + else: + bareosfd.DebugMessage( + context, + 100, + "not checking restoreobject: %s\n" % (type(ROP.object_name)), + ) + return bRCs["bRC_OK"] + + def get_sha256sum(self, context, filename): + f = open(filename, "rb") + m = hashlib.sha256() + while True: + d = f.read(8096) + if not d: + break + m.update(d) + f.close() + return m.hexdigest() + + +# vim: ts=4 tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtests/tests/python-fd-plugin-postgres-test/python-modules/bareos-fd-local-fileset-with-restoreobjects.py b/systemtests/tests/python-fd-plugin-postgres-test/python-modules/bareos-fd-local-fileset-with-restoreobjects.py new file mode 100644 index 00000000000..b64ef29960f --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/python-modules/bareos-fd-local-fileset-with-restoreobjects.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# BAREOS - Backup Archiving REcovery Open Sourced +# +# Copyright (C) 2014-2014 Bareos GmbH & Co. KG +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of version three of the GNU Affero General Public +# License as published by the Free Software Foundation, which is +# listed in the file LICENSE. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# bareos-fd-local-fileset-with-restoreobjects.py is a python Bareos FD Plugin using +# BareosFdPluginLocalFilesetWithRestoreObjects which was made for automated testing +# purposes. +# +# The plugin argument 'filename' is used to read all files listed in that file and +# add it to the fileset +# +# Author: Maik Aussendorf +# + +# Provided by the Bareos FD Python plugin interface +import bareos_fd_consts + +# This module contains the wrapper functions called by the Bareos-FD, the +# functions call the corresponding methods from your plugin class +import BareosFdWrapper + +# from BareosFdWrapper import parse_plugin_definition, handle_plugin_event, start_backup_file, end_backup_file, start_restore_file, end_restore_file, restore_object_data, plugin_io, create_file, check_file, handle_backup_file # noqa +from BareosFdWrapper import * # noqa + +# This module contains the used plugin class +import BareosFdPluginLocalFilesetWithRestoreObjects + + +def load_bareos_plugin(context, plugindef): + """ + This function is called by the Bareos-FD to load the plugin + We use it to instantiate the plugin class + """ + # BareosFdWrapper.bareos_fd_plugin_object is the module attribute that + # holds the plugin class object + BareosFdWrapper.bareos_fd_plugin_object = BareosFdPluginLocalFilesetWithRestoreObjects.BareosFdPluginLocalFilesetWithRestoreObjects( + context, plugindef + ) + return bareos_fd_consts.bRCs["bRC_OK"] + + +# the rest is done in the Plugin module diff --git a/systemtests/tests/python-fd-plugin-postgres-test/testrunner b/systemtests/tests/python-fd-plugin-postgres-test/testrunner new file mode 100755 index 00000000000..7168a6512de --- /dev/null +++ b/systemtests/tests/python-fd-plugin-postgres-test/testrunner @@ -0,0 +1,130 @@ +#!/bin/bash +# +# This systemtest tests the plugin functionality +# of the Bareos FD by using the supplied module +# bareos-fd-local-fileset.py +# +# The module will backup some files. +# This plugin is not intended for production, +# but is only a minimal example that shows +# how to use the python plugin interface. +# File attributes like uses and times will not be saved. +# +TestName="$(basename "$(pwd)")" +export TestName + +JobName=backup-bareos-fd +#shellcheck source=../environment.in +. ./environment + +JobName=backup-bareos-fd + +# setup local database server +pushd "$current_test_directory"/database || exit 1 +sh -x setup_local_db.sh +popd + +TESTPGHOST="$current_test_directory"/database/tmp +PSQL="psql -h $TESTPGHOST" +DBNAME="backuptest" + +#shellcheck source=../scripts/functions +. "${rscripts}"/functions +"${rscripts}"/cleanup +"${rscripts}"/setup + + + +# Directory to backup. +# This directory will be created by setup_data "$@"(). +BackupDirectory="${tmp}/data" + +# Use a tgz to setup data to be backed up. +# Data will be placed at "${tmp}/data/". +setup_data "$@" + +# this test does not work with links because of the restore objects +#rm -r "${tmp}"/data/weird-files >/dev/null 2>&1 + +# Create Test DB with table and 1 statement +echo "CREATE DATABASE $DBNAME" | ${PSQL} postgres +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +setdebug level=100 storage=File +label volume=TestVolume001 storage=File pool=Full +run job=$JobName yes +status director +status client +status storage=File +wait +messages +quit +END_OF_DATA + +run_bareos "$@" + +# Now add data to the database and run an incremental job + +echo "INSERT INTO t (text, created_on) values ('test for INCR backup', current_timestamp)" | ${PSQL} ${DBNAME} + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +run job=$JobName Level=Incremental yes +wait +END_OF_DATA + +run_bconsole +# run another Incr without db changes - should result in empty backup job (only restore object) +run_bconsole + + +# Now stop database and try a restore +# ... +# shut down database and delete directories +pg_ctl -D database/data stop +rm -Rf database/data +rm -Rf database/wal_archive + +cat <$tmp/bconcmds +@$out /dev/null +messages +messages +@$out $tmp/log1.out +restore client=bareos-fd fileset=pgtest where=/ select all done yes +wait +END_OF_DATA +run_bconsole + +check_for_zombie_jobs storage=File +stop_bareos + +# Create a recovery.conf +# This may become a plugin feature later on +echo "restore_command = 'cp $current_test_directory/database/wal_archive/%f %p'" > $current_test_directory/database/data/recovery.conf + +# start DB again - shall recover to latest possible state +pg_ctl -D database/data -l database/log/logfile start +sleep 10 +echo "SELECT * FROM t" | ${PSQL} ${DBNAME} > $tmp/sql.log + +check_two_logs +if (grep -q "for INCR" $tmp/sql.log) +then + estat=0 +else + estat=1 +fi + +end_test From 2fa4a96609ee7c47a21ed8ecbbd372333f8c9819 Mon Sep 17 00:00:00 2001 From: aussendorf Date: Thu, 23 Apr 2020 15:01:00 +0200 Subject: [PATCH 4/9] postgres-plugin: added try/catch block for module psycopg2 and dbHost parameter --- core/src/plugins/filed/bareos-fd-postgres.py | 11 ++++++++++- docs/manuals/source/TasksAndConcepts/Plugins.rst | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/plugins/filed/bareos-fd-postgres.py b/core/src/plugins/filed/bareos-fd-postgres.py index d2330186562..8741f599888 100644 --- a/core/src/plugins/filed/bareos-fd-postgres.py +++ b/core/src/plugins/filed/bareos-fd-postgres.py @@ -29,7 +29,7 @@ # Provided by the Bareos FD Python plugin interface import bareos_fd_consts - +import bareosfd # This module contains the wrapper functions called by the Bareos-FD, the # functions call the corresponding methods from your plugin class import BareosFdWrapper @@ -46,6 +46,15 @@ def load_bareos_plugin(context, plugindef): This function is called by the Bareos-FD to load the plugin We use it to instantiate the plugin class """ + # Check for needed python modules + try: + import psycopg2 + except Exception as e: + bareosfd.JobMessage( context, + bareos_fd_consts.bJobMessageType["M_FATAL"], + "could not import Python module. %s" % e.message, + ) + return bareos_fd_consts.bRCs["bRC_Error"] # BareosFdWrapper.bareos_fd_plugin_object is the module attribute that # holds the plugin class object BareosFdWrapper.bareos_fd_plugin_object = BareosFdPluginPostgres.BareosFdPluginPostgres( diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index e6752ce5561..bb0addb3ff6 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1267,6 +1267,8 @@ You can append options to the plugin call as key=value pairs, separated by ’: - :strong:`dbname` there needs to be a named database for the connection. Default: *postgres* +- :strong:`dbHost` useful, if socket is not in default location. Specify socket-directory with a leading / here + - :strong:`ignoreSubdirs` a list of comma separated directories below the *postgresDataDir*, that will not be backed up. Default: *pg_wal,pg_log,pg_xlog* - :strong:`switchWal` If set to *true* (default), the plugin will let Postgres write a new wal file, if the current Log Sequence Number (LSN) is greater than the LSN from the previous job to make sure changes will go into the backup. From 556cb0d33119e1099f24199c854e24a50354a5c6 Mon Sep 17 00:00:00 2001 From: Frank Ueberschar Date: Fri, 24 Apr 2020 20:34:51 +0200 Subject: [PATCH 5/9] filed: add packaging for postgresql plugin --- core/platforms/packaging/bareos.spec | 15 +++++++++++++++ core/src/plugins/filed/CMakeLists.txt | 2 ++ 2 files changed, 17 insertions(+) diff --git a/core/platforms/packaging/bareos.spec b/core/platforms/packaging/bareos.spec index a01a719f3e6..704abff4186 100644 --- a/core/platforms/packaging/bareos.spec +++ b/core/platforms/packaging/bareos.spec @@ -577,6 +577,11 @@ Requires: python-pycurl Requires: python-lxml Requires: python-ovirt-engine-sdk4 +%package filedaemon-postgresql-python-plugin +Summary: PostgreSQL Python plugin for Bareos File daemon +Group: Productivity/Archiving/Backup +Requires: bareos-filedaemon = %{version} + %package filedaemon-percona-xtrabackup-python-plugin Summary: Percona xtrabackup Python plugin for Bareos File daemon Group: Productivity/Archiving/Backup @@ -609,6 +614,11 @@ This package contains the LDAP python plugin for the file daemon This package contains the Ovirt python plugin for the file daemon +%description filedaemon-postgresql-python-plugin +%{dscr} + +This package contains the PostgreSQL python plugin for the file daemon + %description filedaemon-percona-xtrabackup-python-plugin %{dscr} @@ -1402,6 +1412,11 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-ovirt.conf.example %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-ovirt.conf.example +%files filedaemon-postgresql-python-plugin +%defattr(-, root, root) +%{plugin_dir}/BareosFdPluginPostgres.py* +%{plugin_dir}/bareos-fd-postgres.py* + %files filedaemon-percona-xtrabackup-python-plugin %defattr(-, root, root) %{plugin_dir}/bareos-fd-percona-xtrabackup.py* diff --git a/core/src/plugins/filed/CMakeLists.txt b/core/src/plugins/filed/CMakeLists.txt index 85af2718121..244f37015c1 100644 --- a/core/src/plugins/filed/CMakeLists.txt +++ b/core/src/plugins/filed/CMakeLists.txt @@ -114,6 +114,8 @@ set(PYFILES BareosFdPluginOvirt.py bareos-fd-percona-xtrabackup.py BareosFdPluginPerconaXtraBackup.py + BareosFdPluginPostgres.py + bareos-fd-postgres.py ) install( From 02a9f4d51d6f43e4eea89a02cf42a82e6d52a483 Mon Sep 17 00:00:00 2001 From: Maik Aussendorf Date: Mon, 27 Apr 2020 11:09:25 +0200 Subject: [PATCH 6/9] postgres-plugin: accept PR comments Typo Co-Authored-By: Frank Ueberschar --- core/src/plugins/filed/BareosFdPluginPostgres.py | 4 ++-- core/src/plugins/filed/bareos-fd-postgres.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginPostgres.py b/core/src/plugins/filed/BareosFdPluginPostgres.py index 8576e7b2680..96d78b7b5f2 100644 --- a/core/src/plugins/filed/BareosFdPluginPostgres.py +++ b/core/src/plugins/filed/BareosFdPluginPostgres.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # BAREOS - Backup Archiving REcovery Open Sourced # -# Copyright (C) 2014-2014 Bareos GmbH & Co. KG +# Copyright (C) 2014-2020 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public @@ -64,7 +64,7 @@ def __init__(self, context, plugindef): self.dbCursor = None # Additional db options for psycopg2 connectino self.dbOpts = "" - # This will be set to True between SELCET pg_start_backup + # This will be set to True between SELECT pg_start_backup # and SELECT pg_stop_backup. We backup database file during # this time self.PostgressFullBackupRunning = False diff --git a/core/src/plugins/filed/bareos-fd-postgres.py b/core/src/plugins/filed/bareos-fd-postgres.py index 8741f599888..5bf353bf4e3 100644 --- a/core/src/plugins/filed/bareos-fd-postgres.py +++ b/core/src/plugins/filed/bareos-fd-postgres.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # BAREOS - Backup Archiving REcovery Open Sourced # -# Copyright (C) 2014-2014 Bareos GmbH & Co. KG +# Copyright (C) 2014-2020 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public From b25dc5ad19d40b0a5a00709ce4b2ec68145a4f95 Mon Sep 17 00:00:00 2001 From: Maik Aussendorf Date: Mon, 27 Apr 2020 11:12:10 +0200 Subject: [PATCH 7/9] docs: postgres-plugin PR comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit format string Co-Authored-By: Jörg Steffens --- docs/manuals/source/TasksAndConcepts/Plugins.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index bb0addb3ff6..8ed7c32fa27 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1166,7 +1166,7 @@ If things don't work as expected, make sure that configured properly PostgreSQL Plugin -~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~ :index:`\ ` @@ -1186,7 +1186,7 @@ Requires |postgresql| Version 9.x or newer. Concept -^^^^^^^^^^^^^ +^^^^^^^ Please make sure to read the |postgresql| documentation about the backup and restore process: https://www.postgresql.org/docs/current/continuous-archiving.html @@ -1688,5 +1688,3 @@ Write your own Python Plugin Some plugin examples are available on https://github.com/bareos/bareos-contrib. The class-based approach lets you easily reuse stuff already defined in the baseclass BareosDirPluginBaseclass, which ships with the **bareos-director-python-plugin** package. The examples contain the plugin bareos-dir-nsca-sender, that submits the results and performance data of a backup job directly to Icinga:index:`\ `\ or Nagios:index:`\ `\ using the NSCA protocol. - - From 310b1be0a194e15dede12b6ec95ee759cbb2f41c Mon Sep 17 00:00:00 2001 From: aussendorf Date: Mon, 27 Apr 2020 11:34:01 +0200 Subject: [PATCH 8/9] postgres-plugin: stop with error, if accurate is enabled - typos and pep8 / black formatted --- .../plugins/filed/BareosFdPluginPostgres.py | 69 ++++++++++--------- core/src/plugins/filed/bareos-fd-postgres.py | 2 - 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginPostgres.py b/core/src/plugins/filed/BareosFdPluginPostgres.py index 96d78b7b5f2..5e09e5442f5 100644 --- a/core/src/plugins/filed/BareosFdPluginPostgres.py +++ b/core/src/plugins/filed/BareosFdPluginPostgres.py @@ -21,8 +21,8 @@ # # Author: Maik Aussendorf # -# Bareos python plugins class that adds files from a local list to -# the backup fileset +# Bareos python plugins class that performs online backups (full and incremental) +# from Postgres databases. Supports point-in-time (PIT) restores. from bareosfd import * from bareos_fd_consts import bJobMessageType, bFileType, bRCs @@ -96,6 +96,15 @@ def check_options(self, context, mandatory_options=None): result = super(BareosFdPluginPostgres, self).check_options( context, mandatory_options ) + # Accurate may cause problems with plugins + accurate_enabled = GetValue(context, bVariable["bVarAccurate"]) + if accurate_enabled is not None and accurate_enabled != 0: + JobMessage( + context, + bJobMessageType["M_FATAL"], + "start_backup_job: Accurate backup not allowed please disable in Job\n", + ) + return bRCs["bRC_Error"] if not result == bRCs["bRC_OK"]: return result if not self.options["postgresDataDir"].endswith("/"): @@ -117,8 +126,8 @@ def check_options(self, context, mandatory_options=None): self.switchWal = True else: self.switchWal = self.options["switchWal"].lower() == "true" - if 'dbHost' in self.options: - self.dbOpts += " host='%s'" %self.options["dbHost"] + if "dbHost" in self.options: + self.dbOpts += " host='%s'" % self.options["dbHost"] return bRCs["bRC_OK"] def execute_SQL(self, context, sqlStatement): @@ -131,7 +140,7 @@ def execute_SQL(self, context, sqlStatement): bareosfd.JobMessage( context, bJobMessageType["M_ERROR"], - "Query \"%s\" failed: \"%s\"" % (sqlStatement, e.message), + 'Query "%s" failed: "%s"' % (sqlStatement, e.message), ) return False return True @@ -141,9 +150,7 @@ def start_backup_job(self, context): Make filelist in super class and tell Postgres that we start a backup now """ - bareosfd.DebugMessage( - context, 100, "start_backup_job in PostgresPlugin called", - ) + bareosfd.DebugMessage(context, 100, "start_backup_job in PostgresPlugin called") try: self.dbCon = psycopg2.connect( "dbname=%s user=%s %s" % (self.dbname, self.dbuser, self.dbOpts) @@ -151,15 +158,14 @@ def start_backup_job(self, context): self.dbCursor = self.dbCon.cursor() self.dbCursor.execute("SELECT current_setting('server_version_num')") self.pgVersion = int(self.dbCursor.fetchone()[0]) - #bareosfd.DebugMessage( + # bareosfd.DebugMessage( # context, 1, "Connected to Postgres version %d\n" % self.pgVersion, - #) + # ) ## WARNING: JobMessages cause fatal errors at this stage JobMessage( - context, - bJobMessageType["M_INFO"], - "Connected to Postgres version %d\n" - % (self.pgVersion), + context, + bJobMessageType["M_INFO"], + "Connected to Postgres version %d\n" % (self.pgVersion), ) except: bareosfd.JobMessage( @@ -176,7 +182,7 @@ def start_backup_job(self, context): startDir = self.options["postgresDataDir"] self.files_to_backup.append(startDir) bareosfd.DebugMessage( - context, 100, "dataDir: %s\n" % self.options["postgresDataDir"], + context, 100, "dataDir: %s\n" % self.options["postgresDataDir"] ) else: # If level is not Full, we only backup WAL files @@ -249,9 +255,7 @@ def start_backup_job(self, context): # We need a trailing '/' for directories if os.path.isdir(fullName) and not fullName.endswith("/"): fullName += "/" - bareosfd.DebugMessage( - context, 100, "fullName: %s\n" % fullName, - ) + bareosfd.DebugMessage(context, 100, "fullName: %s\n" % fullName) # Usually Bareos takes care about timestamps when doing incremental backups # but here we have to compare against last BackupPostgres timestamp try: @@ -260,7 +264,8 @@ def start_backup_job(self, context): bareosfd.JobMessage( context, bJobMessageType["M_ERROR"], - "Could net get stat-info for file %s: \"%s\"" % (file_to_backup, e.message), + 'Could net get stat-info for file %s: "%s"' + % (file_to_backup, e.message), ) bareosfd.DebugMessage( context, @@ -311,11 +316,11 @@ def start_backup_job(self, context): self.backupStartTime = datetime.datetime.now( tz=dateutil.tz.tzoffset(None, self.tzOffset) ) - if not self.execute_SQL(context, "SELECT pg_start_backup('%s');" % self.backupLabelString): + if not self.execute_SQL( + context, "SELECT pg_start_backup('%s');" % self.backupLabelString + ): bareosfd.JobMessage( - context, - bJobMessageType["M_FATAL"], - 'pg_start_backup statement failed.', + context, bJobMessageType["M_FATAL"], "pg_start_backup statement failed." ) return bRCs["bRC_Error"] results = self.dbCursor.fetchall() @@ -324,9 +329,7 @@ def start_backup_job(self, context): context, 150, "Adding label file %s to fileset\n" % self.labelFileName ) self.files_to_backup.append(self.labelFileName) - bareosfd.DebugMessage( - context, 150, "Filelist: %s\n" % (self.files_to_backup), - ) + bareosfd.DebugMessage(context, 150, "Filelist: %s\n" % (self.files_to_backup)) self.PostgressFullBackupRunning = True return bRCs["bRC_OK"] @@ -341,7 +344,7 @@ def parseBackupLabelFile(self, context): ) for labelItem in labelFile.read().splitlines(): - print labelItem + print(labelItem) k, v = labelItem.split(":", 1) self.labelItems.update({k.strip(): v.strip()}) labelFile.close() @@ -398,7 +401,8 @@ def restore_object_data(self, context, ROP): bareosfd.JobMessage( context, bJobMessageType["M_ERROR"], - "Could net parse restore object json-data \"%s\ / \"%s\"" % (self.row_rop_raw, e.message), + 'Could net parse restore object json-data "%s\ / "%s"' + % (self.row_rop_raw, e.message), ) if "lastBackupStopTime" in self.rop_data[ROP.jobid]: @@ -440,14 +444,11 @@ def closeDbConnection(self, context): + "START WAL LOCATION: %s\n" % self.labelItems["START WAL LOCATION"], ) self.PostgressFullBackupRunning = False - else: + else: bareosfd.JobMessage( - context, - bJobMessageType["M_ERROR"], - 'pg_stop_backup statement failed.', + context, bJobMessageType["M_ERROR"], "pg_stop_backup statement failed." ) - def checkForWalFiles(self, context): """ Look for new WAL files and backup @@ -466,7 +467,7 @@ def checkForWalFiles(self, context): bareosfd.JobMessage( context, bJobMessageType["M_ERROR"], - "Could net get stat-info for file %s: \"%s\"" % (fullPath, e.message), + 'Could net get stat-info for file %s: "%s"' % (fullPath, e.message), ) continue fileMtime = datetime.datetime.fromtimestamp(st.st_mtime) diff --git a/core/src/plugins/filed/bareos-fd-postgres.py b/core/src/plugins/filed/bareos-fd-postgres.py index 5bf353bf4e3..8afb9307345 100644 --- a/core/src/plugins/filed/bareos-fd-postgres.py +++ b/core/src/plugins/filed/bareos-fd-postgres.py @@ -21,8 +21,6 @@ # # Bareos-fd-postgres is a plugin to backup Postgres SQL databases using # BareosFdPluginPostgres. -# TODO: The plugin argument 'filename' is used to read -# all files listed in that file and add it to the fileset # # Author: Maik Aussendorf # From a9523eddd2a10e3aad8504e4a59ffc5b8473b3cf Mon Sep 17 00:00:00 2001 From: aussendorf Date: Wed, 29 Apr 2020 16:23:10 +0200 Subject: [PATCH 9/9] postgres-plugin: normalize postgres lsn to make it fully comparable - output of select pg_current_wal_lsn(); is a hex-number formatted as string of varying length --- .../plugins/filed/BareosFdPluginPostgres.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginPostgres.py b/core/src/plugins/filed/BareosFdPluginPostgres.py index 5e09e5442f5..78d5e4c3f82 100644 --- a/core/src/plugins/filed/BareosFdPluginPostgres.py +++ b/core/src/plugins/filed/BareosFdPluginPostgres.py @@ -96,6 +96,8 @@ def check_options(self, context, mandatory_options=None): result = super(BareosFdPluginPostgres, self).check_options( context, mandatory_options ) + if not result == bRCs["bRC_OK"]: + return result # Accurate may cause problems with plugins accurate_enabled = GetValue(context, bVariable["bVarAccurate"]) if accurate_enabled is not None and accurate_enabled != 0: @@ -105,8 +107,6 @@ def check_options(self, context, mandatory_options=None): "start_backup_job: Accurate backup not allowed please disable in Job\n", ) return bRCs["bRC_Error"] - if not result == bRCs["bRC_OK"]: - return result if not self.options["postgresDataDir"].endswith("/"): self.options["postgresDataDir"] += "/" self.labelFileName = self.options["postgresDataDir"] + "/backup_label" @@ -145,6 +145,16 @@ def execute_SQL(self, context, sqlStatement): return False return True + def formatLSN(self, rawLSN): + """ + Postgres returns LSN in a non-comparable format with varying length, e.g. + 0/3A6A710 + 0/F00000 + We fill the part before the / and after it with leading 0 to get strings with equal length + """ + lsnPre, lsnPost = rawLSN.split("/") + return lsnPre.zfill(8) + "/" + lsnPost.zfill(8) + def start_backup_job(self, context): """ Make filelist in super class and tell Postgres @@ -208,10 +218,10 @@ def start_backup_job(self, context): ) else: if self.execute_SQL(context, getLsnStmt): - currentLSN = self.dbCursor.fetchone()[0].zfill(17) - bareosfd.DebugMessage( + currentLSN = self.formatLSN(self.dbCursor.fetchone()[0]) + bareosfd.JobMessage( context, - 100, + bJobMessageType["M_INFO"], "Current LSN %s, last LSN: %s\n" % (currentLSN, self.lastLSN), ) else: @@ -230,7 +240,7 @@ def start_backup_job(self, context): "Could not switch to next WAL segment\n", ) if self.execute_SQL(context, getLsnStmt): - currentLSN = self.dbCursor.fetchone()[0].zfill(17) + currentLSN = self.formatLSN(self.dbCursor.fetchone()[0]) self.lastLSN = currentLSN # wait some seconds to make sure WAL file gets written time.sleep(10) @@ -344,7 +354,6 @@ def parseBackupLabelFile(self, context): ) for labelItem in labelFile.read().splitlines(): - print(labelItem) k, v = labelItem.split(":", 1) self.labelItems.update({k.strip(): v.strip()}) labelFile.close() @@ -433,7 +442,7 @@ def closeDbConnection(self, context): # self.backupStartTime = self.dbCursor.fetchone()[0] # Tell Postgres we are done if self.execute_SQL(context, "SELECT pg_stop_backup();"): - self.lastLSN = self.dbCursor.fetchone()[0].zfill(17) + self.lastLSN = self.formatLSN(self.dbCursor.fetchone()[0]) self.lastBackupStopTime = int(time.time()) results = self.dbCursor.fetchall() bareosfd.JobMessage(